Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_kafka_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Still waiting to schedule task All nodes of label β€˜pingcap_tiflow_pull_cdc_integration_kafka_test_1732-d7s7d’ are offline Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-bhgll Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-d30jp Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-0zkqm Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-d0md4 ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-bhgll io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-bhgll] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-rkhlr ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-d30jp io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-d30jp] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-0zkqm io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-0zkqm] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-d0md4 io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-d0md4] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-rkhlr io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-rkhlr] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-h3tfl Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-g5cmr Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-cwrtl Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-l085f ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-h3tfl io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-h3tfl] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-sbcr6 ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-g5cmr io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-g5cmr] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-610fq ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-cwrtl io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-cwrtl] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-l085f io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-l085f] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-d7s7d-7ljrm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c5d323d2efac723d607cd0e1c29cd3aeaba836b9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-d7s7d" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-list --no-walk cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=b721e885-3d00-40f3-86c5-c83b40f9bbae BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-d7s7d GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-236df335481f9578f70eb859f68d5ceead3aa27f6c9385fda1ec4c08661c0305 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm pingcap_tiflow_pull_cdc_integration_kafka_test_1732-d7s7d GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.0' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build2673107364=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-6mczm bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 54342 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 4e1d1462c55456b8 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 12 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 26 11:17:33 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, errors.toml, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-d0329d7-23327bc) 197093376 bytes in 0.89 secs (220695336 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/.git/ .git HEAD is now at cf8946a3a Merge commit '23327bce12e22486d293c05f3e682e66d2bbc2ae' into HEAD POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/10919/head -> origin/pr/10919/head Warning: you are leaving 1 commit behind, not connected to any of your branches: cf8946a3a Merge commit '23327bce12e22486d293c05f3e682e66d2bbc2ae' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch cf8946a3a HEAD is now at d0329d7f1 ddl_puller (ticdc): handle dorp pk/uk ddl correctly (#10965) 🚧 Checkouting to base SHA:d0329d7f1ca9a1d0de81a565051a09fe7e9231bd... HEAD is now at d0329d7f1 ddl_puller (ticdc): handle dorp pk/uk ddl correctly (#10965) βœ… Checked. πŸŽ‰ 🧾 HEAD info: d0329d7f1ca9a1d0de81a565051a09fe7e9231bd d0329d7f1 ddl_puller (ticdc): handle dorp pk/uk ddl correctly (#10965) 4fd08424c simple (ticdc): reduce map access and memory allocation to save some cpu (#10942) 830e1622c Makefile: bump version to v8.2.0-master (#10893) 🚧 Pre-merge heads of pull requests to base SHA: d0329d7f1ca9a1d0de81a565051a09fe7e9231bd ... Auto-merging cdc/model/sink.go Merge made by the 'ort' strategy. cdc/model/kv.go | 5 ++ cdc/model/sink.go | 11 ++- cdc/model/sink_test.go | 9 ++- cdc/processor/processor.go | 21 ++++- cdc/processor/sinkmanager/manager.go | 5 ++ cdc/processor/sourcemanager/manager.go | 66 +++++++++++++++- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 ++++++++++------------ cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- errors.toml | 5 ++ pkg/applier/redo_test.go | 7 +- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 +++++ .../conf/diff_config.toml | 29 +++++++ .../changefeed_dup_error_restart/conf/workload | 13 ++++ .../changefeed_dup_error_restart/run.sh | 54 +++++++++++++ tests/integration_tests/run_group.sh | 2 +- 17 files changed, 279 insertions(+), 70 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh 🧾 Pre-merged result: b1f701782ea718940d93ff862c3bb61a7a85c96f b1f701782 Merge commit '23327bce12e22486d293c05f3e682e66d2bbc2ae' into HEAD 23327bce1 fix unit test 0ec367d2d fix verify βœ… Pre merged πŸŽ‰ βœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-d0329d7-23327bc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 840 0 --:--:-- --:--:-- --:--:-- 854 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1139 0 --:--:-- --:--:-- --:--:-- 1171 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 795 0 --:--:-- --:--:-- --:--:-- 803 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1580 0 --:--:-- --:--:-- --:--:-- 1640 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/944fff519c90039747affb94067439ff9541f2fb/centos7/tidb-server.tar.gz 2024-04-26 19:18:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/944fff519c90039747affb94067439ff9541f2fb/centos7/tidb-server.tar.gz [536672508/536672508] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/50c80407cd57e96a85452dd1601fcb41c1f263cf/centos7/pd-server.tar.gz 2024-04-26 19:18:17 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/50c80407cd57e96a85452dd1601fcb41c1f263cf/centos7/pd-server.tar.gz [187359310/187359310] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/2118288ceb3f52a4a8431acba4fc2c560def0732/centos7/tikv-server.tar.gz 2024-04-26 19:18:55 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/2118288ceb3f52a4a8431acba4fc2c560def0732/centos7/tikv-server.tar.gz [918823274/918823274] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/37701038fbd30f5eafac11c3ad180f14c6dcab9e/centos7/tiflash.tar.gz 2024-04-26 19:19:16 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/37701038fbd30f5eafac11c3ad180f14c6dcab9e/centos7/tiflash.tar.gz [456037936/456037936] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-04-26 19:19:21 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-04-26 19:19:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-04-26 19:19:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-04-26 19:19:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-04-26 19:19:29 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-04-26 19:19:40 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 26 19:19 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:19 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 19:19 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 19:19 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:07 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 26 19:07 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:07 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 26 19:07 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 26 18:25 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 26 19:08 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 26 10:19 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 10:18 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 10:18 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 10:18 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 26 10:18 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 26 10:18 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 26 10:19 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 10:18 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 10:19 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M Apr 26 17:54 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 26 19:07 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 26 06:00 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 26 10:19 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 26 19:19 . drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:19 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 19:19 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 19:19 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:07 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 26 19:07 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:07 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 26 19:07 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 26 18:25 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 26 19:08 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 26 10:19 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 10:18 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 10:18 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 10:18 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 26 10:18 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 26 10:18 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 26 10:19 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 10:18 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 10:19 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M Apr 26 17:54 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 26 19:07 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 26 06:00 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 26 10:19 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/cdc-integration-test/rev-d0329d7-23327bc) 1193872896 bytes in 3.34 secs (357385067 bytes/sec) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 19:19 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:19 .. -rwxr-xr-x. 1 jenkins jenkins 220M Apr 26 17:57 cdc -rwxr-xr-x. 1 jenkins jenkins 363M Apr 26 17:57 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M Apr 26 17:51 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 17:52 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 17:51 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M Apr 26 17:52 oauth2-server + '[' -f ./bin/cdc ']' + '[' -f ./bin/cdc_kafka_consumer ']' + '[' -f ./bin/cdc_storage_consumer ']' + '[' -f ./bin/cdc.test ']' + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 19:19 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:19 .. -rwxr-xr-x. 1 jenkins jenkins 220M Apr 26 17:57 cdc -rwxr-xr-x. 1 jenkins jenkins 363M Apr 26 17:57 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M Apr 26 17:51 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 17:52 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 17:51 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M Apr 26 17:52 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-37-gcf8946a3a Git Commit Hash: cf8946a3a9cb6bf28577ea67d4848a4fad982d67 Git Branch: HEAD UTC Build Time: 2024-04-26 09:51:33 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-d0329d7-23327bc already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 26 19:19 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:19 .. drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 19:19 bin -rwxr-xr-x. 1 jenkins jenkins 220M Apr 26 17:57 cdc -rwxr-xr-x. 1 jenkins jenkins 363M Apr 26 17:57 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M Apr 26 17:51 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 17:52 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 17:51 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:19 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 26 19:19 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 19:19 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 19:19 jq drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:19 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:19 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Apr 26 19:19 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 26 19:19 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Apr 26 19:19 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:19 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 26 19:19 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 26 19:19 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 26 19:19 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Apr 26 19:19 minio -rwxr-xr-x. 1 jenkins jenkins 12M Apr 26 17:52 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M Apr 26 19:19 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 19:19 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 19:19 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 19:19 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 26 19:19 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 26 19:19 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 26 19:19 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 19:19 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:19 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 26 19:19 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Apr 26 19:19 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M Apr 26 19:19 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 26 19:19 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 26 19:19 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 26 19:19 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 75.00 secs (49586356 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76 [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64 [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-96drt-phgx0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "400deec9e56294f8b7af7e2c5f35dac2a7312e9d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-96drt" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63 [Pipeline] { [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@66b83074; decorates RemoteLauncher[hudson.remoting.Channel@70c9d7d0:JNLP4-connect connection from 10.233.100.55/10.233.100.55:38000] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-n5sw2-x3jf8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e3da65128c714da7873c31964367b7bf181b7a17" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-n5sw2" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8 The recommended git tool is: git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-09rd0-vw4lm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9edc1ef29cbd960f8faa67d362898fbfeabe8bac" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-09rd0" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-tl08z-wdd4x --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "adef5747b92fdedd0a7d6876dc4cf2276c352061" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-tl08z" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" [Pipeline] checkout Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-9ctfl-9jfdt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "38a6e63b77261699daf6396cd09843e0d2db493c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-9ctfl" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5d45f66; decorates RemoteLauncher[hudson.remoting.Channel@21df0cf4:JNLP4-connect connection from 10.233.70.114/10.233.70.114:38926] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-j4c34-742s9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2114cf9e858e574f0d4660a2da85073d7066d597" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-j4c34" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@41e5698d; decorates RemoteLauncher[hudson.remoting.Channel@1be9ceb5:JNLP4-connect connection from 10.233.97.201/10.233.97.201:40164] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3ad80188; decorates RemoteLauncher[hudson.remoting.Channel@77381e75:JNLP4-connect connection from 10.233.67.218/10.233.67.218:42788] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m [Pipeline] node Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1cc36d32; decorates RemoteLauncher[hudson.remoting.Channel@57e9dec7:JNLP4-connect connection from 10.233.108.30/10.233.108.30:57570] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Commit message: "feat(tidb): test flashbacktest package (#2942)" [Pipeline] withEnv [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@251a5b9; decorates RemoteLauncher[hudson.remoting.Channel@24367df6:JNLP4-connect connection from 10.233.97.1/10.233.97.1:34748] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-ts800 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-ljzmt-fqz9d --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7d399d3aa358bb106b567fdcc2379d260d80535d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-ljzmt" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-llwp6-qq59l --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9d2735d3431076e6483c0fb41cc137c68c6b8569" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-llwp6" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-4h6bf-vrk5b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c6bc70c9db421a82ec4c0638ea27ea04a7a98d9a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-4h6bf" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-wrgs8-f19b8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0bf08fb4d01e50ae733899332ce7f57ba6d4bcdc" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-wrgs8" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-sbcr6 io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-sbcr6] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2q1jv-sj793 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9f3a772a1622bd92e6bd50908b8a6fad17dbfd60" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2q1jv" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63’ is offline Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8’ is offline Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m’ is offline Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 11.55 secs (322039184 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-ts800’ is offline [Pipeline] node [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] checkout [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0 [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@58fd68c6; decorates RemoteLauncher[hudson.remoting.Channel@6703ca3:JNLP4-connect connection from 10.233.105.142/10.233.105.142:43222] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5c8543da; decorates RemoteLauncher[hudson.remoting.Channel@68b18469:JNLP4-connect connection from 10.233.72.140/10.233.72.140:50088] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1aade21e; decorates RemoteLauncher[hudson.remoting.Channel@2bfa8c01:JNLP4-connect connection from 10.233.86.64/10.233.86.64:58550] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@79585560; decorates RemoteLauncher[hudson.remoting.Channel@2714f310:JNLP4-connect connection from 10.233.106.4/10.233.106.4:34498] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7b32a513; decorates RemoteLauncher[hudson.remoting.Channel@13b01c51:JNLP4-connect connection from 10.233.100.35/10.233.100.35:34100] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-jb3h1-4w9rb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "841b7acf8b862b6a0b20db0eaa99e5708ae0b1f5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-jb3h1" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 8.84 secs (420877645 bytes/sec) [Pipeline] { [Pipeline] cache Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0’ is offline Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm’ is offline Still waiting to schedule task β€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b’ is offline Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-0g9sx-5fln6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "63377ea1ee327eb1e1607e6b22baca38e01c01e3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-0g9sx" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 19.62 secs (189590846 bytes/sec) [Pipeline] { [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-fq2th-ftfhz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f9d94e56419415f75b820c42fae89241000da6c0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-fq2th" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 10.49 secs (354500545 bytes/sec) [Pipeline] { [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2jnjw-wv92h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "21bc18a200ec63cde5a53e99a1a2d9229b7a6bc0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2jnjw" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 19.29 secs (192748623 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 6.27 secs (593215550 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] } [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] { [Pipeline] // timeout [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1db08509; decorates RemoteLauncher[hudson.remoting.Channel@1248bdf3:JNLP4-connect connection from 10.233.127.68/10.233.127.68:36722] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@38ce3742; decorates RemoteLauncher[hudson.remoting.Channel@33ec5330:JNLP4-connect connection from 10.233.73.223/10.233.73.223:36690] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git [Pipeline] // timeout [Pipeline] } Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@756e6d54; decorates RemoteLauncher[hudson.remoting.Channel@2c86d6ba:JNLP4-connect connection from 10.233.66.40/10.233.66.40:42698] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@284f1e01; decorates RemoteLauncher[hudson.remoting.Channel@2a9569a7:JNLP4-connect connection from 10.233.67.14/10.233.67.14:44668] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=faaa35c2-6212-40a0-a6c5-b1e40c7d7d55 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-96drt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp pingcap_tiflow_pull_cdc_integration_kafka_test_1732-96drt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:00 CST 2024] <<<<<< run test case http_api success! >>>>>> [Pipeline] // container [Pipeline] // container [Pipeline] // container Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=32569c56-285e-490c-9389-549c76c48f14 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-n5sw2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-n5sw2 pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=d204761d-d6c8-48db-834d-856eb8712230 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-09rd0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-09rd0 pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:01 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=c3d334c9-d67a-4fd8-b797-58148c484ad5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-tl08z GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-tl08z pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // container [Pipeline] sh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow Avoid second fetch [Pipeline] { Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=4873f5fe-4b3f-435b-b725-b74a5927355b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-j4c34 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z pingcap_tiflow_pull_cdc_integration_kafka_test_1732-j4c34 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=abeb43bd-8bae-4445-b507-e9907b145bba BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-9ctfl GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-9ctfl pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:03 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:07 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:08 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D0B090B4E7D8 < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:23:09 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri Apr 26 19:23:10 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: kafka... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: kafka... <<================= kafka downstream isn't support syncpoint record [Fri Apr 26 19:23:12 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:13 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:14 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:15 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6894c1340002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:1286, start at 2024-04-26 19:23:15.920986573 +0800 CST m=+5.155275913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:15.929 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:15.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:15.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6894c1340002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:1286, start at 2024-04-26 19:23:15.920986573 +0800 CST m=+5.155275913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:15.929 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:15.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:15.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6894c1ec000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:1364, start at 2024-04-26 19:23:15.975444155 +0800 CST m=+5.158565362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:15.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:15.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:15.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6894ac64000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:1412, start at 2024-04-26 19:23:14.600081682 +0800 CST m=+5.222319638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:14.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:14.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:14.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6894ac64000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:1412, start at 2024-04-26 19:23:14.600081682 +0800 CST m=+5.222319638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:14.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:14.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:14.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6894ad2c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:1491, start at 2024-04-26 19:23:14.666185457 +0800 CST m=+5.231641956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:14.672 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:14.635 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:14.635 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2800.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 26 19:23:19 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28352837.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449349051853832193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349051853832193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:23:20 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.28362838.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:21 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:20 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:22 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:22 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c553bb87-1898-424c-8f32-5a762cbb7884 {"id":"c553bb87-1898-424c-8f32-5a762cbb7884","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130600} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2502d8cd c553bb87-1898-424c-8f32-5a762cbb7884 /tidb/cdc/default/default/upstream/7362134811956759055 {"id":7362134811956759055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c553bb87-1898-424c-8f32-5a762cbb7884 {"id":"c553bb87-1898-424c-8f32-5a762cbb7884","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130600} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2502d8cd c553bb87-1898-424c-8f32-5a762cbb7884 /tidb/cdc/default/default/upstream/7362134811956759055 {"id":7362134811956759055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c553bb87-1898-424c-8f32-5a762cbb7884 {"id":"c553bb87-1898-424c-8f32-5a762cbb7884","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130600} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2502d8cd c553bb87-1898-424c-8f32-5a762cbb7884 /tidb/cdc/default/default/upstream/7362134811956759055 {"id":7362134811956759055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c62806a7-b688-4454-b214-11181addcd6c Info: {"upstream_id":7362134811956759055,"namespace":"default","id":"c62806a7-b688-4454-b214-11181addcd6c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:23:23.075511317+08:00","start_ts":449349051936931841,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349051936931841,"checkpoint_ts":449349051936931841,"checkpoint_time":"2024-04-26 19:23:19.735"} [Fri Apr 26 19:23:23 CST 2024] <<<<<< START kafka consumer in charset_gbk case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68951dac0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:1420, start at 2024-04-26 19:23:21.836185712 +0800 CST m=+5.774189123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:21.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:21.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:21.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68951dac0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:1420, start at 2024-04-26 19:23:21.836185712 +0800 CST m=+5.774189123 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:21.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:21.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:21.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689515c40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:1509, start at 2024-04-26 19:23:21.333704201 +0800 CST m=+5.214627621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:21.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:21.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:21.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:23 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c988ba3-4918-46e8-8019-6776dab18e43 {"id":"3c988ba3-4918-46e8-8019-6776dab18e43","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130601} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25064acf 3c988ba3-4918-46e8-8019-6776dab18e43 /tidb/cdc/default/default/upstream/7362134813721386676 {"id":7362134813721386676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c988ba3-4918-46e8-8019-6776dab18e43 {"id":"3c988ba3-4918-46e8-8019-6776dab18e43","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130601} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25064acf 3c988ba3-4918-46e8-8019-6776dab18e43 /tidb/cdc/default/default/upstream/7362134813721386676 {"id":7362134813721386676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c988ba3-4918-46e8-8019-6776dab18e43 {"id":"3c988ba3-4918-46e8-8019-6776dab18e43","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130601} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25064acf 3c988ba3-4918-46e8-8019-6776dab18e43 /tidb/cdc/default/default/upstream/7362134813721386676 {"id":7362134813721386676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2893.out cli changefeed create --start-ts=449349051853832193 '--sink-uri=kafka://127.0.0.1:9092/ticdc-row-format-test-30701?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: d70b1f7e-557d-4ec2-9556-815a90bafc3a Info: {"upstream_id":7362134813721386676,"namespace":"default","id":"d70b1f7e-557d-4ec2-9556-815a90bafc3a","sink_uri":"kafka://127.0.0.1:9092/ticdc-row-format-test-30701?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:23:24.435208052+08:00","start_ts":449349051853832193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349051853832193,"checkpoint_ts":449349051853832193,"checkpoint_time":"2024-04-26 19:23:19.418"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:24 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:26 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> + set +x [Fri Apr 26 19:23:25 CST 2024] <<<<<< START kafka consumer in row_format case >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 21.01 secs (176984680 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend [Fri Apr 26 19:23:29 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.29772979.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D0B5298431EA < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:23:29 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri Apr 26 19:23:29 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d020999-10ba-4dd8-94b8-b786244a5b17 {"id":"2d020999-10ba-4dd8-94b8-b786244a5b17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2518ea96 2d020999-10ba-4dd8-94b8-b786244a5b17 /tidb/cdc/default/default/upstream/7362134833528840843 {"id":7362134833528840843,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d020999-10ba-4dd8-94b8-b786244a5b17 {"id":"2d020999-10ba-4dd8-94b8-b786244a5b17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2518ea96 2d020999-10ba-4dd8-94b8-b786244a5b17 /tidb/cdc/default/default/upstream/7362134833528840843 {"id":7362134833528840843,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d020999-10ba-4dd8-94b8-b786244a5b17 {"id":"2d020999-10ba-4dd8-94b8-b786244a5b17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2518ea96 2d020999-10ba-4dd8-94b8-b786244a5b17 /tidb/cdc/default/default/upstream/7362134833528840843 {"id":7362134833528840843,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:23:32 CST 2024] <<<<<< START kafka consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 829f0933-a163-4802-9af0-81c3bb3c0273 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=829f0933-a163-4802-9af0-81c3bb3c0273 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 829f0933-a163-4802-9af0-81c3bb3c0273 -s + info='{ "upstream_id": 7362134833528840843, "namespace": "default", "id": "829f0933-a163-4802-9af0-81c3bb3c0273", "state": "normal", "checkpoint_tso": 449349055213731841, "checkpoint_time": "2024-04-26 19:23:32.235", "error": null }' + echo '{ "upstream_id": 7362134833528840843, "namespace": "default", "id": "829f0933-a163-4802-9af0-81c3bb3c0273", "state": "normal", "checkpoint_tso": 449349055213731841, "checkpoint_time": "2024-04-26 19:23:32.235", "error": null }' { "upstream_id": 7362134833528840843, "namespace": "default", "id": "829f0933-a163-4802-9af0-81c3bb3c0273", "state": "normal", "checkpoint_tso": 449349055213731841, "checkpoint_time": "2024-04-26 19:23:32.235", "error": null } ++ echo '{' '"upstream_id":' 7362134833528840843, '"namespace":' '"default",' '"id":' '"829f0933-a163-4802-9af0-81c3bb3c0273",' '"state":' '"normal",' '"checkpoint_tso":' 449349055213731841, '"checkpoint_time":' '"2024-04-26' '19:23:32.235",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362134833528840843, '"namespace":' '"default",' '"id":' '"829f0933-a163-4802-9af0-81c3bb3c0273",' '"state":' '"normal",' '"checkpoint_tso":' 449349055213731841, '"checkpoint_time":' '"2024-04-26' '19:23:32.235",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-whfsf-k8v87 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c983c941e3264a16a7ecc07c5baa9ecbe41482e9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-whfsf" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:23:35 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... check diff successfully find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:23:39 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68962c84000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:1725, start at 2024-04-26 19:23:39.181060064 +0800 CST m=+5.168603358 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:39.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:39.169 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:39.169 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68962c84000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:1725, start at 2024-04-26 19:23:39.181060064 +0800 CST m=+5.168603358 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:39.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:39.169 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:39.169 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68962d340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:1809, start at 2024-04-26 19:23:39.241416968 +0800 CST m=+5.174893155 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:39.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:39.213 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:39.213 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 12.98 secs (286590471 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark exists check table exists success check diff successfully start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... table row_format.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:23:44 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:23:44 CST 2024] <<<<<< START cdc server in kafka_simple_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.32643266.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:47 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8182cec-4f0e-413e-91e1-893e98869cb9 {"id":"c8182cec-4f0e-413e-91e1-893e98869cb9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130624} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a255f2bcd c8182cec-4f0e-413e-91e1-893e98869cb9 /tidb/cdc/default/default/upstream/7362134914905623385 {"id":7362134914905623385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8182cec-4f0e-413e-91e1-893e98869cb9 {"id":"c8182cec-4f0e-413e-91e1-893e98869cb9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130624} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a255f2bcd c8182cec-4f0e-413e-91e1-893e98869cb9 /tidb/cdc/default/default/upstream/7362134914905623385 {"id":7362134914905623385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8182cec-4f0e-413e-91e1-893e98869cb9 {"id":"c8182cec-4f0e-413e-91e1-893e98869cb9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130624} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a255f2bcd c8182cec-4f0e-413e-91e1-893e98869cb9 /tidb/cdc/default/default/upstream/7362134914905623385 {"id":7362134914905623385,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.3334.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-29735?protocol=simple' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/conf/changefeed.toml -c simple-basic Create changefeed successfully! ID: simple-basic Info: {"upstream_id":7362134914905623385,"namespace":"default","id":"simple-basic","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-29735?protocol=simple","create_time":"2024-04-26T19:23:47.94055891+08:00","start_ts":449349059285876742,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":5,"send_bootstrap_in_msg_count":100,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349059285876742,"checkpoint_ts":449349059285876742,"checkpoint_time":"2024-04-26 19:23:47.769"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896f4a80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:1917, start at 2024-04-26 19:23:52.016373806 +0800 CST m=+7.487147807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:52.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:52.028 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:52.028 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 7-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 8.69 secs (427843125 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896fff0001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:2112, start at 2024-04-26 19:23:52.745025128 +0800 CST m=+5.146316727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:52.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:52.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:52.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896fff0001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:2112, start at 2024-04-26 19:23:52.745025128 +0800 CST m=+5.146316727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:52.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:52.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:52.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:23:54 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896f4a80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:1917, start at 2024-04-26 19:23:52.016373806 +0800 CST m=+7.487147807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:52.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:52.028 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:52.028 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896d41c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:2000, start at 2024-04-26 19:23:49.935031619 +0800 CST m=+5.344637542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:49.942 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:49.945 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:49.945 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table row_format.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689711180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:2185, start at 2024-04-26 19:23:53.834561877 +0800 CST m=+6.183103699 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:53.840 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:53.798 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:53.798 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table row_format.finish_mark exists check diff successfully [Fri Apr 26 19:23:56 CST 2024] <<<<<< START cdc server in kafka_big_messages case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages.33933395.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [Fri Apr 26 19:23:58 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.36273629.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... table test.finish_mark_for_ddl not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:23:59 CST 2024] <<<<<< run test case row_format success! >>>>>> table test.finish_mark_for_ddl not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:00 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f0f453c-c59a-4b87-9abf-7763ff4f5c0d {"id":"3f0f453c-c59a-4b87-9abf-7763ff4f5c0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a258804d4 3f0f453c-c59a-4b87-9abf-7763ff4f5c0d /tidb/cdc/default/default/upstream/7362134953948599639 {"id":7362134953948599639,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f0f453c-c59a-4b87-9abf-7763ff4f5c0d {"id":"3f0f453c-c59a-4b87-9abf-7763ff4f5c0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a258804d4 3f0f453c-c59a-4b87-9abf-7763ff4f5c0d /tidb/cdc/default/default/upstream/7362134953948599639 {"id":7362134953948599639,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f0f453c-c59a-4b87-9abf-7763ff4f5c0d {"id":"3f0f453c-c59a-4b87-9abf-7763ff4f5c0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a258804d4 3f0f453c-c59a-4b87-9abf-7763ff4f5c0d /tidb/cdc/default/default/upstream/7362134953948599639 {"id":7362134953948599639,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: f318a112-e3b9-4aa4-8881-6eb6bcc42de1 Info: {"upstream_id":7362134953948599639,"namespace":"default","id":"f318a112-e3b9-4aa4-8881-6eb6bcc42de1","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-04-26T19:24:00.290471295+08:00","start_ts":449349061686853633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349061686853633,"checkpoint_ts":449349061686853633,"checkpoint_time":"2024-04-26 19:23:56.928"} [Fri Apr 26 19:24:00 CST 2024] <<<<<< START kafka consumer in kafka_big_messages case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:01 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/542a7ee7-27c3-4b6a-807d-22bd9e81d887 {"id":"542a7ee7-27c3-4b6a-807d-22bd9e81d887","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130638} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a259885d3 542a7ee7-27c3-4b6a-807d-22bd9e81d887 /tidb/cdc/default/default/upstream/7362134968330779533 {"id":7362134968330779533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/542a7ee7-27c3-4b6a-807d-22bd9e81d887 {"id":"542a7ee7-27c3-4b6a-807d-22bd9e81d887","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130638} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a259885d3 542a7ee7-27c3-4b6a-807d-22bd9e81d887 /tidb/cdc/default/default/upstream/7362134968330779533 {"id":7362134968330779533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/542a7ee7-27c3-4b6a-807d-22bd9e81d887 {"id":"542a7ee7-27c3-4b6a-807d-22bd9e81d887","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130638} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a259885d3 542a7ee7-27c3-4b6a-807d-22bd9e81d887 /tidb/cdc/default/default/upstream/7362134968330779533 {"id":7362134968330779533,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 606c7f88-102f-4860-a31c-6ec0db2a9ac8 Info: {"upstream_id":7362134968330779533,"namespace":"default","id":"606c7f88-102f-4860-a31c-6ec0db2a9ac8","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-04-26T19:24:01.374400883+08:00","start_ts":449349061968134145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349061968134145,"checkpoint_ts":449349061968134145,"checkpoint_time":"2024-04-26 19:23:58.001"} [Fri Apr 26 19:24:01 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 table test.finish_mark_for_ddl not exists for 3-th check, retry later Verifying downstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 7.21 secs (515695525 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark_for_ddl not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark_for_ddl exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.3446.out cli changefeed pause -c simple-basic PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table kafka_big_messages.test exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.3483.out cli changefeed resume -c simple-basic table kafka_big_messages.test exists check diff failed 1-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6897fc3c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:4474, start at 2024-04-26 19:24:08.862377333 +0800 CST m=+5.316597868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:08.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:08.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:08.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6897fc3c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:4474, start at 2024-04-26 19:24:08.862377333 +0800 CST m=+5.316597868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:08.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:08.847 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:08.847 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6897fcf00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:4562, start at 2024-04-26 19:24:08.932682232 +0800 CST m=+5.332856575 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:08.941 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:08.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:08.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 7.42 secs (501371306 bytes/sec) [Pipeline] { cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:24:12 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh check diff successfully [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 wait process cdc.test exit for 1-th time... [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68980f440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:4076, start at 2024-04-26 19:24:10.103480434 +0800 CST m=+5.219994653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:10.110 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:10.114 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:10.114 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68980f440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:4076, start at 2024-04-26 19:24:10.103480434 +0800 CST m=+5.219994653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:10.110 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:10.114 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:10.114 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68980fe40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:4149, start at 2024-04-26 19:24:10.125247194 +0800 CST m=+5.194978511 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:10.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:10.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:10.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + nc localhost 2181 + echo dump + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 + sleep 10 [Pipeline] sh wait process cdc.test exit for 2-th time... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [Fri Apr 26 19:24:13 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.59095911.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:24:14 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> [Fri Apr 26 19:24:15 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.55985600.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:16 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd8b894b-8758-477e-96a6-1ed63964c13b {"id":"cd8b894b-8758-477e-96a6-1ed63964c13b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130654} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d66bcd cd8b894b-8758-477e-96a6-1ed63964c13b /tidb/cdc/default/default/upstream/7362135041331549037 {"id":7362135041331549037,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd8b894b-8758-477e-96a6-1ed63964c13b {"id":"cd8b894b-8758-477e-96a6-1ed63964c13b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130654} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d66bcd cd8b894b-8758-477e-96a6-1ed63964c13b /tidb/cdc/default/default/upstream/7362135041331549037 {"id":7362135041331549037,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd8b894b-8758-477e-96a6-1ed63964c13b {"id":"cd8b894b-8758-477e-96a6-1ed63964c13b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130654} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d66bcd cd8b894b-8758-477e-96a6-1ed63964c13b /tidb/cdc/default/default/upstream/7362135041331549037 {"id":7362135041331549037,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:24:17 CST 2024] <<<<<< START kafka consumer in changefeed_reconstruct case >>>>>> ***************** properties ***************** "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "workload"="core" "readallfields"="true" "insertproportion"="0" "recordcount"="50" "scanproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "operationcount"="0" "threadcount"="4" "readproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 18.546404ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3349.7, Avg(us): 1443, Min(us): 967, Max(us): 4159, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print}+ grep -F -w /brokers/ids/1 /brokers/ids/1 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/707b40e2-5de1-436a-9f88-50d8cc3dab9f {"id":"707b40e2-5de1-436a-9f88-50d8cc3dab9f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7a8cc 707b40e2-5de1-436a-9f88-50d8cc3dab9f /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/707b40e2-5de1-436a-9f88-50d8cc3dab9f {"id":"707b40e2-5de1-436a-9f88-50d8cc3dab9f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7a8cc 707b40e2-5de1-436a-9f88-50d8cc3dab9f /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/707b40e2-5de1-436a-9f88-50d8cc3dab9f {"id":"707b40e2-5de1-436a-9f88-50d8cc3dab9f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130655} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7a8cc 707b40e2-5de1-436a-9f88-50d8cc3dab9f /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5655.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7362135042907990264,"namespace":"default","id":"ddl-manager","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:18.732740605+08:00","start_ts":449349067371708419,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349067371708419,"checkpoint_ts":449349067371708419,"checkpoint_time":"2024-04-26 19:24:18.614"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { table changefeed_reconstruct.usertable not exists for 2-th check, retry later [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] { Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage + set +x [Fri Apr 26 19:24:20 CST 2024] <<<<<< START kafka consumer in ddl_manager case >>>>>> [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // container [Pipeline] // container [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@52f7f38f; decorates RemoteLauncher[hudson.remoting.Channel@74a8bddb:JNLP4-connect connection from 10.233.108.253/10.233.108.253:46726] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G09 [Pipeline] // container [Pipeline] sh Run cases: gc_safepoint changefeed_pause_resume cli savepoint synced_status PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=295d8733-7cf0-4da4-bf0b-4a22ada970b5 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-ljzmt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-ljzmt pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table changefeed_reconstruct.usertable exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=2339a1a4-3f3d-4bd3-ae0e-d8c3b6787210 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2q1jv GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2q1jv GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: kafka... <<================= wait process 5914 exit for 1-th time... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=a0c0cdad-d51d-4133-9664-de13a149488b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-wrgs8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk pingcap_tiflow_pull_cdc_integration_kafka_test_1732-wrgs8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache wait process 5914 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5914) - No such process wait process 5914 exit for 3-th time... process 5914 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 19:24:23 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.62046206.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc localhost 2181 + echo dump + awk {$1=$1;print} + grep brokers + grep -F -w /brokers/ids/1 /brokers/ids/1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=e4984c49-bd73-48cd-8a20-2450deda30ed BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-4h6bf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-4h6bf pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898f744000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:4139, start at 2024-04-26 19:24:24.92514436 +0800 CST m=+5.124357648 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:24.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:24.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:24.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898f744000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:4139, start at 2024-04-26 19:24:24.92514436 +0800 CST m=+5.124357648 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:24.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:24.913 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:24.913 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898f7f80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:4221, start at 2024-04-26 19:24:24.99907142 +0800 CST m=+5.147647986 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:25.007 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:25.008 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:25.008 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Commit message: "feat(tidb): test flashbacktest package (#2942)" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/d529f437-0ba1-4d6d-9053-868442981d74 {UpstreamID:7362135041331549037 Namespace:default ID:d529f437-0ba1-4d6d-9053-868442981d74 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-30808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:17.186314291 +0800 CST StartTs:449349066961190916 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035715f0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349066987143173} {CheckpointTs:449349067891539971 MinTableBarrierTs:449349069202522117 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349067891539971, checkpointTs: 449349067891539971, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bf2f405-e714-4b5a-82b9-a0352c0528cd {"id":"3bf2f405-e714-4b5a-82b9-a0352c0528cd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d66c63 3bf2f405-e714-4b5a-82b9-a0352c0528cd /tidb/cdc/default/default/changefeed/info/d529f437-0ba1-4d6d-9053-868442981d74 {"upstream-id":7362135041331549037,"namespace":"default","changefeed-id":"d529f437-0ba1-4d6d-9053-868442981d74","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-30808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:17.186314291+08:00","start-ts":449349066961190916,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349066987143173} /tidb/cdc/default/default/changefeed/status/d529f437-0ba1-4d6d-9053-868442981d74 {"checkpoint-ts":449349067891539971,"min-table-barrier-ts":449349069202522117,"admin-job-type":0} /tidb/cdc/default/default/task/position/3bf2f405-e714-4b5a-82b9-a0352c0528cd/d529f437-0ba1-4d6d-9053-868442981d74 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135041331549037 {"id":7362135041331549037,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d529f437-0ba1-4d6d-9053-868442981d74 {UpstreamID:7362135041331549037 Namespace:default ID:d529f437-0ba1-4d6d-9053-868442981d74 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-30808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:17.186314291 +0800 CST StartTs:449349066961190916 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035715f0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349066987143173} {CheckpointTs:449349067891539971 MinTableBarrierTs:449349069202522117 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349067891539971, checkpointTs: 449349067891539971, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bf2f405-e714-4b5a-82b9-a0352c0528cd {"id":"3bf2f405-e714-4b5a-82b9-a0352c0528cd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d66c63 3bf2f405-e714-4b5a-82b9-a0352c0528cd /tidb/cdc/default/default/changefeed/info/d529f437-0ba1-4d6d-9053-868442981d74 {"upstream-id":7362135041331549037,"namespace":"default","changefeed-id":"d529f437-0ba1-4d6d-9053-868442981d74","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-30808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:17.186314291+08:00","start-ts":449349066961190916,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349066987143173} /tidb/cdc/default/default/changefeed/status/d529f437-0ba1-4d6d-9053-868442981d74 {"checkpoint-ts":449349067891539971,"min-table-barrier-ts":449349069202522117,"admin-job-type":0} /tidb/cdc/default/default/task/position/3bf2f405-e714-4b5a-82b9-a0352c0528cd/d529f437-0ba1-4d6d-9053-868442981d74 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7362135041331549037 {"id":7362135041331549037,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d529f437-0ba1-4d6d-9053-868442981d74 {UpstreamID:7362135041331549037 Namespace:default ID:d529f437-0ba1-4d6d-9053-868442981d74 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-30808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:17.186314291 +0800 CST StartTs:449349066961190916 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035715f0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349066987143173} {CheckpointTs:449349067891539971 MinTableBarrierTs:449349069202522117 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349067891539971, checkpointTs: 449349067891539971, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bf2f405-e714-4b5a-82b9-a0352c0528cd {"id":"3bf2f405-e714-4b5a-82b9-a0352c0528cd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d66c63 3bf2f405-e714-4b5a-82b9-a0352c0528cd /tidb/cdc/default/default/changefeed/info/d529f437-0ba1-4d6d-9053-868442981d74 {"upstream-id":7362135041331549037,"namespace":"default","changefeed-id":"d529f437-0ba1-4d6d-9053-868442981d74","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-30808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:17.186314291+08:00","start-ts":449349066961190916,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349066987143173} /tidb/cdc/default/default/changefeed/status/d529f437-0ba1-4d6d-9053-868442981d74 {"checkpoint-ts":449349067891539971,"min-table-barrier-ts":449349069202522117,"admin-job-type":0} /tidb/cdc/default/default/task/position/3bf2f405-e714-4b5a-82b9-a0352c0528cd/d529f437-0ba1-4d6d-9053-868442981d74 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning"+ grep -q 'etcd info' :null} /tidb/cdc/default/default/upstream/7362135041331549037 {"id":7362135041331549037,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Verifying downstream PD is started... "id": "3bf2f405-e714-4b5a-82b9-a0352c0528cd", "cluster-id": "default" run task successfully start tidb cluster in /tmp/tidb_cdc_test/kafka_compression Starting Upstream PD... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5650.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... The 1 times to try to start tidb cluster... table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... capture_id: 3bf2f405-e714-4b5a-82b9-a0352c0528cd check_processor_table_count http://127.0.0.1:2379 d529f437-0ba1-4d6d-9053-868442981d74 3bf2f405-e714-4b5a-82b9-a0352c0528cd 1 run task successfully check_processor_table_count http://127.0.0.1:2379 d529f437-0ba1-4d6d-9053-868442981d74 3bf2f405-e714-4b5a-82b9-a0352c0528cd 0 + set +x + tso='449349069953564673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349069953564673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:24:29 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.56915693.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 3-th check, retry later table count 1 does equal to expected count 0 run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check_processor_table_count http://127.0.0.1:2379 d529f437-0ba1-4d6d-9053-868442981d74 3bf2f405-e714-4b5a-82b9-a0352c0528cd 0 Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release run task successfully ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "operationcount"="0" "scanproportion"="0" "workload"="core" "mysql.port"="4000" "threadcount"="4" "dotransactions"="false" "recordcount"="50" "updateproportion"="0" "readproportion"="0" "readallfields"="true" "mysql.db"="changefeed_reconstruct" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 17.645218ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3463.8, Avg(us): 1376, Min(us): 982, Max(us): 3883, 95th(us): 4000, 99th(us): 4000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:33 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7511c72e-c930-4e6b-83b7-f71e1f90fa7a {"id":"7511c72e-c930-4e6b-83b7-f71e1f90fa7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130670} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26120dce 7511c72e-c930-4e6b-83b7-f71e1f90fa7a /tidb/cdc/default/default/upstream/7362135112507605440 {"id":7362135112507605440,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7511c72e-c930-4e6b-83b7-f71e1f90fa7a {"id":"7511c72e-c930-4e6b-83b7-f71e1f90fa7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130670} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26120dce 7511c72e-c930-4e6b-83b7-f71e1f90fa7a /tidb/cdc/default/default/upstream/7362135112507605440 {"id":7362135112507605440,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7511c72e-c930-4e6b-83b7-f71e1f90fa7a {"id":"7511c72e-c930-4e6b-83b7-f71e1f90fa7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130670} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26120dce 7511c72e-c930-4e6b-83b7-f71e1f90fa7a /tidb/cdc/default/default/upstream/7362135112507605440 {"id":7362135112507605440,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5750.out cli changefeed create --start-ts=449349069953564673 '--sink-uri=kafka://127.0.0.1:9092/ticdc-drop-tables-test-12167?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 01229911-e040-4693-b6d6-57a9fdffdc60 Info: {"upstream_id":7362135112507605440,"namespace":"default","id":"01229911-e040-4693-b6d6-57a9fdffdc60","sink_uri":"kafka://127.0.0.1:9092/ticdc-drop-tables-test-12167?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:33.484109301+08:00","start_ts":449349069953564673,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349069953564673,"checkpoint_ts":449349069953564673,"checkpoint_time":"2024-04-26 19:24:28.463"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:24:34 CST 2024] <<<<<< START kafka consumer in drop_many_tables case >>>>>> table changefeed_reconstruct.usertable exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:24:35 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 6-th check, retry later table drop_tables.c not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899ab6c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:1357, start at 2024-04-26 19:24:36.483488213 +0800 CST m=+5.355624930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:36.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:36.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:36.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 2-th check, retry later wait process 5603 exit for 1-th time... wait process 5603 exit for 2-th time... wait process 5603 exit for 3-th time... wait process 5603 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5603) - No such process wait process 5603 exit for 5-th time... process 5603 already exit [Fri Apr 26 19:24:37 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.57885790.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899b6180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:1356, start at 2024-04-26 19:24:37.157667858 +0800 CST m=+5.090579568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:37.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:37.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:37.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899b6180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:1356, start at 2024-04-26 19:24:37.157667858 +0800 CST m=+5.090579568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:37.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:37.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:37.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899b9440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:1443, start at 2024-04-26 19:24:37.351549453 +0800 CST m=+5.226145138 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:37.357 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:37.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:37.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899ab6c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:1357, start at 2024-04-26 19:24:36.483488213 +0800 CST m=+5.355624930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:36.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:36.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:36.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899ace80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:1447, start at 2024-04-26 19:24:36.545642183 +0800 CST m=+5.362942155 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:36.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:36.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:36.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899cd20000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:5798, start at 2024-04-26 19:24:38.611344342 +0800 CST m=+5.316591698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:38.618 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:38.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:38.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully table drop_tables.c not exists for 3-th check, retry later [Fri Apr 26 19:24:40 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.28382840.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899ddf40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:1402, start at 2024-04-26 19:24:39.707295482 +0800 CST m=+5.231565822 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:39.713 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:39.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:39.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899cd20000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:5798, start at 2024-04-26 19:24:38.611344342 +0800 CST m=+5.316591698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:38.618 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:38.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:38.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899cd600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:5887, start at 2024-04-26 19:24:38.62009728 +0800 CST m=+5.268799554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:38.628 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:38.616 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:38.616 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 26 19:24:41 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.28282830.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362135042907990264 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:18.732740605 +0800 CST StartTs:449349067371708419 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349067398184963} {CheckpointTs:449349068053282842 MinTableBarrierTs:449349068053282842 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a7414b5-715e-45fc-8576-1527fa20afd3 {"id":"1a7414b5-715e-45fc-8576-1527fa20afd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7abb7 1a7414b5-715e-45fc-8576-1527fa20afd3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362135042907990264,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:18.732740605+08:00","start-ts":449349067371708419,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349067398184963} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449349068053282842,"min-table-barrier-ts":449349068053282842,"admin-job-type":0} /tidb/cdc/default/default/task/position/1a7414b5-715e-45fc-8576-1527fa20afd3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362135042907990264 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:18.732740605 +0800 CST StartTs:449349067371708419 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349067398184963} {CheckpointTs:449349068053282842 MinTableBarrierTs:449349068053282842 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a7414b5-715e-45fc-8576-1527fa20afd3 {"id":"1a7414b5-715e-45fc-8576-1527fa20afd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7abb7 1a7414b5-715e-45fc-8576-1527fa20afd3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362135042907990264,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:18.732740605+08:00","start-ts":449349067371708419,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349067398184963} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449349068053282842,"min-table-barrier-ts":449349068053282842,"admin-job-type":0} /tidb/cdc/default/default/task/position/1a7414b5-715e-45fc-8576-1527fa20afd3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362135042907990264 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:18.732740605 +0800 CST StartTs:449349067371708419 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349067398184963} {CheckpointTs:449349068053282842 MinTableBarrierTs:449349068053282842 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a7414b5-715e-45fc-8576-1527fa20afd3 {"id":"1a7414b5-715e-45fc-8576-1527fa20afd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7abb7 1a7414b5-715e-45fc-8576-1527fa20afd3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362135042907990264,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:18.732740605+08:00","start-ts":449349067371708419,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349067398184963} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449349068053282842,"min-table-barrier-ts":449349068053282842,"admin-job-type":0} /tidb/cdc/default/default/task/position/1a7414b5-715e-45fc-8576-1527fa20afd3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135042907990264 + grep -q 'etcd info' {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 26 19:24:42 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.58465848.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table drop_tables.c not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a02040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:1361, start at 2024-04-26 19:24:42.027176124 +0800 CST m=+5.503361761 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:42.036 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:42.035 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:42.035 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899ddf40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:1402, start at 2024-04-26 19:24:39.707295482 +0800 CST m=+5.231565822 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:39.713 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:39.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:39.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a04240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:1482, start at 2024-04-26 19:24:42.141312565 +0800 CST m=+7.599357770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:42.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:42.121 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:42.121 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:43 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c647a654-1f47-40f8-8476-526bc6fee186 {"id":"c647a654-1f47-40f8-8476-526bc6fee186","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2641ebc4 c647a654-1f47-40f8-8476-526bc6fee186 /tidb/cdc/default/default/upstream/7362135161979231558 {"id":7362135161979231558,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c647a654-1f47-40f8-8476-526bc6fee186 {"id":"c647a654-1f47-40f8-8476-526bc6fee186","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2641ebc4 c647a654-1f47-40f8-8476-526bc6fee186 /tidb/cdc/default/default/upstream/7362135161979231558 {"id":7362135161979231558,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c647a654-1f47-40f8-8476-526bc6fee186 {"id":"c647a654-1f47-40f8-8476-526bc6fee186","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2641ebc4 c647a654-1f47-40f8-8476-526bc6fee186 /tidb/cdc/default/default/upstream/7362135161979231558 {"id":7362135161979231558,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:24:43 CST 2024] <<<<<< START kafka consumer in kv_client_stream_reconnect case >>>>>> [Fri Apr 26 19:24:43 CST 2024] <<<<<< START cdc server in kafka_compression case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.72327234.out server --log-file /tmp/tidb_cdc_test/kafka_compression/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_compression/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table drop_tables.c not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:44 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6 {"id":"548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130681} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a263e1ad0 548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6 /tidb/cdc/default/default/upstream/7362135147680999351 {"id":7362135147680999351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6 {"id":"548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130681} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a263e1ad0 548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6 /tidb/cdc/default/default/upstream/7362135147680999351 {"id":7362135147680999351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6 {"id":"548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130681} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a263e1ad0 548f7ed7-c7c1-4266-9c7a-e7fdd415b7b6 /tidb/cdc/default/default/upstream/7362135147680999351 {"id":7362135147680999351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a02040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:1361, start at 2024-04-26 19:24:42.027176124 +0800 CST m=+5.503361761 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:42.036 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:42.035 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:42.035 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a01200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:1442, start at 2024-04-26 19:24:41.952196145 +0800 CST m=+5.374109541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:41.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:41.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:41.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2822.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 26 19:24:44 CST 2024] <<<<<< START kafka consumer in gc_safepoint case >>>>>> 0 check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a20000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:6020, start at 2024-04-26 19:24:43.906225879 +0800 CST m=+5.128347572 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:43.912 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:43.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:43.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a20000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:6020, start at 2024-04-26 19:24:43.906225879 +0800 CST m=+5.128347572 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:43.912 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:43.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:43.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689a20b00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:6094, start at 2024-04-26 19:24:43.982681436 +0800 CST m=+5.152674567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:43.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:43.948 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:43.948 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table drop_tables.c exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:24:46 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2888.out cli tso query --pd=http://127.0.0.1:2379 < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:46 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61c0ffdc-ff66-436d-94ed-143583412b16 {"id":"61c0ffdc-ff66-436d-94ed-143583412b16","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130683} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a264aa3cd 61c0ffdc-ff66-436d-94ed-143583412b16 /tidb/cdc/default/default/upstream/7362135171741105675 {"id":7362135171741105675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61c0ffdc-ff66-436d-94ed-143583412b16 {"id":"61c0ffdc-ff66-436d-94ed-143583412b16","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130683} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a264aa3cd 61c0ffdc-ff66-436d-94ed-143583412b16 /tidb/cdc/default/default/upstream/7362135171741105675 {"id":7362135171741105675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61c0ffdc-ff66-436d-94ed-143583412b16 {"id":"61c0ffdc-ff66-436d-94ed-143583412b16","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130683} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a264aa3cd 61c0ffdc-ff66-436d-94ed-143583412b16 /tidb/cdc/default/default/upstream/7362135171741105675 {"id":7362135171741105675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7293.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349074335039489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349074335039489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "insertproportion"="0" "updateproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "readproportion"="0" "mysql.port"="4000" "readallfields"="true" "mysql.host"="127.0.0.1" "recordcount"="10" "workload"="core" "mysql.user"="root" "mysql.db"="sink_retry" "scanproportion"="0" "operationcount"="0" "threadcount"="2" ********************************************** Run finished, takes 10.711601ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1742.7, Avg(us): 2051, Min(us): 1234, Max(us): 4871, 95th(us): 5000, 99th(us): 5000 [Fri Apr 26 19:24:46 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.28812883.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later [Fri Apr 26 19:24:47 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.75137515.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362135042907990264 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:18.732740605 +0800 CST StartTs:449349067371708419 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349067398184963} {CheckpointTs:449349068053282842 MinTableBarrierTs:449349068053282842 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a7414b5-715e-45fc-8576-1527fa20afd3 {"id":"1a7414b5-715e-45fc-8576-1527fa20afd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7abb7 1a7414b5-715e-45fc-8576-1527fa20afd3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362135042907990264,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:18.732740605+08:00","start-ts":449349067371708419,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349067398184963} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449349068184616986,"min-table-barrier-ts":449349068184616986,"admin-job-type":0} /tidb/cdc/default/default/task/position/1a7414b5-715e-45fc-8576-1527fa20afd3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362135042907990264 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:18.732740605 +0800 CST StartTs:449349067371708419 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349067398184963} {CheckpointTs:449349068053282842 MinTableBarrierTs:449349068053282842 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a7414b5-715e-45fc-8576-1527fa20afd3 {"id":"1a7414b5-715e-45fc-8576-1527fa20afd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7abb7 1a7414b5-715e-45fc-8576-1527fa20afd3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362135042907990264,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:18.732740605+08:00","start-ts":449349067371708419,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349067398184963} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449349068184616986,"min-table-barrier-ts":449349068184616986,"admin-job-type":0} /tidb/cdc/default/default/task/position/1a7414b5-715e-45fc-8576-1527fa20afd3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362135042907990264 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:24:18.732740605 +0800 CST StartTs:449349067371708419 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d7290 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349067398184963} {CheckpointTs:449349068053282842 MinTableBarrierTs:449349068053282842 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a7414b5-715e-45fc-8576-1527fa20afd3 {"id":"1a7414b5-715e-45fc-8576-1527fa20afd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25d7abb7 1a7414b5-715e-45fc-8576-1527fa20afd3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362135042907990264,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-24552?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:24:18.732740605+08:00","start-ts":449349067371708419,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349067398184963} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449349068184616986,"min-table-barrier-ts":449349068184616986,"admin-job-type":0} /tidb/cdc/default/default/task/position/1a7414b5-715e-45fc-8576-1527fa20afd3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135042907990264 {"id":7362135042907990264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later + set +x + tso='449349074861424641 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349074861424641 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:24:48 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.29292931.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 2-th check, retry later + set +x + tso='449349074826035204 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349074826035204 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7329.out cli changefeed create --start-ts=449349074826035204 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip' -c gzip Create changefeed successfully! ID: gzip Info: {"upstream_id":7362135171741105675,"namespace":"default","id":"gzip","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=gzip","create_time":"2024-04-26T19:24:49.48384901+08:00","start_ts":449349074826035204,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349074826035204,"checkpoint_ts":449349074826035204,"checkpoint_time":"2024-04-26 19:24:47.050"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully check_safepoint_forward http://127.0.0.1:2379 7362135147680999351 449349075112820742 449349074195054595 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c7f98c5-d321-45f7-ba04-4d00e5caad52 {"id":"9c7f98c5-d321-45f7-ba04-4d00e5caad52","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130687} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a264d47e9 9c7f98c5-d321-45f7-ba04-4d00e5caad52 /tidb/cdc/default/default/upstream/7362135175336359218 {"id":7362135175336359218,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c7f98c5-d321-45f7-ba04-4d00e5caad52 {"id":"9c7f98c5-d321-45f7-ba04-4d00e5caad52","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130687} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a264d47e9 9c7f98c5-d321-45f7-ba04-4d00e5caad52 /tidb/cdc/default/default/upstream/7362135175336359218 {"id":7362135175336359218,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c7f98c5-d321-45f7-ba04-4d00e5caad52 {"id":"9c7f98c5-d321-45f7-ba04-4d00e5caad52","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130687} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a264d47e9 9c7f98c5-d321-45f7-ba04-4d00e5caad52 /tidb/cdc/default/default/upstream/7362135175336359218 {"id":7362135175336359218,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2935.out cli changefeed create --start-ts=449349074335039489 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-22937?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' run task successfully Create changefeed successfully! ID: df630853-487f-4c0e-95fc-dfd9cf1f003f Info: {"upstream_id":7362135175336359218,"namespace":"default","id":"df630853-487f-4c0e-95fc-dfd9cf1f003f","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-22937?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:50.399750173+08:00","start_ts":449349074335039489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349074335039489,"checkpoint_ts":449349074335039489,"checkpoint_time":"2024-04-26 19:24:45.177"} PASS check_changefeed_state http://127.0.0.1:2379 e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 -s coverage: 2.4% of statements in github.com/pingcap/tiflow/... + info='{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "stopped", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null }' + echo '{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "stopped", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null }' { "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "stopped", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null } ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"e7aa68a9-bdf6-4eed-aa2b-2b95772c9509",' '"state":' '"stopped",' '"checkpoint_tso":' 449349075649953793, '"checkpoint_time":' '"2024-04-26' '19:24:50.193",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"e7aa68a9-bdf6-4eed-aa2b-2b95772c9509",' '"state":' '"stopped",' '"checkpoint_tso":' 449349075649953793, '"checkpoint_time":' '"2024-04-26' '19:24:50.193",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7362135147680999351 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:50 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5dfb835-1100-424a-b8d5-1fd191f71623 {"id":"f5dfb835-1100-424a-b8d5-1fd191f71623","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130687} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a265c0acc f5dfb835-1100-424a-b8d5-1fd191f71623 /tidb/cdc/default/default/upstream/7362135184060707088 {"id":7362135184060707088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5dfb835-1100-424a-b8d5-1fd191f71623 {"id":"f5dfb835-1100-424a-b8d5-1fd191f71623","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130687} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a265c0acc f5dfb835-1100-424a-b8d5-1fd191f71623 /tidb/cdc/default/default/upstream/7362135184060707088 {"id":7362135184060707088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5dfb835-1100-424a-b8d5-1fd191f71623 {"id":"f5dfb835-1100-424a-b8d5-1fd191f71623","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130687} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a265c0acc f5dfb835-1100-424a-b8d5-1fd191f71623 /tidb/cdc/default/default/upstream/7362135184060707088 {"id":7362135184060707088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7362135184060707088,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-3984?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:50.497569769+08:00","start_ts":449349074853298177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349074853298177,"checkpoint_ts":449349074853298177,"checkpoint_time":"2024-04-26 19:24:47.154"} + set +x [Fri Apr 26 19:24:50 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 25.65 secs (145014323 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 3-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:24:49.442 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:49.478 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:49.603 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:49.612 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:50.577 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:50.587 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:24:49.442 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:49.478 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:49.603 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:49.612 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:50.577 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:24:50.587 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]") table test.gzip_finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7362135184060707088,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-26454?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:51.820569397+08:00","start_ts":449349074853298177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349074853298177,"checkpoint_ts":449349074853298177,"checkpoint_time":"2024-04-26 19:24:47.154"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/629bfccf-7f64-4a4c-806e-adb1eadc2560 {"id":"629bfccf-7f64-4a4c-806e-adb1eadc2560","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130689} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a265745d4 629bfccf-7f64-4a4c-806e-adb1eadc2560 /tidb/cdc/default/default/upstream/7362135185365210057 {"id":7362135185365210057,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/629bfccf-7f64-4a4c-806e-adb1eadc2560 {"id":"629bfccf-7f64-4a4c-806e-adb1eadc2560","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130689} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a265745d4 629bfccf-7f64-4a4c-806e-adb1eadc2560 /tidb/cdc/default/default/upstream/7362135185365210057 {"id":7362135185365210057,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/629bfccf-7f64-4a4c-806e-adb1eadc2560 {"id":"629bfccf-7f64-4a4c-806e-adb1eadc2560","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130689} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a265745d4 629bfccf-7f64-4a4c-806e-adb1eadc2560 /tidb/cdc/default/default/upstream/7362135185365210057 {"id":7362135185365210057,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2986.out cli changefeed create --start-ts=449349074861424641 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resolve-lock-test-27216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x [Fri Apr 26 19:24:51 CST 2024] <<<<<< START kafka consumer in sink_retry case >>>>>> Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7362135184060707088,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-31032?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:52.038135493+08:00","start_ts":449349074853298177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349074853298177,"checkpoint_ts":449349074853298177,"checkpoint_time":"2024-04-26 19:24:47.154"} [Fri Apr 26 19:24:52 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Fri Apr 26 19:24:52 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Fri Apr 26 19:24:52 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> Create changefeed successfully! ID: 61582465-0b21-4873-936a-086562d99e49 Info: {"upstream_id":7362135185365210057,"namespace":"default","id":"61582465-0b21-4873-936a-086562d99e49","sink_uri":"kafka://127.0.0.1:9092/ticdc-resolve-lock-test-27216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:52.244007298+08:00","start_ts":449349074861424641,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349074861424641,"checkpoint_ts":449349074861424641,"checkpoint_time":"2024-04-26 19:24:47.185"} PASS table test.finish_mark not exists for 1-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 2-th check, retry later table test.gzip_finish_mark not exists for 2-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 -s start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + info='{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "normal", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null }' + echo '{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "normal", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null }' { "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "normal", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null } ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"e7aa68a9-bdf6-4eed-aa2b-2b95772c9509",' '"state":' '"normal",' '"checkpoint_tso":' 449349075649953793, '"checkpoint_time":' '"2024-04-26' '19:24:50.193",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"e7aa68a9-bdf6-4eed-aa2b-2b95772c9509",' '"state":' '"normal",' '"checkpoint_tso":' 449349075649953793, '"checkpoint_time":' '"2024-04-26' '19:24:50.193",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7362135147680999351 449349075649953792 449349075649953793 table ddl_manager.finish_mark not exists for 5-th check, retry later table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table test.finish_mark not exists for 3-th check, retry later run task successfully table test.gzip_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7464.out cli changefeed pause -c gzip + set +x [Fri Apr 26 19:24:55 CST 2024] <<<<<< START kafka consumer in resolve_lock case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading golang.org/x/sys v0.19.0 check_changefeed_state http://127.0.0.1:2379 e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 -s PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda + info='{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "stopped", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null }' + echo '{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "stopped", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null }' { "upstream_id": 7362135147680999351, "namespace": "default", "id": "e7aa68a9-bdf6-4eed-aa2b-2b95772c9509", "state": "stopped", "checkpoint_tso": 449349075649953793, "checkpoint_time": "2024-04-26 19:24:50.193", "error": null } ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"e7aa68a9-bdf6-4eed-aa2b-2b95772c9509",' '"state":' '"stopped",' '"checkpoint_tso":' 449349075649953793, '"checkpoint_time":' '"2024-04-26' '19:24:50.193",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"e7aa68a9-bdf6-4eed-aa2b-2b95772c9509",' '"state":' '"stopped",' '"checkpoint_tso":' 449349075649953793, '"checkpoint_time":' '"2024-04-26' '19:24:50.193",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully go: downloading golang.org/x/text v0.14.0 check_changefeed_state http://127.0.0.1:2379 ba277ab9-3b1e-47f5-a17b-5c8c9150751a normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=ba277ab9-3b1e-47f5-a17b-5c8c9150751a + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ba277ab9-3b1e-47f5-a17b-5c8c9150751a -s + info='{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "ba277ab9-3b1e-47f5-a17b-5c8c9150751a", "state": "normal", "checkpoint_tso": 449349077183496198, "checkpoint_time": "2024-04-26 19:24:56.043", "error": null }' + echo '{ "upstream_id": 7362135147680999351, "namespace": "default", "id": "ba277ab9-3b1e-47f5-a17b-5c8c9150751a", "state": "normal", "checkpoint_tso": 449349077183496198, "checkpoint_time": "2024-04-26 19:24:56.043", "error": null }' { "upstream_id": 7362135147680999351, "namespace": "default", "id": "ba277ab9-3b1e-47f5-a17b-5c8c9150751a", "state": "normal", "checkpoint_tso": 449349077183496198, "checkpoint_time": "2024-04-26 19:24:56.043", "error": null } ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"ba277ab9-3b1e-47f5-a17b-5c8c9150751a",' '"state":' '"normal",' '"checkpoint_tso":' 449349077183496198, '"checkpoint_time":' '"2024-04-26' '19:24:56.043",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135147680999351, '"namespace":' '"default",' '"id":' '"ba277ab9-3b1e-47f5-a17b-5c8c9150751a",' '"state":' '"normal",' '"checkpoint_tso":' 449349077183496198, '"checkpoint_time":' '"2024-04-26' '19:24:56.043",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7362135147680999351 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table multi_tables_ddl_test.t55 not exists for 2-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7502.out cli changefeed remove -c gzip table ddl_manager.finish_mark not exists for 6-th check, retry later go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 Changefeed remove successfully. ID: gzip CheckpointTs: 449349076739948548 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip PASS go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c table test.finish_mark not exists for 4-th check, retry later coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/apimachinery v0.28.6 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 3-th check, retry later + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7539.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 7-th check, retry later run task successfully Changefeed remove successfully. ID: e7aa68a9-bdf6-4eed-aa2b-2b95772c9509 CheckpointTs: 449349075649953793 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-24307?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 table test.finish_mark not exists for 5-th check, retry later check_safepoint_forward http://127.0.0.1:2379 7362135147680999351 449349078009511938 449349077183496198 449349075649953793 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 7.16 secs (519326282 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully + set +x + tso='449349078115942407 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349078115942407 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7572.out cli changefeed create --start-ts=449349078115942407 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy' -c snappy start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 8-th check, retry later Create changefeed successfully! ID: snappy Info: {"upstream_id":7362135171741105675,"namespace":"default","id":"snappy","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=snappy","create_time":"2024-04-26T19:25:01.539061645+08:00","start_ts":449349078115942407,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349078115942407,"checkpoint_ts":449349078115942407,"checkpoint_time":"2024-04-26 19:24:59.600"} PASS table multi_tables_ddl_test.t55 not exists for 4-th check, retry later Changefeed remove successfully. ID: ba277ab9-3b1e-47f5-a17b-5c8c9150751a CheckpointTs: 449349078271393795 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-24307?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_cleared http://127.0.0.1:2379 7362135147680999351 run task successfully coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark exists check diff successfully go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:25:02 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x [Fri Apr 26 19:25:03 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:25:01.493 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:01.533 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:01.631 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:01.641 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:02.628 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:02.637 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:25:01.493 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:01.533 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:01.631 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:01.641 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:02.628 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:25:02.637 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]") table test.snappy_finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 9-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:25:03 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> table ddl_manager.finish_mark not exists for 10-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689b5bf40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:7667, start at 2024-04-26 19:25:04.17401103 +0800 CST m=+5.301666949 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:04.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:04.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:04.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.snappy_finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 11-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689b5bf40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:7667, start at 2024-04-26 19:25:04.17401103 +0800 CST m=+5.301666949 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:04.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:04.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:04.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689b5d6c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:7758, start at 2024-04-26 19:25:04.244341378 +0800 CST m=+5.317812880 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:04.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:04.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:04.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s table test.snappy_finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449349076911128597, "checkpoint_time": "2024-04-26 19:24:55.004", "error": null }' + echo '{ "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449349076911128597, "checkpoint_time": "2024-04-26 19:24:55.004", "error": null }' { "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449349076911128597, "checkpoint_time": "2024-04-26 19:24:55.004", "error": null } ++ echo '{' '"upstream_id":' 7362135184060707088, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449349076911128597, '"checkpoint_time":' '"2024-04-26' '19:24:55.004",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135184060707088, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449349076911128597, '"checkpoint_time":' '"2024-04-26' '19:24:55.004",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449349080017534979, "checkpoint_time": "2024-04-26 19:25:06.854", "error": null }' + echo '{ "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449349080017534979, "checkpoint_time": "2024-04-26 19:25:06.854", "error": null }' { "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449349080017534979, "checkpoint_time": "2024-04-26 19:25:06.854", "error": null } ++ echo '{' '"upstream_id":' 7362135184060707088, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449349080017534979, '"checkpoint_time":' '"2024-04-26' '19:25:06.854",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135184060707088, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449349080017534979, '"checkpoint_time":' '"2024-04-26' '19:25:06.854",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-rp194-36l8q --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "be32c19e17fd04d4dea29b71d787d3f9fcbabf18" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-rp194" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test + info='{ "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449349076439269410, "checkpoint_time": "2024-04-26 19:24:53.204", "error": { "time": "2024-04-26T19:24:55.881754571+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449349076439269410, "checkpoint_time": "2024-04-26 19:24:53.204", "error": { "time": "2024-04-26T19:24:55.881754571+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7362135184060707088, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449349076439269410, "checkpoint_time": "2024-04-26 19:24:53.204", "error": { "time": "2024-04-26T19:24:55.881754571+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362135184060707088, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449349076439269410, '"checkpoint_time":' '"2024-04-26' '19:24:53.204",' '"error":' '{' '"time":' '"2024-04-26T19:24:55.881754571+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362135184060707088, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449349076439269410, '"checkpoint_time":' '"2024-04-26' '19:24:53.204",' '"error":' '{' '"time":' '"2024-04-26T19:24:55.881754571+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.snappy_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7703.out cli changefeed pause -c snappy ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 12-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9091.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 3-th time... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:25:10 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> + set +x + tso='449349080717983745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349080717983745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7737.out cli changefeed remove -c snappy ***************** properties ***************** "dotransactions"="false" "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.user"="root" "workload"="core" "mysql.db"="multi_capture_1" "insertproportion"="0" "operationcount"="0" "scanproportion"="0" "requestdistribution"="uniform" "recordcount"="10" "threadcount"="2" "updateproportion"="0" "readallfields"="true" "mysql.port"="4000" ********************************************** Run finished, takes 8.818231ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2039.5, Avg(us): 1694, Min(us): 1042, Max(us): 3917, 95th(us): 4000, 99th(us): 4000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689bbdd0001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:7027, start at 2024-04-26 19:25:10.415179608 +0800 CST m=+5.133145497 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:10.422 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:10.388 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:10.388 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689bbdd0001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:7027, start at 2024-04-26 19:25:10.415179608 +0800 CST m=+5.133145497 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:10.422 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:10.388 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:10.388 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689bbe74000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:7111, start at 2024-04-26 19:25:10.439584144 +0800 CST m=+5.098456552 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:10.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:10.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:10.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 13-th check, retry later ***************** properties ***************** "readproportion"="0" "recordcount"="10" "updateproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "operationcount"="0" "readallfields"="true" "mysql.port"="4000" "threadcount"="2" "insertproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "workload"="core" "mysql.user"="root" "mysql.db"="multi_capture_2" ********************************************** Run finished, takes 9.003855ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2137.1, Avg(us): 1729, Min(us): 960, Max(us): 4254, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "insertproportion"="0" "operationcount"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "threadcount"="2" "readproportion"="0" "mysql.port"="4000" "workload"="core" "readallfields"="true" "recordcount"="10" "scanproportion"="0" "mysql.user"="root" "mysql.db"="multi_capture_3" "requestdistribution"="uniform" "updateproportion"="0" ********************************************** Run finished, takes 10.329727ms INSERT - Takes(s): 0.0, Count: 9, OPS: 1576.2, Avg(us): 1691, Min(us): 1105, Max(us): 4484, 95th(us): 5000, 99th(us): 5000 Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ***************** properties ***************** "mysql.host"="127.0.0.1" "operationcount"="0" "readproportion"="0" "mysql.user"="root" "readallfields"="true" "mysql.db"="multi_capture_4" "mysql.port"="4000" "requestdistribution"="uniform" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" "recordcount"="10" "scanproportion"="0" "threadcount"="2" "workload"="core" ********************************************** Run finished, takes 9.76348ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1832.9, Avg(us): 1859, Min(us): 1180, Max(us): 4147, 95th(us): 5000, 99th(us): 5000 [Fri Apr 26 19:25:11 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.92159217.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: snappy CheckpointTs: 449349080671846404 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 14-th check, retry later + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7770.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 26 19:25:13 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.84198421.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:14 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:25:14 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.92759277.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 15-th check, retry later + set +x + tso='449349081825280002 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349081825280002 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7807.out cli changefeed create --start-ts=449349081825280002 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4' -c lz4 Create changefeed successfully! ID: lz4 Info: {"upstream_id":7362135171741105675,"namespace":"default","id":"lz4","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=lz4","create_time":"2024-04-26T19:25:15.680912406+08:00","start_ts":449349081825280002,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349081825280002,"checkpoint_ts":449349081825280002,"checkpoint_time":"2024-04-26 19:25:13.750"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:16 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/419fb096-df1e-4a7d-a44f-ba0c1f49cccd {"id":"419fb096-df1e-4a7d-a44f-ba0c1f49cccd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26cb7fc4 419fb096-df1e-4a7d-a44f-ba0c1f49cccd /tidb/cdc/default/default/upstream/7362135308669154987 {"id":7362135308669154987,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/419fb096-df1e-4a7d-a44f-ba0c1f49cccd {"id":"419fb096-df1e-4a7d-a44f-ba0c1f49cccd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26cb7fc4 419fb096-df1e-4a7d-a44f-ba0c1f49cccd /tidb/cdc/default/default/upstream/7362135308669154987 {"id":7362135308669154987,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/419fb096-df1e-4a7d-a44f-ba0c1f49cccd {"id":"419fb096-df1e-4a7d-a44f-ba0c1f49cccd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26cb7fc4 419fb096-df1e-4a7d-a44f-ba0c1f49cccd /tidb/cdc/default/default/upstream/7362135308669154987 {"id":7362135308669154987,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:25:16 CST 2024] <<<<<< START kafka consumer in processor_stop_delay case >>>>>> table ddl_manager.finish_mark not exists for 16-th check, retry later table processor_stop_delay.t not exists for 1-th check, retry later + set +x [Fri Apr 26 19:25:17 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:25:15.645 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:15.677 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:15.780 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:15.789 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:16.780 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:16.788 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:25:15.645 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:15.677 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:15.780 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:15.789 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:16.780 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:25:16.788 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]") table test.lz4_finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:18 GMT < Content-Length: 1271 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e57c377e-9982-440f-8653-3129718e81b4 {"id":"e57c377e-9982-440f-8653-3129718e81b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130715} /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f6d e57c377e-9982-440f-8653-3129718e81b4 /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e57c377e-9982-440f-8653-3129718e81b4 {"id":"e57c377e-9982-440f-8653-3129718e81b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130715} /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f6d e57c377e-9982-440f-8653-3129718e81b4 /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e57c377e-9982-440f-8653-3129718e81b4 {"id":"e57c377e-9982-440f-8653-3129718e81b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130715} /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f6d e57c377e-9982-440f-8653-3129718e81b4 /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:25:18 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.93339335.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 15.38 secs (241878818 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 17-th check, retry later table processor_stop_delay.t not exists for 2-th check, retry later table test.lz4_finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later table processor_stop_delay.t exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:21 GMT < Content-Length: 1750 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9 {"id":"6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9","address":"127.0.0.1:8303","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130718} /tidb/cdc/default/__cdc_meta__/capture/e57c377e-9982-440f-8653-3129718e81b4 {"id":"e57c377e-9982-440f-8653-3129718e81b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130715} /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f6d e57c377e-9982-440f-8653-3129718e81b4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f90 6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9 /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9 {"id":"6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9","address":"127.0.0.1:8303","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130718} /tidb/cdc/default/__cdc_meta__/capture/e57c377e-9982-440f-8653-3129718e81b4 {"id":"e57c377e-9982-440f-8653-3129718e81b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130715} /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f6d e57c377e-9982-440f-8653-3129718e81b4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f90 6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9 /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9 {"id":"6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9","address":"127.0.0.1:8303","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130718} /tidb/cdc/default/__cdc_meta__/capture/e57c377e-9982-440f-8653-3129718e81b4 {"id":"e57c377e-9982-440f-8653-3129718e81b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130715} /tidb/cdc/default/__cdc_meta__/capture/f6d91988-3a20-4934-ad02-9bea2ea11ece {"id":"f6d91988-3a20-4934-ad02-9bea2ea11ece","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130712} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f3b f6d91988-3a20-4934-ad02-9bea2ea11ece /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f6d e57c377e-9982-440f-8653-3129718e81b4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26b07f90 6a8e2e5a-1aa5-43e2-b2f4-5ab547d607d9 /tidb/cdc/default/default/upstream/7362135278926966898 {"id":7362135278926966898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9381.out cli changefeed create --start-ts=449349080717983745 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-capture-test-31649?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8301 Create changefeed successfully! ID: 26b0816b-6c41-400d-9c95-28eb9dc62e90 Info: {"upstream_id":7362135278926966898,"namespace":"default","id":"26b0816b-6c41-400d-9c95-28eb9dc62e90","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-capture-test-31649?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:25:21.609089597+08:00","start_ts":449349080717983745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349080717983745,"checkpoint_ts":449349080717983745,"checkpoint_time":"2024-04-26 19:25:09.526"} PASS table test.lz4_finish_mark not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic_avro Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x [Fri Apr 26 19:25:23 CST 2024] <<<<<< START kafka consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later check diff failed 1-th time, retry later table test.lz4_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7956.out cli changefeed pause -c lz4 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "readproportion"="0" "recordcount"="10" "insertproportion"="0" "workload"="core" "updateproportion"="0" "readallfields"="true" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "threadcount"="2" "requestdistribution"="uniform" "scanproportion"="0" "mysql.user"="root" "mysql.db"="sink_retry" ********************************************** Run finished, takes 4.163815ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3244.2, Avg(us): 715, Min(us): 476, Max(us): 1769, 95th(us): 2000, 99th(us): 2000 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table multi_capture_1.usertable not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later check diff failed 2-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7989.out cli changefeed remove -c lz4 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:25:25 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: lz4 CheckpointTs: 449349084342124551 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 21-th check, retry later check diff failed 3-th time, retry later table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8029.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 22-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 7.61 secs (488733275 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh check diff failed 4-th time, retry later [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689cdf500012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:4895, start at 2024-04-26 19:25:28.93155284 +0800 CST m=+5.153345635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:28.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:28.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:28.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689cdf500012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:4895, start at 2024-04-26 19:25:28.93155284 +0800 CST m=+5.153345635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:28.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:28.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:28.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689cdf380006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:4978, start at 2024-04-26 19:25:28.915820243 +0800 CST m=+5.088092849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:28.922 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:28.910 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:28.910 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + set +x + tso='449349085521510406 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349085521510406 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8068.out cli changefeed create --start-ts=449349085521510406 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd' -c zstd Create changefeed successfully! ID: zstd Info: {"upstream_id":7362135171741105675,"namespace":"default","id":"zstd","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=zstd","create_time":"2024-04-26T19:25:30.883184346+08:00","start_ts":449349085521510406,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349085521510406,"checkpoint_ts":449349085521510406,"checkpoint_time":"2024-04-26 19:25:27.850"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later + set +x [Fri Apr 26 19:25:32 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:25:32 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.64186420.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:25:30.845 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:30.878 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:30.981 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:30.989 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:31.980 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:31.988 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:25:30.845 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:30.878 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:30.981 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:30.989 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:31.980 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:25:31.988 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]") table test.zstd_finish_mark not exists for 1-th check, retry later table multi_capture_4.usertable exists check diff successfully ***************** properties ***************** "mysql.port"="4000" "threadcount"="2" "requestdistribution"="uniform" "insertproportion"="0" "operationcount"="0" "workload"="core" "mysql.db"="multi_capture_1" "recordcount"="20" "dotransactions"="false" "scanproportion"="0" "readproportion"="0" "readallfields"="true" "mysql.user"="root" "updateproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 14.627311ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1512.3, Avg(us): 1396, Min(us): 531, Max(us): 4941, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "threadcount"="2" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_2" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "readproportion"="0" "insertproportion"="0" "recordcount"="20" "operationcount"="0" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** Run finished, takes 11.472142ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1949.3, Avg(us): 974, Min(us): 524, Max(us): 1577, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.port"="4000" "recordcount"="20" "updateproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "threadcount"="2" "insertproportion"="0" "dotransactions"="false" "scanproportion"="0" "operationcount"="0" "mysql.db"="multi_capture_3" "workload"="core" "readallfields"="true" "mysql.host"="127.0.0.1" "readproportion"="0" ********************************************** Run finished, takes 11.048064ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2043.1, Avg(us): 1024, Min(us): 523, Max(us): 1622, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.db"="multi_capture_4" "dotransactions"="false" "mysql.port"="4000" "mysql.user"="root" "updateproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "readallfields"="true" "scanproportion"="0" "operationcount"="0" "workload"="core" "insertproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "recordcount"="20" ********************************************** Run finished, takes 10.557231ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2165.1, Avg(us): 1002, Min(us): 475, Max(us): 1710, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689d079c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:4808, start at 2024-04-26 19:25:31.501994148 +0800 CST m=+5.510444605 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:31.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:31.495 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:31.495 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689d079c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:4808, start at 2024-04-26 19:25:31.501994148 +0800 CST m=+5.510444605 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:31.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:31.495 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:31.495 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689d092c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:4877, start at 2024-04-26 19:25:31.621517438 +0800 CST m=+5.572587217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:31.628 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:31.595 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:31.595 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.zstd_finish_mark not exists for 2-th check, retry later [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdba0cde-19ea-4401-b7ab-376069bbbcd3 {"id":"cdba0cde-19ea-4401-b7ab-376069bbbcd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130732} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a270bf6c2 cdba0cde-19ea-4401-b7ab-376069bbbcd3 /tidb/cdc/default/default/upstream/7362135384782445730 {"id":7362135384782445730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdba0cde-19ea-4401-b7ab-376069bbbcd3 {"id":"cdba0cde-19ea-4401-b7ab-376069bbbcd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130732} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a270bf6c2 cdba0cde-19ea-4401-b7ab-376069bbbcd3 /tidb/cdc/default/default/upstream/7362135384782445730 {"id":7362135384782445730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdba0cde-19ea-4401-b7ab-376069bbbcd3 {"id":"cdba0cde-19ea-4401-b7ab-376069bbbcd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130732} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a270bf6c2 cdba0cde-19ea-4401-b7ab-376069bbbcd3 /tidb/cdc/default/default/upstream/7362135384782445730 {"id":7362135384782445730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully [Fri Apr 26 19:25:35 CST 2024] <<<<<< START kafka consumer in changefeed_pause_resume case >>>>>> table ddl_manager.finish_mark not exists for 25-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... [Fri Apr 26 19:25:36 CST 2024] <<<<<< START cdc server in kafka_simple_basic_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.62176219.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:25:36 CST 2024] <<<<<< run test case multi_capture success! >>>>>> wait process cdc.test exit for 2-th time... table test.zstd_finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 26-th check, retry later wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689d50140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:9078, start at 2024-04-26 19:25:36.140694242 +0800 CST m=+5.166274361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:36.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:36.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:36.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:25:37 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: kafka... <<================= table changefeed_pause_resume.t1 not exists for 2-th check, retry later table test.zstd_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8201.out cli changefeed pause -c zstd PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 27-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689d50140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:9078, start at 2024-04-26 19:25:36.140694242 +0800 CST m=+5.166274361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:36.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:36.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:36.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689d4f2c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-9ctfl-whmhp, pid:9160, start at 2024-04-26 19:25:36.089986515 +0800 CST m=+5.063366268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:36.096 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:36.075 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:36.075 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:39 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c891549f-88c2-4c3d-b0b4-28f15ded3d21 {"id":"c891549f-88c2-4c3d-b0b4-28f15ded3d21","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130736} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a271a09cd c891549f-88c2-4c3d-b0b4-28f15ded3d21 /tidb/cdc/default/default/upstream/7362135395884452688 {"id":7362135395884452688,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c891549f-88c2-4c3d-b0b4-28f15ded3d21 {"id":"c891549f-88c2-4c3d-b0b4-28f15ded3d21","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130736} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a271a09cd c891549f-88c2-4c3d-b0b4-28f15ded3d21 /tidb/cdc/default/default/upstream/7362135395884452688 {"id":7362135395884452688,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c891549f-88c2-4c3d-b0b4-28f15ded3d21 {"id":"c891549f-88c2-4c3d-b0b4-28f15ded3d21","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130736} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a271a09cd c891549f-88c2-4c3d-b0b4-28f15ded3d21 /tidb/cdc/default/default/upstream/7362135395884452688 {"id":7362135395884452688,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.cli.6278.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-avro-789?protocol=simple&encoding-format=avro' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/conf/changefeed.toml table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later Create changefeed successfully! ID: 6d230044-ac83-418e-8dcf-87ae786adf93 Info: {"upstream_id":7362135395884452688,"namespace":"default","id":"6d230044-ac83-418e-8dcf-87ae786adf93","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-avro-789?protocol=simple\u0026encoding-format=avro","create_time":"2024-04-26T19:25:39.926424253+08:00","start_ts":449349088652820482,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349088652820482,"checkpoint_ts":449349088652820482,"checkpoint_time":"2024-04-26 19:25:39.795"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8237.out cli changefeed remove -c zstd [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10625.out cli tso query --pd=http://127.0.0.1:2379 + set +x Changefeed remove successfully. ID: zstd CheckpointTs: 449349088287129602 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] // timeout [Pipeline] } table ddl_manager.finish_mark not exists for 28-th check, retry later [Pipeline] { table changefeed_pause_resume.t3 exists [Pipeline] checkout [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout + set +x [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // container + set +x + tso='449349089134641154 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349089134641154 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:25:43 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1066510667.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh wait process cdc.test exit for 1-th time... No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4189a8f0; decorates RemoteLauncher[hudson.remoting.Channel@5f673531:JNLP4-connect connection from 10.233.127.99/10.233.127.99:57766] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { table ddl_manager.finish_mark not exists for 29-th check, retry later [Pipeline] // container check diff failed 1-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=b7d7a5bb-982d-42dd-82a1-4edfb2df3019 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-llwp6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3 pingcap_tiflow_pull_cdc_integration_kafka_test_1732-llwp6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // container [Pipeline] sh wait process cdc.test exit for 2-th time... [Pipeline] sh cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:25:44 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=ebeb8ef5-a065-49f5-b78d-cc52901f0eec BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-jb3h1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-jb3h1 pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: kafka... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=aa1e67b5-64d0-4b90-821b-c05b8e4e8451 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-0g9sx GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m pingcap_tiflow_pull_cdc_integration_kafka_test_1732-0g9sx GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=4ed61858-b959-44ac-90ad-7deb8c97f30b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-fq2th GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-fq2th pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-fq2th-dkzjm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:25:44 CST 2024] <<<<<< run test case owner_resign success! >>>>>> [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=63c218eb-dcd5-4b6d-994d-bc27d269f676 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2jnjw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-2jnjw pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 30-th check, retry later [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Verifying downstream PD is started... Commit message: "feat(tidb): test flashbacktest package (#2942)" check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:46 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe39ef4b-ea88-4239-b8c2-1d2883bf978b {"id":"fe39ef4b-ea88-4239-b8c2-1d2883bf978b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130743} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a272a01d5 fe39ef4b-ea88-4239-b8c2-1d2883bf978b /tidb/cdc/default/default/upstream/7362135416540719725 {"id":7362135416540719725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe39ef4b-ea88-4239-b8c2-1d2883bf978b {"id":"fe39ef4b-ea88-4239-b8c2-1d2883bf978b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130743} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a272a01d5 fe39ef4b-ea88-4239-b8c2-1d2883bf978b /tidb/cdc/default/default/upstream/7362135416540719725 {"id":7362135416540719725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe39ef4b-ea88-4239-b8c2-1d2883bf978b {"id":"fe39ef4b-ea88-4239-b8c2-1d2883bf978b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130743} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a272a01d5 fe39ef4b-ea88-4239-b8c2-1d2883bf978b /tidb/cdc/default/default/upstream/7362135416540719725 {"id":7362135416540719725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10726.out cli changefeed create --start-ts=449349089134641154 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml Create changefeed successfully! ID: 6e55ef79-324f-4ed9-a1f5-ca30713bfe45 Info: {"upstream_id":7362135416540719725,"namespace":"default","id":"6e55ef79-324f-4ed9-a1f5-ca30713bfe45","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:25:46.688625467+08:00","start_ts":449349089134641154,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349089134641154,"checkpoint_ts":449349089134641154,"checkpoint_time":"2024-04-26 19:25:41.633"} PASS [Fri Apr 26 19:25:46 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic_avro case >>>>>> coverage: 2.5% of statements in github.com/pingcap/tiflow/... > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 table ddl_manager.finish_mark not exists for 31-th check, retry later The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... check diff failed 3-th time, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:25:48 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> + set +x start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 32-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 33-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:25:51 CST 2024] <<<<<< run test case sink_hang success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 34-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Starting Upstream TiDB... \033[0;36m<<< Run all test success >>>\033[0m Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 35-th check, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:25:53 CST 2024] <<<<<< run test case sink_retry success! >>>>>> check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:25:55 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> Starting Upstream TiDB... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 36-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689e96f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:1292, start at 2024-04-26 19:25:57.088118467 +0800 CST m=+5.093095091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:57.095 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:57.053 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:57.053 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689e96f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:1292, start at 2024-04-26 19:25:57.088118467 +0800 CST m=+5.093095091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:57.095 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:57.053 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:57.053 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689e97b40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:1370, start at 2024-04-26 19:25:57.148660687 +0800 CST m=+5.102419355 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:57.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:57.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:57.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689e839c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:4507, start at 2024-04-26 19:25:55.828254219 +0800 CST m=+5.633434483 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:55.837 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:55.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:55.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689e839c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:4507, start at 2024-04-26 19:25:55.828254219 +0800 CST m=+5.633434483 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:55.837 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:55.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:55.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689e8444000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:4583, start at 2024-04-26 19:25:55.870921434 +0800 CST m=+5.624866880 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:55.877 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:55.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:55.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 37-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 11.85 secs (313969696 bytes/sec) [Pipeline] { Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2812.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:26:00 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.59795981.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ec3600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:1356, start at 2024-04-26 19:25:59.899020173 +0800 CST m=+5.225999947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:59.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:59.896 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:59.896 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 38-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh + set +x + tso='449349094107512833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349094107512833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:26:02 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.28482850.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later [Pipeline] // dir [Pipeline] } [Pipeline] container [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] // withCredentials [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ede080018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:1449, start at 2024-04-26 19:26:01.651132091 +0800 CST m=+5.153284826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:01.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:01.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:01.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ede080018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:1449, start at 2024-04-26 19:26:01.651132091 +0800 CST m=+5.153284826 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:01.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:01.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:01.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ee0400003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:1515, start at 2024-04-26 19:26:01.747744395 +0800 CST m=+5.190447014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:01.754 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:01.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:01.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout start tidb cluster in /tmp/tidb_cdc_test/kafka_sink_error_resume Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ec3600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:1356, start at 2024-04-26 19:25:59.899020173 +0800 CST m=+5.225999947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:59.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:59.896 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:59.896 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ec40c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:1437, start at 2024-04-26 19:25:59.956338912 +0800 CST m=+5.230251533 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:59.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:59.939 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:59.939 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // timeout [Pipeline] } [Pipeline] stage [Pipeline] { (Test) table ddl_manager.finish_mark not exists for 39-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] sh [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:03 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9264a9ba-88c1-40c6-b62b-ae0ef411d3b2 {"id":"9264a9ba-88c1-40c6-b62b-ae0ef411d3b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130760} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2772c8d2 9264a9ba-88c1-40c6-b62b-ae0ef411d3b2 /tidb/cdc/default/default/upstream/7362135491107185545 {"id":7362135491107185545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9264a9ba-88c1-40c6-b62b-ae0ef411d3b2 {"id":"9264a9ba-88c1-40c6-b62b-ae0ef411d3b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130760} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2772c8d2 9264a9ba-88c1-40c6-b62b-ae0ef411d3b2 /tidb/cdc/default/default/upstream/7362135491107185545 {"id":7362135491107185545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9264a9ba-88c1-40c6-b62b-ae0ef411d3b2 {"id":"9264a9ba-88c1-40c6-b62b-ae0ef411d3b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130760} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2772c8d2 9264a9ba-88c1-40c6-b62b-ae0ef411d3b2 /tidb/cdc/default/default/upstream/7362135491107185545 {"id":7362135491107185545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.6041.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cdc-test-11522?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml [Pipeline] // container [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] withCredentials + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=b75d88b6-dcd5-4b12-9450-8790de523734 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-whfsf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-whfsf pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: kafka... <<================= Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689edffc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:1352, start at 2024-04-26 19:26:01.749551132 +0800 CST m=+5.182170415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:01.759 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:01.727 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:01.727 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689edffc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:1352, start at 2024-04-26 19:26:01.749551132 +0800 CST m=+5.182170415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:01.759 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:01.727 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:01.727 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689ee1740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:1426, start at 2024-04-26 19:26:01.848587265 +0800 CST m=+5.225039178 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:01.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:01.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:01.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: 2af9f93c-903a-4833-a049-03e8d387b4c1 Info: {"upstream_id":7362135491107185545,"namespace":"default","id":"2af9f93c-903a-4833-a049-03e8d387b4c1","sink_uri":"kafka://127.0.0.1:9092/ticdc-cdc-test-11522?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:04.072155872+08:00","start_ts":449349094975733764,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349094975733764,"checkpoint_ts":449349094975733764,"checkpoint_time":"2024-04-26 19:26:03.915"} PASS [Pipeline] // node [Pipeline] } [Pipeline] { coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] // podTemplate [Pipeline] } Verifying downstream PD is started... check diff failed 5-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2830.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689efc880010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:10079, start at 2024-04-26 19:26:03.574325466 +0800 CST m=+5.237050071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:03.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:03.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:03.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:05 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2fcc9427-484f-49d3-89f4-f8890907cb8d {"id":"2fcc9427-484f-49d3-89f4-f8890907cb8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130762} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2781fbce 2fcc9427-484f-49d3-89f4-f8890907cb8d /tidb/cdc/default/default/upstream/7362135505038026120 {"id":7362135505038026120,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2fcc9427-484f-49d3-89f4-f8890907cb8d {"id":"2fcc9427-484f-49d3-89f4-f8890907cb8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130762} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2781fbce 2fcc9427-484f-49d3-89f4-f8890907cb8d /tidb/cdc/default/default/upstream/7362135505038026120 {"id":7362135505038026120,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2fcc9427-484f-49d3-89f4-f8890907cb8d {"id":"2fcc9427-484f-49d3-89f4-f8890907cb8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130762} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2781fbce 2fcc9427-484f-49d3-89f4-f8890907cb8d /tidb/cdc/default/default/upstream/7362135505038026120 {"id":7362135505038026120,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: b63c56ce-9007-4214-b0c4-4cc04726aeb1 Info: {"upstream_id":7362135505038026120,"namespace":"default","id":"b63c56ce-9007-4214-b0c4-4cc04726aeb1","sink_uri":"kafka://127.0.0.1:9092/ticdc-tiflash-test-16240?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:05.416523767+08:00","start_ts":449349094107512833,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349094107512833,"checkpoint_ts":449349094107512833,"checkpoint_time":"2024-04-26 19:26:00.603"} [Fri Apr 26 19:26:05 CST 2024] <<<<<< START kafka consumer in tiflash case >>>>>> [Fri Apr 26 19:26:05 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.28562858.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [Fri Apr 26 19:26:05 CST 2024] <<<<<< START kafka consumer in cdc case >>>>>> go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689f0e880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:10737, start at 2024-04-26 19:26:04.741343791 +0800 CST m=+5.109696521 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:04.747 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:04.706 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:04.706 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689f0e880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:10737, start at 2024-04-26 19:26:04.741343791 +0800 CST m=+5.109696521 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:04.747 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:04.706 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:04.706 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689f10d80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:10828, start at 2024-04-26 19:26:04.876958568 +0800 CST m=+5.180868300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:04.883 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:04.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:04.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 40-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2810.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading golang.org/x/text v0.14.0 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449349095286898689 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349095286898689 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:26:06 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.28632865.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689efc880010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:10079, start at 2024-04-26 19:26:03.574325466 +0800 CST m=+5.237050071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:03.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:03.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:03.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689efe000010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-tl08z-xxbx7, pid:10162, start at 2024-04-26 19:26:03.669166266 +0800 CST m=+5.277967706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:03.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:03.648 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:03.648 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later table test.finish_mark not exists for 3-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 41-th check, retry later + set +x + tso='449349095555334145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349095555334145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:26:07 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.28442846.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/stretchr/testify v1.9.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading cloud.google.com/go v0.112.2 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:08 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30ebe200-3a12-41b0-983b-5173d45f55a0 {"id":"30ebe200-3a12-41b0-983b-5173d45f55a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130765} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27847ccd 30ebe200-3a12-41b0-983b-5173d45f55a0 /tidb/cdc/default/default/upstream/7362135515011943953 {"id":7362135515011943953,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30ebe200-3a12-41b0-983b-5173d45f55a0 {"id":"30ebe200-3a12-41b0-983b-5173d45f55a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130765} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27847ccd 30ebe200-3a12-41b0-983b-5173d45f55a0 /tidb/cdc/default/default/upstream/7362135515011943953 {"id":7362135515011943953,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/30ebe200-3a12-41b0-983b-5173d45f55a0 {"id":"30ebe200-3a12-41b0-983b-5173d45f55a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130765} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27847ccd 30ebe200-3a12-41b0-983b-5173d45f55a0 /tidb/cdc/default/default/upstream/7362135515011943953 {"id":7362135515011943953,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.12201.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.opentelemetry.io/otel/metric v1.24.0 Create changefeed successfully! ID: 702c2af6-c864-436f-9c92-2b1e8305d247 Info: {"upstream_id":7362135515011943953,"namespace":"default","id":"702c2af6-c864-436f-9c92-2b1e8305d247","sink_uri":"kafka://127.0.0.1:9092/ticdc-new_ci_collation-test-22329?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:08.4490864+08:00","start_ts":449349095285325825,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349095285325825,"checkpoint_ts":449349095285325825,"checkpoint_time":"2024-04-26 19:26:05.096"} [Fri Apr 26 19:26:08 CST 2024] <<<<<< START kafka consumer in new_ci_collation case >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11534.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 check diff successfully table test.finish_mark not exists for 4-th check, retry later table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 42-th check, retry later table new_ci_collation_test.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/369fbd11-c8b8-41b5-a63a-29accb0e5ee4 {"id":"369fbd11-c8b8-41b5-a63a-29accb0e5ee4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130766} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a278db1d1 369fbd11-c8b8-41b5-a63a-29accb0e5ee4 /tidb/cdc/default/default/upstream/7362135528398469400 {"id":7362135528398469400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/369fbd11-c8b8-41b5-a63a-29accb0e5ee4 {"id":"369fbd11-c8b8-41b5-a63a-29accb0e5ee4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130766} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a278db1d1 369fbd11-c8b8-41b5-a63a-29accb0e5ee4 /tidb/cdc/default/default/upstream/7362135528398469400 {"id":7362135528398469400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/369fbd11-c8b8-41b5-a63a-29accb0e5ee4 {"id":"369fbd11-c8b8-41b5-a63a-29accb0e5ee4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130766} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a278db1d1 369fbd11-c8b8-41b5-a63a-29accb0e5ee4 /tidb/cdc/default/default/upstream/7362135528398469400 {"id":7362135528398469400,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2914.out cli changefeed create --start-ts=449349095286898689 '--sink-uri=kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-29102?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 2160e6c7-3f88-4393-8b52-1c1e18b249d9 Info: {"upstream_id":7362135528398469400,"namespace":"default","id":"2160e6c7-3f88-4393-8b52-1c1e18b249d9","sink_uri":"kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-29102?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:10.140006735+08:00","start_ts":449349095286898689,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349095286898689,"checkpoint_ts":449349095286898689,"checkpoint_time":"2024-04-26 19:26:05.102"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449349096310046723 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349096310046723 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:26:10 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1157311575.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:10 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e97ea89-2f20-4f29-a7d6-5d843fd5709c {"id":"3e97ea89-2f20-4f29-a7d6-5d843fd5709c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130767} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a278feed0 3e97ea89-2f20-4f29-a7d6-5d843fd5709c /tidb/cdc/default/default/upstream/7362135528159751583 {"id":7362135528159751583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e97ea89-2f20-4f29-a7d6-5d843fd5709c {"id":"3e97ea89-2f20-4f29-a7d6-5d843fd5709c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130767} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a278feed0 3e97ea89-2f20-4f29-a7d6-5d843fd5709c /tidb/cdc/default/default/upstream/7362135528159751583 {"id":7362135528159751583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e97ea89-2f20-4f29-a7d6-5d843fd5709c {"id":"3e97ea89-2f20-4f29-a7d6-5d843fd5709c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130767} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a278feed0 3e97ea89-2f20-4f29-a7d6-5d843fd5709c /tidb/cdc/default/default/upstream/7362135528159751583 {"id":7362135528159751583,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2904.out cli changefeed create --start-ts=449349095555334145 '--sink-uri=kafka://127.0.0.1:9092/ticdc-foreign-key-test-26312?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table test.finish_mark not exists for 5-th check, retry later Create changefeed successfully! ID: b3442a11-e74f-4cbe-aa2b-d8f9589852df Info: {"upstream_id":7362135528159751583,"namespace":"default","id":"b3442a11-e74f-4cbe-aa2b-d8f9589852df","sink_uri":"kafka://127.0.0.1:9092/ticdc-foreign-key-test-26312?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:11.167065306+08:00","start_ts":449349095555334145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349095555334145,"checkpoint_ts":449349095555334145,"checkpoint_time":"2024-04-26 19:26:06.126"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 43-th check, retry later + set +x [Fri Apr 26 19:26:11 CST 2024] <<<<<< START kafka consumer in many_pk_or_uk case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 check diff successfully + set +x + tso='449349096126545921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349096126545921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349096126545921 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 26 19:26:11 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1223912241.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 wait process cdc.test exit for 1-th time... go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/text v0.14.0 + set +x [Fri Apr 26 19:26:12 CST 2024] <<<<<< START kafka consumer in foreign_key case >>>>>> wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:26:12 CST 2024] <<<<<< run test case tiflash success! >>>>>> table test.finish_mark not exists for 6-th check, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table ddl_manager.finish_mark not exists for 44-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/shirou/gopsutil/v3 v3.24.2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:13 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65250c9e-cf68-42ba-8d2a-00730376d709 {"id":"65250c9e-cf68-42ba-8d2a-00730376d709","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130770} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2792c7d8 65250c9e-cf68-42ba-8d2a-00730376d709 /tidb/cdc/default/default/upstream/7362135528587744382 {"id":7362135528587744382,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65250c9e-cf68-42ba-8d2a-00730376d709 {"id":"65250c9e-cf68-42ba-8d2a-00730376d709","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130770} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2792c7d8 65250c9e-cf68-42ba-8d2a-00730376d709 /tidb/cdc/default/default/upstream/7362135528587744382 {"id":7362135528587744382,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65250c9e-cf68-42ba-8d2a-00730376d709 {"id":"65250c9e-cf68-42ba-8d2a-00730376d709","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130770} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2792c7d8 65250c9e-cf68-42ba-8d2a-00730376d709 /tidb/cdc/default/default/upstream/7362135528587744382 {"id":7362135528587744382,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11632.out cli changefeed create --start-ts=449349096310046723 '--sink-uri=kafka://127.0.0.1:9092/ticdc-partition-table-test-29829?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 Create changefeed successfully! ID: 8b92ea5d-ee99-4071-a891-b997a7fee11a Info: {"upstream_id":7362135528587744382,"namespace":"default","id":"8b92ea5d-ee99-4071-a891-b997a7fee11a","sink_uri":"kafka://127.0.0.1:9092/ticdc-partition-table-test-29829?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:14.092338531+08:00","start_ts":449349096310046723,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349096310046723,"checkpoint_ts":449349096310046723,"checkpoint_time":"2024-04-26 19:26:09.005"} PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 coverage: 2.4% of statements in github.com/pingcap/tiflow/... table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 not exists for 1-th check, retry later go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 table test.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:14 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4de4a0ed-9868-4aa4-8254-dfd4350b33bf {"id":"4de4a0ed-9868-4aa4-8254-dfd4350b33bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130772} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2797deda 4de4a0ed-9868-4aa4-8254-dfd4350b33bf /tidb/cdc/default/default/upstream/7362135537098808607 {"id":7362135537098808607,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4de4a0ed-9868-4aa4-8254-dfd4350b33bf {"id":"4de4a0ed-9868-4aa4-8254-dfd4350b33bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130772} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2797deda 4de4a0ed-9868-4aa4-8254-dfd4350b33bf /tidb/cdc/default/default/upstream/7362135537098808607 {"id":7362135537098808607,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4de4a0ed-9868-4aa4-8254-dfd4350b33bf {"id":"4de4a0ed-9868-4aa4-8254-dfd4350b33bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130772} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2797deda 4de4a0ed-9868-4aa4-8254-dfd4350b33bf /tidb/cdc/default/default/upstream/7362135537098808607 {"id":7362135537098808607,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349096126545921 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.12294.out cli changefeed create --start-ts=449349096126545921 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362135537098808607,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:26:15.167570096+08:00","start_ts":449349096126545921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349096126545921,"checkpoint_ts":449349096126545921,"checkpoint_time":"2024-04-26 19:26:08.305"} PASS table foreign_key.finish_mark not exists for 1-th check, retry later check diff failed 3-th time, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689fa17c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:9408, start at 2024-04-26 19:26:14.120464061 +0800 CST m=+6.071884755 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:14.126 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:14.111 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:14.111 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689fa17c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:9408, start at 2024-04-26 19:26:14.120464061 +0800 CST m=+6.071884755 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:14.126 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:14.111 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:14.111 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689fa0dc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:9492, start at 2024-04-26 19:26:14.096454767 +0800 CST m=+5.992178460 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:14.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:14.071 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:14.071 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x [Fri Apr 26 19:26:15 CST 2024] <<<<<< START kafka consumer in partition_table case >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 45-th check, retry later table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2850 0 --:--:-- --:--:-- --:--:-- 2870 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:26:08.305","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:26:16.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:26:08.305","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:26:16.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:26:08.305","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:26:16.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-04-26 19:26:08.305' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:26:08.305","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:26:16.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:26:08.305","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:26:16.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-04-26 19:26:16' + echo 'sink_checkpoint_ts is 2024-04-26' 19:26:08.305 sink_checkpoint_ts is 2024-04-26 19:26:08.305 ++ date -d '2024-04-26 19:26:08.305' +%s + checkpoint_timestamp=1714130768 ++ date -d '2024-04-26 19:26:16' +%s + current_timestamp=1714130776 + '[' 8 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 46-th check, retry later table test.finish_mark not exists for 8-th check, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Fri Apr 26 19:26:17 CST 2024] <<<<<< START cdc server in kafka_sink_error_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_sink_error_resume.1086210864.out server --log-file /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table foreign_key.finish_mark not exists for 3-th check, retry later check diff failed 5-th time, retry later table test.finish_mark not exists for 9-th check, retry later table test.t1 exists + sleep 5 table ddl_manager.finish_mark not exists for 47-th check, retry later table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table foreign_key.finish_mark not exists for 4-th check, retry later check diff successfully table test.finish_mark not exists for 10-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3a82680-fd5a-48c9-bb2f-91e48fd892b4 {"id":"c3a82680-fd5a-48c9-bb2f-91e48fd892b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27baa5d1 c3a82680-fd5a-48c9-bb2f-91e48fd892b4 /tidb/cdc/default/default/upstream/7362135569839624949 {"id":7362135569839624949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3a82680-fd5a-48c9-bb2f-91e48fd892b4 {"id":"c3a82680-fd5a-48c9-bb2f-91e48fd892b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27baa5d1 c3a82680-fd5a-48c9-bb2f-91e48fd892b4 /tidb/cdc/default/default/upstream/7362135569839624949 {"id":7362135569839624949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3a82680-fd5a-48c9-bb2f-91e48fd892b4 {"id":"c3a82680-fd5a-48c9-bb2f-91e48fd892b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27baa5d1 c3a82680-fd5a-48c9-bb2f-91e48fd892b4 /tidb/cdc/default/default/upstream/7362135569839624949 {"id":7362135569839624949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:26:21 CST 2024] <<<<<< START kafka consumer in kafka_sink_error_resume case >>>>>> check_changefeed_status 127.0.0.1:8300 496fd1ac-7f95-4de3-ac07-c02a435c5a3f warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=496fd1ac-7f95-4de3-ac07-c02a435c5a3f + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/496fd1ac-7f95-4de3-ac07-c02a435c5a3f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table ddl_manager.finish_mark not exists for 48-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 751 0 --:--:-- --:--:-- --:--:-- 754 + info='{"state":"normal","resolved_ts":449349099457609734,"checkpoint_ts":449349099457609734}' + echo '{"state":"normal","resolved_ts":449349099457609734,"checkpoint_ts":449349099457609734}' {"state":"normal","resolved_ts":449349099457609734,"checkpoint_ts":449349099457609734} ++ echo '{"state":"normal","resolved_ts":449349099457609734,"checkpoint_ts":449349099457609734}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a0123c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:1363, start at 2024-04-26 19:26:21.338645765 +0800 CST m=+5.215316475 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:21.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:21.327 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:21.327 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 5-th check, retry later check diff failed 1-th time, retry later table test.finish_mark not exists for 11-th check, retry later table ddl_manager.finish_mark not exists for 49-th check, retry later check_changefeed_status 127.0.0.1:8300 496fd1ac-7f95-4de3-ac07-c02a435c5a3f warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=496fd1ac-7f95-4de3-ac07-c02a435c5a3f + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/496fd1ac-7f95-4de3-ac07-c02a435c5a3f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 785 0 --:--:-- --:--:-- --:--:-- 788 + info='{"state":"normal","resolved_ts":449349099535990809,"checkpoint_ts":449349099535990809}' + echo '{"state":"normal","resolved_ts":449349099535990809,"checkpoint_ts":449349099535990809}' {"state":"normal","resolved_ts":449349099535990809,"checkpoint_ts":449349099535990809} ++ echo '{"state":"normal","resolved_ts":449349099535990809,"checkpoint_ts":449349099535990809}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 2-th time, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 6-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2999 0 --:--:-- --:--:-- --:--:-- 3037 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:26:22.705","puller_resolved_ts":"2024-04-26 19:26:16.855","last_synced_ts":"2024-04-26 19:26:16.906","now_ts":"2024-04-26 19:26:24.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:26:22.705","puller_resolved_ts":"2024-04-26' '19:26:16.855","last_synced_ts":"2024-04-26' '19:26:16.906","now_ts":"2024-04-26' '19:26:24.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:26:22.705","puller_resolved_ts":"2024-04-26' '19:26:16.855","last_synced_ts":"2024-04-26' '19:26:16.906","now_ts":"2024-04-26' '19:26:24.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a0123c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:1363, start at 2024-04-26 19:26:21.338645765 +0800 CST m=+5.215316475 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:21.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:21.327 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:21.327 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a012f40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:1450, start at 2024-04-26 19:26:21.387156377 +0800 CST m=+5.205801101 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:21.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:21.373 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:21.373 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table partition_table.t2 not exists for 2-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later table test.finish_mark exists check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2911.out cli tso query --pd=http://127.0.0.1:2379 table partition_table.t2 not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 51-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... check_changefeed_status 127.0.0.1:8300 496fd1ac-7f95-4de3-ac07-c02a435c5a3f warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=496fd1ac-7f95-4de3-ac07-c02a435c5a3f + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/496fd1ac-7f95-4de3-ac07-c02a435c5a3f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2183 0 --:--:-- --:--:-- --:--:-- 2198 + info='{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .last_warning.message + error_msg='kafka sink injected error' + [[ ! kafka sink injected error =~ kafka ]] run task successfully + set +x + tso='449349100968869890 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349100968869890 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:26:28 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.29572959.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 21.74 secs (171053940 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { check_changefeed_status 127.0.0.1:8300 496fd1ac-7f95-4de3-ac07-c02a435c5a3f normal + endpoint=127.0.0.1:8300 + changefeed_id=496fd1ac-7f95-4de3-ac07-c02a435c5a3f + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/496fd1ac-7f95-4de3-ac07-c02a435c5a3f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1846 0 --:--:-- --:--:-- --:--:-- 1848 + info='{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449349101174390788,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a053740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:4192, start at 2024-04-26 19:26:25.530646012 +0800 CST m=+5.259709329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:25.538 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:25.501 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:25.501 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a053740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:4192, start at 2024-04-26 19:26:25.530646012 +0800 CST m=+5.259709329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:25.538 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:25.501 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:25.501 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a054200013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:4268, start at 2024-04-26 19:26:25.566900214 +0800 CST m=+5.243632930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:25.573 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:25.544 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:25.544 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table foreign_key.finish_mark not exists for 8-th check, retry later check diff failed 4-th time, retry later table partition_table.t2 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 52-th check, retry later check_changefeed_status 127.0.0.1:8300 496fd1ac-7f95-4de3-ac07-c02a435c5a3f normal + endpoint=127.0.0.1:8300 + changefeed_id=496fd1ac-7f95-4de3-ac07-c02a435c5a3f + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/496fd1ac-7f95-4de3-ac07-c02a435c5a3f/status check diff successfully check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5709.out cli tso query --pd=http://127.0.0.1:2379 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2164 0 --:--:-- --:--:-- --:--:-- 2178 + info='{"state":"warning","resolved_ts":449349101698678789,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449349101698678789,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449349101698678789,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449349101698678789,"checkpoint_ts":449349099562205207,"last_warning":{"time":"2024-04-26T19:26:25.247228783+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table foreign_key.finish_mark not exists for 9-th check, retry later table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:31 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d193d69f-d45d-4def-a7bb-a0144038e7df {"id":"d193d69f-d45d-4def-a7bb-a0144038e7df","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130788} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27de41d5 d193d69f-d45d-4def-a7bb-a0144038e7df /tidb/cdc/default/default/upstream/7362135612567647011 {"id":7362135612567647011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d193d69f-d45d-4def-a7bb-a0144038e7df {"id":"d193d69f-d45d-4def-a7bb-a0144038e7df","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130788} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27de41d5 d193d69f-d45d-4def-a7bb-a0144038e7df /tidb/cdc/default/default/upstream/7362135612567647011 {"id":7362135612567647011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d193d69f-d45d-4def-a7bb-a0144038e7df {"id":"d193d69f-d45d-4def-a7bb-a0144038e7df","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130788} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27de41d5 d193d69f-d45d-4def-a7bb-a0144038e7df /tidb/cdc/default/default/upstream/7362135612567647011 {"id":7362135612567647011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.3009.out cli changefeed create --start-ts=449349100968869890 '--sink-uri=kafka://127.0.0.1:9092/ticdc-default-value-test-28543?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table ddl_manager.finish_mark not exists for 53-th check, retry later Create changefeed successfully! ID: bf7f7414-5cae-4730-86b2-7fdb693b6262 Info: {"upstream_id":7362135612567647011,"namespace":"default","id":"bf7f7414-5cae-4730-86b2-7fdb693b6262","sink_uri":"kafka://127.0.0.1:9092/ticdc-default-value-test-28543?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:31.889875937+08:00","start_ts":449349100968869890,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349100968869890,"checkpoint_ts":449349100968869890,"checkpoint_time":"2024-04-26 19:26:26.777"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449349102049951745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349102049951745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table foreign_key.finish_mark not exists for 10-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... table partition_table.finish_mark not exists for 2-th check, retry later + set +x [Fri Apr 26 19:26:33 CST 2024] <<<<<< START kafka consumer in default_value case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/sync v0.7.0 go: downloading golang.org/x/time v0.5.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 check diff successfully [Pipeline] sh ***************** properties ***************** "threadcount"="4" "insertproportion"="0" "mysql.user"="root" "updateproportion"="0" "mysql.db"="changefeed_error" "dotransactions"="false" "mysql.host"="127.0.0.1" "workload"="core" "scanproportion"="0" "readproportion"="0" "recordcount"="20" "readallfields"="true" "mysql.port"="4000" "operationcount"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.762449ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3245.3, Avg(us): 1674, Min(us): 889, Max(us): 3789, 95th(us): 4000, 99th(us): 4000 [Fri Apr 26 19:26:33 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57725774.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 table ddl_manager.finish_mark not exists for 54-th check, retry later wait process cdc.test exit for 2-th time... go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 go: downloading golang.org/x/text v0.14.0 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... [Pipeline] } [Pipeline] // timeout [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:26:34 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> check_changefeed_status 127.0.0.1:8300 496fd1ac-7f95-4de3-ac07-c02a435c5a3f normal + endpoint=127.0.0.1:8300 + changefeed_id=496fd1ac-7f95-4de3-ac07-c02a435c5a3f + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/496fd1ac-7f95-4de3-ac07-c02a435c5a3f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 780 0 --:--:-- --:--:-- --:--:-- 781 + info='{"state":"normal","resolved_ts":449349103009398789,"checkpoint_ts":449349102747254789}' + echo '{"state":"normal","resolved_ts":449349103009398789,"checkpoint_ts":449349102747254789}' {"state":"normal","resolved_ts":449349103009398789,"checkpoint_ts":449349102747254789} ++ echo '{"state":"normal","resolved_ts":449349103009398789,"checkpoint_ts":449349102747254789}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z '' ]] ++ echo '{"state":"normal","resolved_ts":449349103009398789,"checkpoint_ts":449349102747254789}' ++ jq -r .last_error + error_msg=null + [[ ! null == \n\u\l\l ]] ++ echo '{"state":"normal","resolved_ts":449349103009398789,"checkpoint_ts":449349102747254789}' ++ jq -r .last_warning + error_msg=null + [[ ! null == \n\u\l\l ]] + exit 0 run task successfully table kafka_sink_error_resume.t1 exists table kafka_sink_error_resume.t2 exists [Pipeline] // container [Pipeline] sh table foreign_key.finish_mark not exists for 11-th check, retry later check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 3-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 55-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=057723f5-5ba4-4997-8aef-66b686317906 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-rp194 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b pingcap_tiflow_pull_cdc_integration_kafka_test_1732-rp194 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-rp194-zw24b GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:26:35 CST 2024] <<<<<< skip test case clustered_index for kafka! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:26:35 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:36 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 {"id":"822ff622-fb8d-472c-a6a7-8a19e1c42323","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130793} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea75f9 822ff622-fb8d-472c-a6a7-8a19e1c42323 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 {"id":"822ff622-fb8d-472c-a6a7-8a19e1c42323","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130793} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea75f9 822ff622-fb8d-472c-a6a7-8a19e1c42323 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 {"id":"822ff622-fb8d-472c-a6a7-8a19e1c42323","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130793} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea75f9 822ff622-fb8d-472c-a6a7-8a19e1c42323 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5835.out cli changefeed create --start-ts=449349102049951745 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error table foreign_key.finish_mark not exists for 12-th check, retry later Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7362135623954906196,"namespace":"default","id":"changefeed-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:36.988474952+08:00","start_ts":449349102049951745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349102049951745,"checkpoint_ts":449349102049951745,"checkpoint_time":"2024-04-26 19:26:30.901"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table partition_table.finish_mark not exists for 4-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 56-th check, retry later check diff successfully check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:26:38 CST 2024] <<<<<< START kafka consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s table test.finish_mark not exists for 1-th check, retry later + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449349102049951745, "checkpoint_time": "2024-04-26 19:26:30.901", "error": { "time": "2024-04-26T19:26:37.11930775+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at 449349102049951745" } }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449349102049951745, "checkpoint_time": "2024-04-26 19:26:30.901", "error": { "time": "2024-04-26T19:26:37.11930775+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at 449349102049951745" } }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449349102049951745, "checkpoint_time": "2024-04-26 19:26:30.901", "error": { "time": "2024-04-26T19:26:37.11930775+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at 449349102049951745" } } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449349102049951745, '"checkpoint_time":' '"2024-04-26' '19:26:30.901",' '"error":' '{' '"time":' '"2024-04-26T19:26:37.11930775+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at '449349102049951745"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449349102049951745, '"checkpoint_time":' '"2024-04-26' '19:26:30.901",' '"error":' '{' '"time":' '"2024-04-26T19:26:37.11930775+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at '449349102049951745"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at 449349102049951745' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449349102049951445 is earlier than or equal to GC safepoint at 449349102049951745 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5942.out cli changefeed resume -c changefeed-error table foreign_key.finish_mark exists check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) check diff failed 3-th time, retry later [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 2-th check, retry later [Pipeline] } PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table partition_table.finish_mark not exists for 5-th check, retry later [Pipeline] // timeout go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 57-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } check diff successfully [Pipeline] // node [2024/04/26 19:26:36.012 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/04/26 19:26:36.012 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7362135185365210057] [2024/04/26 19:26:36.012 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:26:36.012 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/04/26 19:26:36.013 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/04/26 19:26:36.013 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:26:36.014 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/04/26 19:26:36.014 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7362135185365210057] [2024/04/26 19:26:36.014 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:26:36.014 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/04/26 19:26:36.015 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/04/26 19:26:36.015 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:26:36.015 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/04/26 19:26:36.016 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/04/26 19:26:36.017 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=2] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:26:40 CST 2024] <<<<<< run test case foreign_key success! >>>>>> [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... + set +x table changefeed_error.usertable not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table partition_table.finish_mark not exists for 6-th check, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:26:41 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> table test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a12324001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:4229, start at 2024-04-26 19:26:38.838607198 +0800 CST m=+5.187622208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:38.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:38.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:38.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a12324001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:4229, start at 2024-04-26 19:26:38.838607198 +0800 CST m=+5.187622208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:38.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:38.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:38.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a126300010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:4282, start at 2024-04-26 19:26:39.005218317 +0800 CST m=+5.292107854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:39.011 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:38.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:38.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-610fq io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1732-d7s7d-610fq] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) table changefeed_error.usertable exists check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 7-th check, retry later check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 59-th check, retry later table test.finish_mark not exists for 4-th check, retry later [Fri Apr 26 19:26:43 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.56695671.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark exists check diff successfully check diff successfully ***************** properties ***************** "dotransactions"="false" "updateproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "insertproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="4" "mysql.db"="changefeed_error" "recordcount"="20" "operationcount"="0" "workload"="core" "mysql.user"="root" "readallfields"="true" "mysql.port"="4000" ********************************************** Run finished, takes 4.841558ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5822.6, Avg(us): 816, Min(us): 445, Max(us): 2031, 95th(us): 2000, 99th(us): 3000 check diff successfully {"id":"822ff622-fb8d-472c-a6a7-8a19e1c42323","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130793} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 + [[ /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 60-th check, retry later table test.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... [2024/04/26 19:26:46.034 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/04/26 19:26:46.034 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/04/26 19:26:46.034 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/04/26 19:26:46.034 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/04/26 19:26:46.034 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] process cdc.test already exit [Fri Apr 26 19:26:46 CST 2024] <<<<<< run test case partition_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 + [[ /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:26:47 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e35b986-bd72-4684-a5bc-3290d33a8520 {"id":"0e35b986-bd72-4684-a5bc-3290d33a8520","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130804} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28249f32 0e35b986-bd72-4684-a5bc-3290d33a8520 /tidb/cdc/default/default/upstream/7362135687575736693 {"id":7362135687575736693,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e35b986-bd72-4684-a5bc-3290d33a8520 {"id":"0e35b986-bd72-4684-a5bc-3290d33a8520","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130804} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28249f32 0e35b986-bd72-4684-a5bc-3290d33a8520 /tidb/cdc/default/default/upstream/7362135687575736693 {"id":7362135687575736693,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e35b986-bd72-4684-a5bc-3290d33a8520 {"id":"0e35b986-bd72-4684-a5bc-3290d33a8520","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130804} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28249f32 0e35b986-bd72-4684-a5bc-3290d33a8520 /tidb/cdc/default/default/upstream/7362135687575736693 {"id":7362135687575736693,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c3819d48-2630-42d0-b5c8-cdb2cf1a68bb Info: {"upstream_id":7362135687575736693,"namespace":"default","id":"c3819d48-2630-42d0-b5c8-cdb2cf1a68bb","sink_uri":"kafka://127.0.0.1:9092/ticdc-region-merge-test-22740?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:26:47.246976166+08:00","start_ts":449349106294849539,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349106294849539,"checkpoint_ts":449349106294849539,"checkpoint_time":"2024-04-26 19:26:47.094"} [Fri Apr 26 19:26:47 CST 2024] <<<<<< START kafka consumer in region_merge case >>>>>> table ddl_manager.finish_mark not exists for 61-th check, retry later table test.finish_mark exists check diff successfully Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1732-c3p4d-r8xs5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0d7cb36286045afaca19d333f7b7a5f37512c36a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1732-c3p4d" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" check diff successfully Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git wait process cdc.test exit for 1-th time... split_and_random_merge scale: 20 wait process cdc.test exit for 2-th time... No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1719aab6; decorates RemoteLauncher[hudson.remoting.Channel@34b786dd:JNLP4-connect connection from 10.233.66.160/10.233.66.160:38744] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:26:49 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 table ddl_manager.finish_mark not exists for 62-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... check diff failed 1-th time, retry later > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 + [[ /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/822ff622-fb8d-472c-a6a7-8a19e1c42323 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 63-th check, retry later Commit message: "feat(tidb): test flashbacktest package (#2942)" [Pipeline] withEnv [Pipeline] { table test.t2 not exists for 1-th check, retry later [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 64-th check, retry later table test.t2 not exists for 2-th check, retry later *************************** 1. row *************************** count(distinct region_id): 2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 65-th check, retry later check diff failed 4-th time, retry later table test.t2 not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 66-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:26:57 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62026204.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later table test.t2 not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... split_and_random_merge scale: 40 \033[0;36m<<< Run all test success >>>\033[0m table ddl_manager.finish_mark not exists for 67-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:26:57 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> check diff failed 6-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12241913-6ec5-41cd-b512-60193d538607 {"id":"12241913-6ec5-41cd-b512-60193d538607","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea76ee 12241913-6ec5-41cd-b512-60193d538607 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362135623954906196,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:26:36.988474952+08:00","start-ts":449349102049951745,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-04-26T19:26:57.856469352+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349109114732545} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449349105562681354,"min-table-barrier-ts":449349105562681354,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12241913-6ec5-41cd-b512-60193d538607 {"id":"12241913-6ec5-41cd-b512-60193d538607","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea76ee 12241913-6ec5-41cd-b512-60193d538607 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362135623954906196,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:26:36.988474952+08:00","start-ts":449349102049951745,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-04-26T19:26:57.856469352+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349109114732545} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449349105562681354,"min-table-barrier-ts":449349105562681354,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12241913-6ec5-41cd-b512-60193d538607 {"id":"12241913-6ec5-41cd-b512-60193d538607","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea76ee 12241913-6ec5-41cd-b512-60193d538607 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362135623954906196,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:26:36.988474952+08:00","start-ts":449349102049951745,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-04-26T19:26:57.856469352+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349109114732545} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449349105562681354,"min-table-barrier-ts":449349105562681354,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362135623954906196 + grep -q 'etcd info' {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449349105562681354, "checkpoint_time": "2024-04-26 19:26:44.301", "error": { "time": "2024-04-26T19:26:57.856469352+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449349105562681354, "checkpoint_time": "2024-04-26 19:26:44.301", "error": { "time": "2024-04-26T19:26:57.856469352+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449349105562681354, "checkpoint_time": "2024-04-26 19:26:44.301", "error": { "time": "2024-04-26T19:26:57.856469352+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449349105562681354, '"checkpoint_time":' '"2024-04-26' '19:26:44.301",' '"error":' '{' '"time":' '"2024-04-26T19:26:57.856469352+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449349105562681354, '"checkpoint_time":' '"2024-04-26' '19:26:44.301",' '"error":' '{' '"time":' '"2024-04-26T19:26:57.856469352+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6315.out cli changefeed remove -c changefeed-error table test.t2 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2736c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:4468, start at 2024-04-26 19:27:00.348483322 +0800 CST m=+5.195389236 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:00.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:00.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:00.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449349105562681354 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 68-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully table test.t2 not exists for 6-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc) 3719054848 bytes in 8.04 secs (462338562 bytes/sec) [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2736c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:4468, start at 2024-04-26 19:27:00.348483322 +0800 CST m=+5.195389236 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:00.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:00.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:00.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a27340000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:4549, start at 2024-04-26 19:27:00.31910626 +0800 CST m=+5.115443660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:00.326 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:00.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:00.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 69-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:27:04 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) check diff failed 1-th time, retry later [Pipeline] container [Pipeline] { + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.64176419.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.t2 not exists for 7-th check, retry later [Pipeline] // cache [Pipeline] } [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] // dir [Pipeline] } [Pipeline] sh [Pipeline] // withCredentials [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/mq_sink_dispatcher Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [Pipeline] // timeout start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [Pipeline] } + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] // stage [Pipeline] } [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container table ddl_manager.finish_mark not exists for 70-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // container [Pipeline] } [Pipeline] sh check diff failed 2-th time, retry later table test.t2 not exists for 8-th check, retry later [Pipeline] // node [Pipeline] } [Fri Apr 26 19:27:06 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.58955897.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // podTemplate + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=d80a164c-9852-4613-a4cc-d82657781c87 JENKINS_NODE_COOKIE=57ec79f8-18ea-425f-95ed-4af1daadbe04 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1783794331430162442","prowjobid":"d80a164c-9852-4613-a4cc-d82657781c87","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","base_link":"https://github.com/pingcap/tiflow/commit/d0329d7f1ca9a1d0de81a565051a09fe7e9231bd","pulls":[{"number":10919,"author":"lidezhu","sha":"23327bce12e22486d293c05f3e682e66d2bbc2ae","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/23327bce12e22486d293c05f3e682e66d2bbc2ae","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1732 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783794331430162442 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1732/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1732-c3p4d GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh pingcap_tiflow_pull_cdc_integration_kafka_test_1732-c3p4d GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1732 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 71-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:07 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2f082cd-d0ed-40c9-8dd3-999427fb1828 {"id":"d2f082cd-d0ed-40c9-8dd3-999427fb1828","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130824} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea773c d2f082cd-d0ed-40c9-8dd3-999427fb1828 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2f082cd-d0ed-40c9-8dd3-999427fb1828 {"id":"d2f082cd-d0ed-40c9-8dd3-999427fb1828","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130824} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea773c d2f082cd-d0ed-40c9-8dd3-999427fb1828 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2f082cd-d0ed-40c9-8dd3-999427fb1828 {"id":"d2f082cd-d0ed-40c9-8dd3-999427fb1828","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130824} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea773c d2f082cd-d0ed-40c9-8dd3-999427fb1828 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6473.out cli changefeed create --start-ts=449349102049951745 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-1 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7362135623954906196,"namespace":"default","id":"changefeed-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:08.006540305+08:00","start_ts":449349102049951745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349102049951745,"checkpoint_ts":449349102049951745,"checkpoint_time":"2024-04-26 19:26:30.901"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 9-th check, retry later check diff failed 3-th time, retry later + set +x table ddl_manager.finish_mark not exists for 72-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54d29071-5140-4f75-adbc-c2ef336e0b36 {"id":"54d29071-5140-4f75-adbc-c2ef336e0b36","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2872d917 54d29071-5140-4f75-adbc-c2ef336e0b36 /tidb/cdc/default/default/upstream/7362135781067549529 {"id":7362135781067549529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54d29071-5140-4f75-adbc-c2ef336e0b36 {"id":"54d29071-5140-4f75-adbc-c2ef336e0b36","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2872d917 54d29071-5140-4f75-adbc-c2ef336e0b36 /tidb/cdc/default/default/upstream/7362135781067549529 {"id":7362135781067549529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54d29071-5140-4f75-adbc-c2ef336e0b36 {"id":"54d29071-5140-4f75-adbc-c2ef336e0b36","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2872d917 54d29071-5140-4f75-adbc-c2ef336e0b36 /tidb/cdc/default/default/upstream/7362135781067549529 {"id":7362135781067549529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5953.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-batch-add-table-test-16027?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 3c488278-63a8-4525-810f-f9db5565764f Info: {"upstream_id":7362135781067549529,"namespace":"default","id":"3c488278-63a8-4525-810f-f9db5565764f","sink_uri":"kafka://127.0.0.1:9092/ticdc-batch-add-table-test-16027?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:09.985898859+08:00","start_ts":449349112251023365,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349112251023365,"checkpoint_ts":449349112251023365,"checkpoint_time":"2024-04-26 19:27:09.815"} PASS check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2409 0 --:--:-- --:--:-- --:--:-- 2432 + info='{"state":"warning","resolved_ts":449349102430322713,"checkpoint_ts":449349102430322713,"last_warning":{"time":"2024-04-26T19:27:09.588343217+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449349102430322713,"checkpoint_ts":449349102430322713,"last_warning":{"time":"2024-04-26T19:27:09.588343217+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449349102430322713,"checkpoint_ts":449349102430322713,"last_warning":{"time":"2024-04-26T19:27:09.588343217+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449349102430322713,"checkpoint_ts":449349102430322713,"last_warning":{"time":"2024-04-26T19:27:09.588343217+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2f8d00003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:4219, start at 2024-04-26 19:27:08.853538274 +0800 CST m=+5.153808692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:08.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:08.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:08.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2f8d00003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:4219, start at 2024-04-26 19:27:08.853538274 +0800 CST m=+5.153808692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:08.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:08.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:08.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2fb140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:4291, start at 2024-04-26 19:27:09.027466162 +0800 CST m=+5.270553951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:09.036 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:08.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:08.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.t2 not exists for 10-th check, retry later + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449349102430322713,"checkpoint_ts":449349102430322713,"last_warning":{"time":"2024-04-26T19:27:09.588343217+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6524.out cli changefeed remove -c changefeed-error-1 check diff failed 4-th time, retry later Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449349102430322713 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 73-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:27:11 CST 2024] <<<<<< START kafka consumer in batch_add_table case >>>>>> table batch_add_table.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5607.out cli tso query --pd=http://127.0.0.1:2379 + set +x check diff failed 5-th time, retry later table test.t2 not exists for 11-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table batch_add_table.finish_mark not exists for 2-th check, retry later + set +x + tso='449349112916082689 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349112916082689 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:27:13 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.65956597.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 74-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later table test.t2 not exists for 12-th check, retry later [Fri Apr 26 19:27:14 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.56595661.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 75-th check, retry later table batch_add_table.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:27:16 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a35a3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:12473, start at 2024-04-26 19:27:15.128779053 +0800 CST m=+5.250055729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:15.137 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:15.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:15.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 exists check diff successfully check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:17 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/350dce61-d260-4819-95a2-4fa1a7114377 {"id":"350dce61-d260-4819-95a2-4fa1a7114377","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea77bc 350dce61-d260-4819-95a2-4fa1a7114377 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/350dce61-d260-4819-95a2-4fa1a7114377 {"id":"350dce61-d260-4819-95a2-4fa1a7114377","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea77bc 350dce61-d260-4819-95a2-4fa1a7114377 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/350dce61-d260-4819-95a2-4fa1a7114377 {"id":"350dce61-d260-4819-95a2-4fa1a7114377","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea77bc 350dce61-d260-4819-95a2-4fa1a7114377 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6647.out cli changefeed create --start-ts=449349102049951745 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7362135623954906196,"namespace":"default","id":"changefeed-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:17.467837506+08:00","start_ts":449349102049951745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349102049951745,"checkpoint_ts":449349102049951745,"checkpoint_time":"2024-04-26 19:26:30.901"} PASS wait process cdc.test exit for 2-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 76-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a35a3c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:12473, start at 2024-04-26 19:27:15.128779053 +0800 CST m=+5.250055729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:15.137 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:15.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:15.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a35bb00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:12554, start at 2024-04-26 19:27:15.222736085 +0800 CST m=+5.290548356 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:15.230 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:15.230 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:15.230 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:27:18 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:17 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8fd4db9-4e14-413b-9f43-36d2e7c5f63f {"id":"b8fd4db9-4e14-413b-9f43-36d2e7c5f63f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130835} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a289430ec b8fd4db9-4e14-413b-9f43-36d2e7c5f63f /tidb/cdc/default/default/upstream/7362135813865603484 {"id":7362135813865603484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8fd4db9-4e14-413b-9f43-36d2e7c5f63f {"id":"b8fd4db9-4e14-413b-9f43-36d2e7c5f63f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130835} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a289430ec b8fd4db9-4e14-413b-9f43-36d2e7c5f63f /tidb/cdc/default/default/upstream/7362135813865603484 {"id":7362135813865603484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b8fd4db9-4e14-413b-9f43-36d2e7c5f63f {"id":"b8fd4db9-4e14-413b-9f43-36d2e7c5f63f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130835} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a289430ec b8fd4db9-4e14-413b-9f43-36d2e7c5f63f /tidb/cdc/default/default/upstream/7362135813865603484 {"id":7362135813865603484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5720.out cli changefeed create --start-ts=449349112916082689 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-15531?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 325800a0-b6c1-439a-9914-270b16005c11 Info: {"upstream_id":7362135813865603484,"namespace":"default","id":"325800a0-b6c1-439a-9914-270b16005c11","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-15531?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=ddl_puller_lag\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:18.440240439+08:00","start_ts":449349112916082689,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349112916082689,"checkpoint_ts":449349112916082689,"checkpoint_time":"2024-04-26 19:27:12.352"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449349102049951745, "checkpoint_time": "2024-04-26 19:26:30.901", "error": { "time": "2024-04-26T19:27:17.552109621+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449349102049951745, "checkpoint_time": "2024-04-26 19:26:30.901", "error": { "time": "2024-04-26T19:27:17.552109621+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449349102049951745, "checkpoint_time": "2024-04-26 19:26:30.901", "error": { "time": "2024-04-26T19:27:17.552109621+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449349102049951745, '"checkpoint_time":' '"2024-04-26' '19:26:30.901",' '"error":' '{' '"time":' '"2024-04-26T19:27:17.552109621+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449349102049951745, '"checkpoint_time":' '"2024-04-26' '19:26:30.901",' '"error":' '{' '"time":' '"2024-04-26T19:27:17.552109621+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449349102049951745 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6727.out cli changefeed remove -c changefeed-error-2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a37408000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:7574, start at 2024-04-26 19:27:16.749788335 +0800 CST m=+5.205621461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:16.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:16.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:16.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a37408000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:7574, start at 2024-04-26 19:27:16.749788335 +0800 CST m=+5.205621461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:16.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:16.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:16.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a37588000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:7656, start at 2024-04-26 19:27:16.844843587 +0800 CST m=+5.252867987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:16.855 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:16.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:16.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 77-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:27:19 CST 2024] <<<<<< START kafka consumer in ddl_puller_lag case >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 26 19:27:20 CST 2024] <<<<<< START cdc server in mq_sink_dispatcher case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.1394513947.out server --log-file /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc.log --log-level info --data-dir /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449349102049951745 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + set +x table ddl_manager.finish_mark not exists for 78-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a3ae400012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh, pid:1284, start at 2024-04-26 19:27:20.489601899 +0800 CST m=+5.188371537 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:20.496 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:20.464 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:20.464 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a3ae400012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh, pid:1284, start at 2024-04-26 19:27:20.489601899 +0800 CST m=+5.188371537 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:20.496 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:20.464 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:20.464 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a3af080015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh, pid:1368, start at 2024-04-26 19:27:20.564213021 +0800 CST m=+5.209771414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:20.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:20.564 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:20.564 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 26 19:27:21 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.91169118.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:27:22 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.67766778.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:23 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/737b9efd-d537-400f-a78e-a8d760ff9e3e {"id":"737b9efd-d537-400f-a78e-a8d760ff9e3e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28ae4dcc 737b9efd-d537-400f-a78e-a8d760ff9e3e /tidb/cdc/default/default/upstream/7362135842719348432 {"id":7362135842719348432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/737b9efd-d537-400f-a78e-a8d760ff9e3e {"id":"737b9efd-d537-400f-a78e-a8d760ff9e3e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28ae4dcc 737b9efd-d537-400f-a78e-a8d760ff9e3e /tidb/cdc/default/default/upstream/7362135842719348432 {"id":7362135842719348432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/737b9efd-d537-400f-a78e-a8d760ff9e3e {"id":"737b9efd-d537-400f-a78e-a8d760ff9e3e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28ae4dcc 737b9efd-d537-400f-a78e-a8d760ff9e3e /tidb/cdc/default/default/upstream/7362135842719348432 {"id":7362135842719348432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark exists [Fri Apr 26 19:27:23 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.28052807.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14026.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:24 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dae5fbc-33e3-464d-b276-bb62a54a3752 {"id":"0dae5fbc-33e3-464d-b276-bb62a54a3752","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130842} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28b0d9ca 0dae5fbc-33e3-464d-b276-bb62a54a3752 /tidb/cdc/default/default/upstream/7362135847945042033 {"id":7362135847945042033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dae5fbc-33e3-464d-b276-bb62a54a3752 {"id":"0dae5fbc-33e3-464d-b276-bb62a54a3752","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130842} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28b0d9ca 0dae5fbc-33e3-464d-b276-bb62a54a3752 /tidb/cdc/default/default/upstream/7362135847945042033 {"id":7362135847945042033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0dae5fbc-33e3-464d-b276-bb62a54a3752 {"id":"0dae5fbc-33e3-464d-b276-bb62a54a3752","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130842} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28b0d9ca 0dae5fbc-33e3-464d-b276-bb62a54a3752 /tidb/cdc/default/default/upstream/7362135847945042033 {"id":7362135847945042033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.9175.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 4-th time, retry later + set +x + tso='449349115927068675 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349115927068675 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14072.out cli changefeed create --start-ts=449349115927068675 '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/changefeed.toml Create changefeed successfully! ID: test Info: {"upstream_id":7362135842719348432,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:27:25.795680698+08:00","start_ts":449349115927068675,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["verify.t"],"partition":"index-value"},{"matcher":["dispatcher.index"],"partition":"index-value","index":"idx_a"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349115927068675,"checkpoint_ts":449349115927068675,"checkpoint_time":"2024-04-26 19:27:23.838"} PASS wait process cdc.test exit for 4-th time... coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:26 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d33c267e-1300-46b4-9dd0-a759841de1b7 {"id":"d33c267e-1300-46b4-9dd0-a759841de1b7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130843} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea7821 d33c267e-1300-46b4-9dd0-a759841de1b7 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d33c267e-1300-46b4-9dd0-a759841de1b7 {"id":"d33c267e-1300-46b4-9dd0-a759841de1b7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130843} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea7821 d33c267e-1300-46b4-9dd0-a759841de1b7 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d33c267e-1300-46b4-9dd0-a759841de1b7 {"id":"d33c267e-1300-46b4-9dd0-a759841de1b7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130843} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f1a27ea7821 d33c267e-1300-46b4-9dd0-a759841de1b7 /tidb/cdc/default/default/upstream/7362135623954906196 {"id":7362135623954906196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6830.out cli changefeed create --start-ts=0 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-initialize-error cdc.test: no process found wait process cdc.test exit for 5-th time... process cdc.test already exit [Fri Apr 26 19:27:26 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7362135623954906196,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:26.554473973+08:00","start_ts":449349116598943756,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349116598943756,"checkpoint_ts":449349116598943756,"checkpoint_time":"2024-04-26 19:27:26.401"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:26 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39a951b0-c9a2-45b4-af67-cbbd2bce4f74 {"id":"39a951b0-c9a2-45b4-af67-cbbd2bce4f74","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130843} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28c382c3 39a951b0-c9a2-45b4-af67-cbbd2bce4f74 /tidb/cdc/default/default/upstream/7362135867920361408 {"id":7362135867920361408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39a951b0-c9a2-45b4-af67-cbbd2bce4f74 {"id":"39a951b0-c9a2-45b4-af67-cbbd2bce4f74","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130843} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28c382c3 39a951b0-c9a2-45b4-af67-cbbd2bce4f74 /tidb/cdc/default/default/upstream/7362135867920361408 {"id":7362135867920361408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39a951b0-c9a2-45b4-af67-cbbd2bce4f74 {"id":"39a951b0-c9a2-45b4-af67-cbbd2bce4f74","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130843} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28c382c3 39a951b0-c9a2-45b4-af67-cbbd2bce4f74 /tidb/cdc/default/default/upstream/7362135867920361408 {"id":7362135867920361408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:27:27 CST 2024] <<<<<< START kafka consumer in changefeed_finish case >>>>>> + set +x + tso='449349116307177476 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349116307177476 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.9214.out cli changefeed create --start-ts=449349116307177476 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-29759?protocol=simple' -c simple-handle-key-only --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: simple-handle-key-only Info: {"upstream_id":7362135847945042033,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-29759?protocol=simple","create_time":"2024-04-26T19:27:27.214223807+08:00","start_ts":449349116307177476,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349116307177476,"checkpoint_ts":449349116307177476,"checkpoint_time":"2024-04-26 19:27:25.288"} PASS check diff failed 5-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349115927068675, "checkpoint_time": "2024-04-26 19:27:23.838", "error": null }' + echo '{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349115927068675, "checkpoint_time": "2024-04-26 19:27:23.838", "error": null }' { "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349115927068675, "checkpoint_time": "2024-04-26 19:27:23.838", "error": null } ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449349115927068675, '"checkpoint_time":' '"2024-04-26' '19:27:23.838",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449349115927068675, '"checkpoint_time":' '"2024-04-26' '19:27:23.838",' '"error":' null '}' ++ jq -r .error.message coverage: 2.5% of statements in github.com/pingcap/tiflow/... + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s check diff failed 1-th time, retry later + info='{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349115927068675, "checkpoint_time": "2024-04-26 19:27:23.838", "error": null }' + echo '{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349115927068675, "checkpoint_time": "2024-04-26 19:27:23.838", "error": null }' { "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349115927068675, "checkpoint_time": "2024-04-26 19:27:23.838", "error": null } ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449349115927068675, '"checkpoint_time":' '"2024-04-26' '19:27:23.838",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449349116598943756, "checkpoint_time": "2024-04-26 19:27:26.401", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449349116598943756, "checkpoint_time": "2024-04-26 19:27:26.401", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449349116598943756, "checkpoint_time": "2024-04-26 19:27:26.401", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449349116598943756, '"checkpoint_time":' '"2024-04-26' '19:27:26.401",' '"error":' '{' '"time":' '"2024-04-26T19:27:26.732625976+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x check diff successfully check diff failed 2-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": null }' + echo '{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": null }' { "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": null } ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449349116910108701, '"checkpoint_time":' '"2024-04-26' '19:27:27.588",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 2-th time, retry later + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449349116598943756, "checkpoint_time": "2024-04-26 19:27:26.401", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449349116598943756, "checkpoint_time": "2024-04-26 19:27:26.401", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449349116598943756, "checkpoint_time": "2024-04-26 19:27:26.401", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449349116598943756, '"checkpoint_time":' '"2024-04-26' '19:27:26.401",' '"error":' '{' '"time":' '"2024-04-26T19:27:26.732625976+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... check diff failed 1-th time, retry later check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": { "time": "2024-04-26T19:27:31.233385137+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' + echo '{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": { "time": "2024-04-26T19:27:31.233385137+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' { "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": { "time": "2024-04-26T19:27:31.233385137+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } } ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449349116910108701, '"checkpoint_time":' '"2024-04-26' '19:27:27.588",' '"error":' '{' '"time":' '"2024-04-26T19:27:31.233385137+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449349116910108701, '"checkpoint_time":' '"2024-04-26' '19:27:27.588",' '"error":' '{' '"time":' '"2024-04-26T19:27:31.233385137+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a' + [[ ! [CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a =~ ErrDispatcherFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14350.out cli changefeed update -c test '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/new_changefeed.toml --no-confirm check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449349118420844552, "checkpoint_time": "2024-04-26 19:27:33.351", "error": null }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449349118420844552, "checkpoint_time": "2024-04-26 19:27:33.351", "error": null }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449349118420844552, "checkpoint_time": "2024-04-26 19:27:33.351", "error": null } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449349118420844552, '"checkpoint_time":' '"2024-04-26' '19:27:33.351",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449349118420844552, '"checkpoint_time":' '"2024-04-26' '19:27:33.351",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7031.out cli changefeed pause -c changefeed-initialize-error Diff of changefeed config: {Type:update Path:[Config SyncPointInterval] From: To:0xc003934ce8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003934cf8} {Type:update Path:[Config Sink DispatchRules 0 Matcher 0] From:verify.t To:dispatcher.index} {Type:delete Path:[Config Sink DispatchRules 1 Matcher 0] From:dispatcher.index To:} {Type:delete Path:[Config Sink DispatchRules 1 PartitionRule] From:index-value To:} {Type:delete Path:[Config Sink DispatchRules 1 IndexName] From:idx_a To:} {Type:update Path:[Config Consistent] From: To:0xc0013fc000} Update changefeed config successfully! ID: test Info: {"upstream_id":7362135842719348432,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:27:25.795680698+08:00","start_ts":449349115927068675,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","dispatchers":[{"matcher":["dispatcher.index"],"partition":"index-value"}],"encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"failed","error":{"addr":"127.0.0.1:8300","code":"CDC:ErrDispatcherFailed","message":"[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a"},"creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":0,"checkpoint_ts":449349116910108701,"checkpoint_time":"2024-04-26 19:27:27.588"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.9278.out cli changefeed pause -c simple-handle-key-only check diff failed 3-th time, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14385.out cli changefeed resume -c test Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449349118683250694, "checkpoint_time": "2024-04-26 19:27:34.352", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449349118683250694, "checkpoint_time": "2024-04-26 19:27:34.352", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449349118683250694, "checkpoint_time": "2024-04-26 19:27:34.352", "error": { "time": "2024-04-26T19:27:26.732625976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449349118683250694, '"checkpoint_time":' '"2024-04-26' '19:27:34.352",' '"error":' '{' '"time":' '"2024-04-26T19:27:26.732625976+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449349118683250694, '"checkpoint_time":' '"2024-04-26' '19:27:34.352",' '"error":' '{' '"time":' '"2024-04-26T19:27:26.732625976+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7112.out cli changefeed resume -c changefeed-initialize-error + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.9315.out cli changefeed update -c simple-handle-key-only '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-29759?protocol=simple&max-message-bytes=700' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml --no-confirm check diff failed 4-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Debugger for raftstore-v2 is used Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-29759?protocol=simple To:kafka://127.0.0.1:9092/simple-handle-key-only-29759?protocol=simple&max-message-bytes=700} {Type:update Path:[Config SyncPointInterval] From: To:0xc0001b3b28} {Type:update Path:[Config SyncPointRetention] From: To:0xc0001b3b38} {Type:update Path:[Config Consistent] From: To:0xc0012ee4d0} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Update changefeed config successfully! ID: simple-handle-key-only Info: {"upstream_id":7362135847945042033,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-29759?protocol=simple\u0026max-message-bytes=700","create_time":"2024-04-26T19:27:27.214223807+08:00","start_ts":449349116307177476,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":0,"checkpoint_ts":449349118745116677,"checkpoint_time":"2024-04-26 19:27:34.588"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: kafka... <<================= coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": null }' + echo '{ "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": null }' { "upstream_id": 7362135842719348432, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449349116910108701, "checkpoint_time": "2024-04-26 19:27:27.588", "error": null } ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449349116910108701, '"checkpoint_time":' '"2024-04-26' '19:27:27.588",' '"error":' null '}' ++ jq -r .state + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449349118683250694, "checkpoint_time": "2024-04-26 19:27:34.352", "error": null }' + echo '{ "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449349118683250694, "checkpoint_time": "2024-04-26 19:27:34.352", "error": null }' { "upstream_id": 7362135623954906196, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449349118683250694, "checkpoint_time": "2024-04-26 19:27:34.352", "error": null } ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449349118683250694, '"checkpoint_time":' '"2024-04-26' '19:27:34.352",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135623954906196, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449349118683250694, '"checkpoint_time":' '"2024-04-26' '19:27:34.352",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7198.out cli changefeed remove -c changefeed-initialize-error check diff failed 5-th time, retry later + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135842719348432, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449349116910108701, '"checkpoint_time":' '"2024-04-26' '19:27:27.588",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.9348.out cli changefeed resume -c simple-handle-key-only table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449349118683250694 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-27322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later The 1 times to try to start tidb cluster... + set +x + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:27:42 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 8-th check, retry later table test.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:27:45 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a52ad00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:23474, start at 2024-04-26 19:27:44.825646528 +0800 CST m=+5.613847171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:44.832 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:44.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:44.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully table test.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a52ad00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:23474, start at 2024-04-26 19:27:44.825646528 +0800 CST m=+5.613847171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:44.832 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:44.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:44.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a523ac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:23562, start at 2024-04-26 19:27:44.401667312 +0800 CST m=+5.133480503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:44.408 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:44.413 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:44.413 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:27:48 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> table test.finish_mark not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25024.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 3-th time, retry later table test.finish_mark not exists for 11-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449349122751463425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349122751463425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "updateproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "mysql.port"="4000" "operationcount"="0" "insertproportion"="0" "scanproportion"="0" "readproportion"="0" "threadcount"="10" "recordcount"="10000" "workload"="core" "mysql.db"="move_table" "mysql.user"="root" "dotransactions"="false" "mysql.host"="127.0.0.1" ********************************************** check diff failed 4-th time, retry later table test.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 1.227968529s INSERT - Takes(s): 1.2, Count: 10000, OPS: 8284.0, Avg(us): 1185, Min(us): 791, Max(us): 20967, 95th(us): 2000, 99th(us): 2000 [Fri Apr 26 19:27:52 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2511825120.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table test.finish_mark not exists for 13-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:27:48 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 1 [Fri Apr 26 19:27:48 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 2 [Fri Apr 26 19:27:48 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 3 table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later table test.table10 exists table test.table20 exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:55 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25176.out cli changefeed create --start-ts=449349122751463425 '--sink-uri=kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' check diff failed 6-th time, retry later Create changefeed successfully! ID: e6facbd3-e8d8-4294-85fe-bd8056a02d5a Info: {"upstream_id":7362135968232592497,"namespace":"default","id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink_uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:56.314745483+08:00","start_ts":449349122751463425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349122751463425,"checkpoint_ts":449349122751463425,"checkpoint_time":"2024-04-26 19:27:49.871"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:27:57 CST 2024] <<<<<< START kafka consumer in move_table case >>>>>> [Fri Apr 26 19:27:57 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2521525217.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 15-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a5de600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:7474, start at 2024-04-26 19:27:56.352760932 +0800 CST m=+5.248587457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:56.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:56.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:56.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a5de600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:7474, start at 2024-04-26 19:27:56.352760932 +0800 CST m=+5.248587457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:56.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:56.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:56.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a5e010000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-n5sw2-5zl0n, pid:7557, start at 2024-04-26 19:27:56.431891678 +0800 CST m=+5.274103972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:56.438 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:56.420 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:56.420 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:27:59 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh: line 1: 14471 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/new_changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh: line 1: 9387 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a607780018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:7175, start at 2024-04-26 19:27:58.980192263 +0800 CST m=+22.495451626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:58.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:58.992 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:58.992 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a607780018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:7175, start at 2024-04-26 19:27:58.980192263 +0800 CST m=+22.495451626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:58.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:58.992 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:58.992 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a515f40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2jnjw-9fjb0, pid:7261, start at 2024-04-26 19:27:43.493617403 +0800 CST m=+6.954916840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:43.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:43.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:43.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 16-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {UpstreamID:7362135968232592497 Namespace:default ID:e6facbd3-e8d8-4294-85fe-bd8056a02d5a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:27:56.314745483 +0800 CST StartTs:449349122751463425 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00272e2d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349124429185032} {CheckpointTs:449349123183739045 MinTableBarrierTs:449349125477761029 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349125477761026, checkpointTs: 449349123223060858, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2563d2d1-91d5-4213-b907-620ed6dd6720 {"id":"2563d2d1-91d5-4213-b907-620ed6dd6720","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130878} /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb75 2563d2d1-91d5-4213-b907-620ed6dd6720 /tidb/cdc/default/default/changefeed/info/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"upstream-id":7362135968232592497,"namespace":"default","changefeed-id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:27:56.314745483+08:00","start-ts":449349122751463425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349124429185032} /tidb/cdc/default/default/changefeed/status/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":449349123209953586,"min-table-barrier-ts":449349125477761029,"admin-job-type":0} /tidb/cdc/default/default/task/position/2563d2d1-91d5-4213-b907-620ed6dd6720/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/77194287-2bf6-4f20-8ab1-33168ec7546c/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {UpstreamID:7362135968232592497 Namespace:default ID:e6facbd3-e8d8-4294-85fe-bd8056a02d5a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:27:56.314745483 +0800 CST StartTs:449349122751463425 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00272e2d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349124429185032} {CheckpointTs:449349123183739045 MinTableBarrierTs:449349125477761029 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349125477761026, checkpointTs: 449349123223060858, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2563d2d1-91d5-4213-b907-620ed6dd6720 {"id":"2563d2d1-91d5-4213-b907-620ed6dd6720","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130878} /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb75 2563d2d1-91d5-4213-b907-620ed6dd6720 /tidb/cdc/default/default/changefeed/info/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"upstream-id":7362135968232592497,"namespace":"default","changefeed-id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:27:56.314745483+08:00","start-ts":449349122751463425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":+ grep -q 'failed to get info:' null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349124429185032} /tidb/cdc/default/default/changefeed/status/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":449349123209953586,"min-table-barrier-ts":449349125477761029,"admin-job-type":0} /tidb/cdc/default/default/task/position/2563d2d1-91d5-4213-b907-620ed6dd6720/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/77194287-2bf6-4f20-8ab1-33168ec7546c/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {UpstreamID:7362135968232592497 Namespace:default ID:e6facbd3-e8d8-4294-85fe-bd8056a02d5a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:27:56.314745483 +0800 CST StartTs:449349122751463425 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00272e2d0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349124429185032} {CheckpointTs:449349123183739045 MinTableBarrierTs:449349125477761029 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349125477761026, checkpointTs: 449349123223060858, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2563d2d1-91d5-4213-b907-620ed6dd6720 {"id":"2563d2d1-91d5-4213-b907-620ed6dd6720","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130878} /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb75 2563d2d1-91d5-4213-b907-620ed6dd6720 /tidb/cdc/default/default/changefeed/info/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"upstream-id":7362135968232592497,"namespace":"default","changefeed-id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:27:56.314745483+08:00","start-ts":449349122751463425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349124429185032} /tidb/cdc/default/default/changefeed/status/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":449349123209953586,"min-table-barrier-ts":449349125477761029,"admin-job-type":0} /tidb/cdc/default/default/task/position/2563d2d1-91d5-4213-b907-620ed6dd6720/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/77194287-2bf6-4f20-8ab1-33168ec7546c/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 26 19:28:00 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2532825330.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8889.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:27:53.203 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/04/26 19:27:53.204 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/04/26 19:27:53.293 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs02bfc237_6593_4f9e_b025_468940e9010a"] [2024/04/26 19:27:53.364 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2e02b954_c19e_4634_8857_e265222ef76e"] [2024/04/26 19:27:53.449 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsedacf278_35c0_48a3_a438_0dc4e7fa3019"] [2024/04/26 19:27:53.522 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs93c05aff_c878_4d6d_ad80_8b8f86a75c8f"] [2024/04/26 19:27:53.592 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:53.610 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:53.676 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:53.684 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:53.730 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs603f98e8_96a4_4081_a52b_06f9e451c9a3"] [2024/04/26 19:27:53.733 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs252a4eaf_7bc2_4a79_b476_e52ca9823c87"] [2024/04/26 19:27:53.734 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0507d335_23f4_49e8_be50_89e2e02e70b8"] [2024/04/26 19:27:53.735 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3c2e9f03_b007_413b_b3f6_26ddaa683669"] [2024/04/26 19:27:53.826 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:53.827 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:53.852 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/04/26 19:27:53.856 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/04/26 19:27:53.944 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:54.043 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/04/26 19:27:54.047 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/04/26 19:27:54.060 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.124 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:54.129 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.228 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.239 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.248 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:54.251 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:54.256 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:54.258 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:54.260 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:54.262 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:54.263 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:27:54.264 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:27:54.351 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.352 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.422 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/04/26 19:27:54.425 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/04/26 19:27:54.461 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.633 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:54.644 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.652 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:54.756 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:54.763 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:54.825 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/04/26 19:27:54.827 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.830 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/04/26 19:27:54.831 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/04/26 19:27:54.834 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.835 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/04/26 19:27:54.835 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.844 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.847 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.849 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.850 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:27:54.858 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:27:54.878 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:54.934 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:54.936 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/04/26 19:27:54.938 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/04/26 19:27:54.972 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:55.144 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:55.162 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:55.224 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:55.327 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:55.336 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:55.348 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:55.353 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:55.427 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:55.428 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:55.432 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:27:55.436 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:55.441 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:55.457 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:27:55.458 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:55.525 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/04/26 19:27:55.527 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/04/26 19:27:55.533 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:55.551 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:55.559 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/04/26 19:27:55.567 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/04/26 19:27:55.667 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:55.740 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:55.747 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:55.853 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:55.860 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:55.927 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:55.934 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:55.957 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:56.020 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:56.021 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:27:56.022 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:56.027 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:56.039 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:27:56.060 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:56.126 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/04/26 19:27:56.128 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:56.132 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/04/26 19:27:56.155 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:56.330 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:56.330 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:56.330 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:56.357 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/04/26 19:27:56.360 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/04/26 19:27:56.361 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/04/26 19:27:56.364 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/04/26 19:27:56.433 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:56.450 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:56.463 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:56.526 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:56.534 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:56.604 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:56.605 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:56.605 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:56.608 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:27:56.624 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:27:56.644 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:56.660 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/04/26 19:27:56.662 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:56.665 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/04/26 19:27:56.726 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:56.846 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:56.847 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:56.849 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:56.953 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:56.964 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:56.964 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.050 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:57.051 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:57.122 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/04/26 19:27:57.126 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/04/26 19:27:57.144 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:57.144 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:57.147 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:57.148 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:27:57.149 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:27:57.226 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.232 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/04/26 19:27:57.238 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:57.245 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/04/26 19:27:57.254 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.358 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:57.364 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:57.420 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:57.527 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:57.536 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.542 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:57.566 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:57.627 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.732 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.733 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:27:57.733 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:57.737 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:57.741 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:27:57.826 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/04/26 19:27:57.830 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:57.841 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:57.847 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/04/26 19:27:57.857 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:57.860 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/04/26 19:27:57.920 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/04/26 19:27:57.924 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/04/26 19:27:57.928 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/04/26 19:27:57.958 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:57.967 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.021 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.073 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.123 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:58.130 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.149 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:58.162 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:58.321 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:58.325 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:27:58.326 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:58.334 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:58.334 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:27:58.425 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/04/26 19:27:58.430 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.441 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.441 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/04/26 19:27:58.453 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.560 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.567 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:58.628 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:58.723 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/04/26 19:27:58.724 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/04/26 19:27:58.725 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.730 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:58.736 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:58.746 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.755 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.858 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.859 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.859 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:27:58.921 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.923 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:27:58.961 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/04/26 19:27:59.022 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:59.027 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:59.029 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/04/26 19:27:59.044 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:59.135 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:59.148 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:27:59.153 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:27:59.252 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:59.258 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:27:59.319 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:27:59.328 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:59.333 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:59.442 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:59.442 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:27:59.452 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:59.463 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/04/26 19:27:59.465 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:59.467 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/04/26 19:27:59.468 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/04/26 19:27:59.470 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:27:59.519 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/04/26 19:27:59.547 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/04/26 19:27:59.561 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:27:59.562 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/04/26 19:27:59.621 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:27:59.624 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:27:59.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:27:59.737 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:27:59.755 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:27:59.839 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:27:59.856 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:27:59.859 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:27:59.926 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:27:59.940 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:00.034 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:00.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:00.121 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:00.139 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:00.150 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:00.162 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/04/26 19:28:00.227 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:00.229 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/04/26 19:28:00.238 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:00.253 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:00.327 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:00.340 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/04/26 19:28:00.349 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/04/26 19:28:00.359 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:00.427 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:00.435 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:00.458 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:00.466 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:00.522 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:00.539 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:00.624 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:00.629 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:00.664 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:00.738 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:00.749 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:00.758 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/04/26 19:28:00.824 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:00.824 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/04/26 19:28:00.840 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:00.851 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:00.869 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:00.955 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:01.022 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.027 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:01.045 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.058 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.066 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:01.126 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:01.140 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/04/26 19:28:01.145 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/04/26 19:28:01.150 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/04/26 19:28:01.157 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/04/26 19:28:01.165 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:01.222 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:01.255 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:01.327 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:01.333 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:01.353 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/04/26 19:28:01.361 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/04/26 19:28:01.420 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.436 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.451 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.464 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:01.546 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.562 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:01.567 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.630 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:01.646 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:01.654 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.662 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.828 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.828 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:01.862 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.941 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.946 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:01.959 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/04/26 19:28:02.021 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/04/26 19:28:02.026 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/04/26 19:28:02.037 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/04/26 19:28:02.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:02.055 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:02.068 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:02.129 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.172 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.172 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:02.173 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:02.234 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:02.237 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:02.325 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/04/26 19:28:02.326 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] Verifying downstream PD is started... [2024/04/26 19:28:02.331 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.346 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.420 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:02.460 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:02.463 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.516 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.517 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:02.612 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/04/26 19:28:02.612 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/04/26 19:28:02.622 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:02.638 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:02.643 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:02.656 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:02.659 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:02.664 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:02.822 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] table test.finish_mark exists check diff successfully [2024/04/26 19:28:02.931 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:02.938 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:02.940 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:02.941 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:03.021 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:03.023 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:03.052 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:03.061 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:03.062 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] wait process cdc.test exit for 1-th time... [2024/04/26 19:28:03.163 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/04/26 19:28:03.220 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:03.222 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/04/26 19:28:03.237 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/04/26 19:28:03.242 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:03.242 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/04/26 19:28:03.252 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:03.253 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/04/26 19:28:03.257 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/04/26 19:28:03.267 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:03.269 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] + set +x + tso='449349125881724929 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349125881724929 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:28:03 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.89288930.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 26 19:28:02 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.2077820780.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:28:03.447 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:03.454 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:03.459 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:03.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:03.545 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:03.549 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:03.575 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... [2024/04/26 19:28:03.622 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:03.622 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:03.723 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/04/26 19:28:03.726 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/04/26 19:28:03.727 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:03.750 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:03.766 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:03.829 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:03.840 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:03.952 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/04/26 19:28:03.961 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:04.020 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/04/26 19:28:04.026 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:04.032 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:04.036 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:04.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:04.070 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:28:04 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/04/26 19:28:04.129 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:04.133 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:04.178 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/04/26 19:28:04.184 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/04/26 19:28:04.238 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:04.254 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:04.266 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {UpstreamID:7362135968232592497 Namespace:default ID:e6facbd3-e8d8-4294-85fe-bd8056a02d5a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:27:56.314745483 +0800 CST StartTs:449349122751463425 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d92480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349124429185032} {CheckpointTs:449349126263930886 MinTableBarrierTs:449349126263930890 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2563d2d1-91d5-4213-b907-620ed6dd6720 {"id":"2563d2d1-91d5-4213-b907-620ed6dd6720","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130878} /tidb/cdc/default/__cdc_meta__/capture/6d4ae25d-b6ad-4c96-906d-83750a3f53d8 {"id":"6d4ae25d-b6ad-4c96-906d-83750a3f53d8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130881} /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb75 2563d2d1-91d5-4213-b907-620ed6dd6720 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cba7 6d4ae25d-b6ad-4c96-906d-83750a3f53d8 /tidb/cdc/default/default/changefeed/info/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"upstream-id":7362135968232592497,"namespace":"default","changefeed-id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:27:56.314745483+08:00","start-ts":449349122751463425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349124429185032} /tidb/cdc/default/default/changefeed/status/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":449349126263930886,"min-table-barrier-ts":449349126263930890,"admin-job-type":0} /tidb/cdc/default/default/task/position/2563d2d1-91d5-4213-b907-620ed6dd6720/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d4ae25d-b6ad-4c96-906d-83750a3f53d8/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/77194287-2bf6-4f20-8ab1-33168ec7546c/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {UpstreamID:7362135968232592497 Namespace:default ID:e6facbd3-e8d8-4294-85fe-bd8056a02d5a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:27:56.314745483 +0800 CST StartTs:449349122751463425 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d92480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349124429185032} {CheckpointTs:449349126263930886 MinTableBarrierTs:449349126263930890 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2563d2d1-91d5-4213-b907-620ed6dd6720 {"id":"2563d2d1-91d5-4213-b907-620ed6dd6720","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130878} /tidb/cdc/default/__cdc_meta__/capture/6d4ae25d-b6ad-4c96-906d-83750a3f53d8 {"id":"6d4ae25d-b6ad-4c96-906d-83750a3f53d8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130881} /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb75 2563d2d1-91d5-4213-b907-620ed6dd6720 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cba7 6d4ae25d-b6ad-4c96-906d-83750a3f53d8 /tidb/cdc/default/default/changefeed/info/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"upstream-id":7362135968232592497,"namespace":"default","changefeed-id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:27:56.314745483+08:00","start-ts":449349122751463425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-thresh+ grep -q 'failed to get info:' old":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349124429185032} /tidb/cdc/default/default/changefeed/status/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":449349126263930886,"min-table-barrier-ts":449349126263930890,"admin-job-type":0} /tidb/cdc/default/default/task/position/2563d2d1-91d5-4213-b907-620ed6dd6720/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d4ae25d-b6ad-4c96-906d-83750a3f53d8/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/77194287-2bf6-4f20-8ab1-33168ec7546c/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {UpstreamID:7362135968232592497 Namespace:default ID:e6facbd3-e8d8-4294-85fe-bd8056a02d5a SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:27:56.314745483 +0800 CST StartTs:449349122751463425 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d92480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349124429185032} {CheckpointTs:449349126263930886 MinTableBarrierTs:449349126263930890 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2563d2d1-91d5-4213-b907-620ed6dd6720 {"id":"2563d2d1-91d5-4213-b907-620ed6dd6720","address":"127.0.0.1:8301","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130878} /tidb/cdc/default/__cdc_meta__/capture/6d4ae25d-b6ad-4c96-906d-83750a3f53d8 {"id":"6d4ae25d-b6ad-4c96-906d-83750a3f53d8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130881} /tidb/cdc/default/__cdc_meta__/capture/77194287-2bf6-4f20-8ab1-33168ec7546c {"id":"77194287-2bf6-4f20-8ab1-33168ec7546c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130873} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb20 77194287-2bf6-4f20-8ab1-33168ec7546c /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cb75 2563d2d1-91d5-4213-b907-620ed6dd6720 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2924cba7 6d4ae25d-b6ad-4c96-906d-83750a3f53d8 /tidb/cdc/default/default/changefeed/info/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"upstream-id":7362135968232592497,"namespace":"default","changefeed-id":"e6facbd3-e8d8-4294-85fe-bd8056a02d5a","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-18031?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:27:56.314745483+08:00","start-ts":449349122751463425,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349124429185032} /tidb/cdc/default/default/changefeed/status/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":449349126263930886,"min-table-barrier-ts":449349126263930890,"admin-job-type":0} /tidb/cdc/default/default/task/position/2563d2d1-91d5-4213-b907-620ed6dd6720/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6d4ae25d-b6ad-4c96-906d-83750a3f53d8/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/77194287-2bf6-4f20-8ab1-33168ec7546c/e6facbd3-e8d8-4294-85fe-bd8056a02d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362135968232592497 {"id":7362135968232592497,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 [2024/04/26 19:28:04.365 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:04.439 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:04.445 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:04.473 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:04.475 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:04.483 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/04/26 19:28:04.521 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/04/26 19:28:04.535 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:04.541 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:04.553 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/04/26 19:28:04.557 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/04/26 19:28:04.587 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/04/26 19:28:04.591 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 [2024/04/26 19:28:04.633 +08:00] [INFO] [main.go:178] ["73 insert success: 2100"] [2024/04/26 19:28:04.654 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:04.662 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:04.727 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:04.754 +08:00] [INFO] [main.go:178] ["73 insert success: 2100"] [2024/04/26 19:28:04.762 +08:00] [INFO] [main.go:178] ["73 insert success: 2100"] [2024/04/26 19:28:04.796 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:04.842 +08:00] [INFO] [main.go:178] ["73 insert success: 2100"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/04/26 19:28:04.906 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/04/26 19:28:04.953 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/04/26 19:28:04.975 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/04/26 19:28:05.027 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/04/26 19:28:05.058 +08:00] [INFO] [main.go:178] ["73 insert success: 2200"] [2024/04/26 19:28:05.066 +08:00] [INFO] [main.go:178] ["73 insert success: 2200"] [2024/04/26 19:28:05.096 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/04/26 19:28:05.131 +08:00] [INFO] [main.go:178] ["73 insert success: 2200"] [2024/04/26 19:28:05.200 +08:00] [INFO] [main.go:178] ["1 insert success: 2300"] [2024/04/26 19:28:05.248 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc3b031a3_dbe5_44d6_9035_613d2399dc94"] [2024/04/26 19:28:05.322 +08:00] [INFO] [main.go:178] ["72 insert success: 2100"] [2024/04/26 19:28:05.329 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/04/26 19:28:05.333 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] [2024/04/26 19:28:05.360 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse0d16cf9_c2ac_4412_828a_9cffe57769ee"] [2024/04/26 19:28:05.441 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3468f1de_2b57_4190_a28f_4af9cc00257e"] [2024/04/26 19:28:05.467 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc48ffdc0_b641_4425_bb95_6c7266de3c1e"] [2024/04/26 19:28:05.487 +08:00] [INFO] [main.go:178] ["1 insert success: 2400"] [2024/04/26 19:28:05.539 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:05.551 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs85753a34_05f8_41c7_b4fd_8d4380686d47"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/04/26 19:28:05.639 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:05.654 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:05.654 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:05.734 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] [2024/04/26 19:28:05.747 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsafba26d1_84cc_4e7e_8caa_f8c4a7271c03"] [2024/04/26 19:28:05.761 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:05.761 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:05.821 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:05.839 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1b87da1b_7316_49bb_9796_8f165a06d993"] [2024/04/26 19:28:05.842 +08:00] [INFO] [main.go:178] ["1 insert success: 2500"] [2024/04/26 19:28:05.847 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsad3ba701_b06a_4721_983f_2f034a00feca"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:05 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4f2f686-c063-4dcb-b5d7-62e2d331684b {"id":"c4f2f686-c063-4dcb-b5d7-62e2d331684b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a291448f6 c4f2f686-c063-4dcb-b5d7-62e2d331684b /tidb/cdc/default/default/upstream/7362135957720031188 {"id":7362135957720031188,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4f2f686-c063-4dcb-b5d7-62e2d331684b {"id":"c4f2f686-c063-4dcb-b5d7-62e2d331684b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a291448f6 c4f2f686-c063-4dcb-b5d7-62e2d331684b /tidb/cdc/default/default/upstream/7362135957720031188 {"id":7362135957720031188,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4f2f686-c063-4dcb-b5d7-62e2d331684b {"id":"c4f2f686-c063-4dcb-b5d7-62e2d331684b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a291448f6 c4f2f686-c063-4dcb-b5d7-62e2d331684b /tidb/cdc/default/default/upstream/7362135957720031188 {"id":7362135957720031188,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.21685.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-rocks-test-8544?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' go: downloading github.com/ardielle/ardielle-go v1.5.2 [2024/04/26 19:28:05.932 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:05.946 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:05.950 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:05.961 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:06.065 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.076 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:06.077 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.178 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:06.179 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:06.228 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:06.229 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.253 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.272 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:06.273 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:06.280 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:06.281 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:06.345 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:06.361 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.366 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:06.419 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Create changefeed successfully! ID: 7c15f8ca-2417-4e3b-b812-b236a8f64c9f Info: {"upstream_id":7362135957720031188,"namespace":"default","id":"7c15f8ca-2417-4e3b-b812-b236a8f64c9f","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-rocks-test-8544?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:28:06.388983518+08:00","start_ts":449349127043022853,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349127043022853,"checkpoint_ts":449349127043022853,"checkpoint_time":"2024-04-26 19:28:06.242"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19956d68-be05-4748-bb47-7491c422bc6d {"id":"19956d68-be05-4748-bb47-7491c422bc6d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a294b6fd4 19956d68-be05-4748-bb47-7491c422bc6d /tidb/cdc/default/default/upstream/7362136019883933923 {"id":7362136019883933923,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19956d68-be05-4748-bb47-7491c422bc6d {"id":"19956d68-be05-4748-bb47-7491c422bc6d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a294b6fd4 19956d68-be05-4748-bb47-7491c422bc6d /tidb/cdc/default/default/upstream/7362136019883933923 {"id":7362136019883933923,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19956d68-be05-4748-bb47-7491c422bc6d {"id":"19956d68-be05-4748-bb47-7491c422bc6d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a294b6fd4 19956d68-be05-4748-bb47-7491c422bc6d /tidb/cdc/default/default/upstream/7362136019883933923 {"id":7362136019883933923,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8984.out cli changefeed create --start-ts=449349125881724929 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-source-test-19751?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/04/26 19:28:06.529 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:06.535 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:06.538 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:06.647 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:06.651 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.666 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:06.671 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:06.732 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:06.743 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:06.745 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:06.751 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.753 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:06.771 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:06.820 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:06.829 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:06.838 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:06.929 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:06.933 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:06.938 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Create changefeed successfully! ID: d00048d6-75a8-4b00-87be-284b65bb0656 Info: {"upstream_id":7362136019883933923,"namespace":"default","id":"d00048d6-75a8-4b00-87be-284b65bb0656","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-source-test-19751?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:28:06.878104023+08:00","start_ts":449349125881724929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349125881724929,"checkpoint_ts":449349125881724929,"checkpoint_time":"2024-04-26 19:28:01.812"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:28:07.042 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:07.043 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:07.066 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:07.066 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:07.141 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:07.151 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:07.154 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:07.163 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:07.165 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:07.180 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:07.231 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:07.235 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:07.243 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:07.364 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:07.371 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:07.378 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:07.439 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:07.456 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:07.466 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:07.481 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:07.548 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:07.572 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:07.573 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:07.581 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:07.582 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:07.635 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:07.641 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:07.655 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:07.657 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x [Fri Apr 26 19:28:07 CST 2024] <<<<<< START kafka consumer in multi_rocks case >>>>>> [2024/04/26 19:28:07.771 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:07.820 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:07.839 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:07.861 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:07.929 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:07.933 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:07.958 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ***************** properties ***************** "insertproportion"="0" "readallfields"="true" "mysql.port"="4000" "threadcount"="2" "mysql.user"="root" "operationcount"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "dotransactions"="false" "table"="a1" "recordcount"="1000" "mysql.db"="multi_rocks" "updateproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "workload"="core" ********************************************** [2024/04/26 19:28:08.040 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:08.066 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:08.068 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:08.124 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:08.128 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:08.147 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:08.152 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.166 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:08.167 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:08.254 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.260 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.278 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.334 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:08.348 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:08.351 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.375 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.442 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.468 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:08.473 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + set +x [Fri Apr 26 19:28:08 CST 2024] <<<<<< START kafka consumer in multi_source case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 Run finished, takes 553.39656ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1849.4, Avg(us): 1077, Min(us): 814, Max(us): 12586, 95th(us): 2000, 99th(us): 2000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:08.540 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:08.549 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:08.561 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.573 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:08.624 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.625 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.664 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:08.672 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:08.719 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:08.745 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.761 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.767 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda ***************** properties ***************** "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "workload"="core" "threadcount"="2" "mysql.db"="multi_rocks" "readproportion"="0" "recordcount"="1000" "operationcount"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "table"="a2" "updateproportion"="0" "readallfields"="true" "requestdistribution"="uniform" ********************************************** [2024/04/26 19:28:08.823 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:08.858 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:08.881 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.924 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:08.945 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.951 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:08.977 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:08.982 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:09.023 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 15.819534763s"] go: downloading golang.org/x/text v0.14.0 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/04/26 19:28:09.030 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:09.030 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:09.083 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.119 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.120 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.161 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:09.166 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:09.221 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.231 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.269 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 544.311571ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1895.2, Avg(us): 1062, Min(us): 784, Max(us): 16603, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.host"="127.0.0.1" "recordcount"="1000" "operationcount"="0" "scanproportion"="0" "workload"="core" "updateproportion"="0" "insertproportion"="0" "table"="a3" "requestdistribution"="uniform" "dotransactions"="false" "mysql.user"="root" "mysql.port"="4000" "mysql.db"="multi_rocks" "readallfields"="true" "threadcount"="2" "readproportion"="0" ********************************************** [2024/04/26 19:28:09.324 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:09.334 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:09.351 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:09.354 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:09.426 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:09.427 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.437 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.443 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.486 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:09.520 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:09.521 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:09.566 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.574 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.630 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:09.657 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:09.723 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:09.742 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.751 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:09.821 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.824 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:09.856 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:09.869 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:09.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:09.930 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:09.953 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:09.960 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:09.962 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] Run finished, takes 514.426107ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 2025.7, Avg(us): 1001, Min(us): 770, Max(us): 20698, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "table"="a4" "insertproportion"="0" "workload"="core" "mysql.port"="4000" "recordcount"="1000" "operationcount"="0" "mysql.user"="root" "dotransactions"="false" "threadcount"="2" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "requestdistribution"="uniform" "updateproportion"="0" "readallfields"="true" ********************************************** [2024/04/26 19:28:10.045 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:10.052 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:10.063 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:10.130 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:10.149 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:10.169 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:10.178 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:10.251 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:10.255 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:10.275 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:10.336 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:10.356 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:10.359 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:10.371 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:10.421 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:10.428 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:10.469 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:10.476 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:10.486 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:10.538 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 551.732491ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1867.0, Avg(us): 1071, Min(us): 783, Max(us): 29952, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "recordcount"="1000" "threadcount"="2" "operationcount"="0" "readallfields"="true" "dotransactions"="false" "workload"="core" "mysql.user"="root" "updateproportion"="0" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_rocks" "table"="a5" "insertproportion"="0" ********************************************** [2024/04/26 19:28:10.553 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:10.569 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:10.579 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:10.640 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:10.648 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:10.663 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:10.686 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:10.749 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:10.751 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:10.755 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:10.776 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/coocood/freecache v1.2.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/google/btree v1.1.2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 [2024/04/26 19:28:10.835 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:10.870 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:10.880 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:10.928 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:10.953 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:10.964 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:10.977 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:11.020 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:11.050 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Run finished, takes 527.050864ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1957.2, Avg(us): 1025, Min(us): 769, Max(us): 16051, 95th(us): 2000, 99th(us): 2000 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da [2024/04/26 19:28:11.058 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:11.065 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:11.083 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:11.147 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:11.148 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:11.151 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.174 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.224 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:11.257 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:11.274 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:11.280 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 [2024/04/26 19:28:11.345 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:11.354 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.369 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:11.419 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:11.453 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:11.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.463 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:11.486 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:11.552 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:11.556 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:11.558 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 table multi_rocks.finish_mark not exists for 1-th check, retry later [2024/04/26 19:28:11.575 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:11.587 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:11.657 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.671 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:11.675 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:11.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:11.758 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:11.772 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.840 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:11.874 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:11.921 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:11.938 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:28:12.122 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:12.230 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:12.244 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:12.250 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:12.253 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:12.337 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:12.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:12.456 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:12.542 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:12.644 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:12.645 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:12.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:12.742 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:12.752 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:12.752 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:12.833 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:12.862 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:12.955 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:12.967 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:12.971 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:13.020 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:13.028 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.050 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:13.058 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:13.078 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6ccd0001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:8345, start at 2024-04-26 19:28:11.605039679 +0800 CST m=+5.151658023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:11.612 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:11.572 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:11.572 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6ccd0001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:8345, start at 2024-04-26 19:28:11.605039679 +0800 CST m=+5.151658023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:11.612 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:11.572 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:11.572 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6cf240003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:8438, start at 2024-04-26 19:28:11.72210311 +0800 CST m=+5.207637921 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:11.730 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:11.721 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:11.721 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:28:13.143 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.145 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:13.160 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:13.174 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:13.180 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:13.180 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:13.230 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:13.333 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.337 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.364 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:13.420 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:13.437 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:13.449 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:13.475 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.546 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:13.551 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:13.564 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:13.625 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:13.630 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:13.632 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.663 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:13.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:13.752 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:13.825 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6dfcc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:15712, start at 2024-04-26 19:28:12.787835306 +0800 CST m=+5.196681037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:12.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:12.787 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:12.787 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6dfcc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:15712, start at 2024-04-26 19:28:12.787835306 +0800 CST m=+5.196681037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:12.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:12.787 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:12.787 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6dfb00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:15798, start at 2024-04-26 19:28:12.828454768 +0800 CST m=+5.186661153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:12.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:12.830 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:12.830 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:28:13.855 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:13.927 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:14.020 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:14.074 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:14.097 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:14.098 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:14.101 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] table multi_rocks.finish_mark not exists for 2-th check, retry later [2024/04/26 19:28:14.140 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs157d952e_f1fa_4f2f_8cdc_104b337623ac"] [2024/04/26 19:28:14.149 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:14.187 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:14.265 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:14.449 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:14.467 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:14.470 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:14.471 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/04/26 19:28:14.525 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:14.527 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:14.534 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/04/26 19:28:14.773 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/04/26 19:28:14.778 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/04/26 19:28:14.808 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs98c47ec1_db6a_470d_bb2e_541314a98cb0"] [2024/04/26 19:28:14.831 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:14.835 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9779.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:28:14.888 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2cfd6db1_ba4d_4493_8507_052f56072069"] [2024/04/26 19:28:14.985 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc29fc35e_e807_45eb_8b44_867d1c4a2bdd"] Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [2024/04/26 19:28:15.121 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3fb44eca_d786_4b63_b377_06a0a4a05e5e"] [2024/04/26 19:28:15.243 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1ed7745a_670b_4604_a156_c4bf5f926f55"] [2024/04/26 19:28:15.252 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.258 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:15.336 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:15.336 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4820f45a_4a68_4c26_a1d5_31702d14593e"] [2024/04/26 19:28:15.347 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:15.431 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6f0eb30a_9936_4639_aeb6_915b15fe15a8"] [2024/04/26 19:28:15.448 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.450 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:15.547 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.548 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:15.631 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:15.633 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.728 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:15.728 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.728 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:15.746 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:15.770 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:15.772 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.822 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:15.828 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:15.862 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:15.872 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:15.874 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 [2024/04/26 19:28:15.881 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:15.967 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:15.970 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:16.053 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:16.055 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table multi_rocks.finish_mark not exists for 3-th check, retry later go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [2024/04/26 19:28:16.149 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:16.158 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:16.161 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:16.170 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.241 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:16.252 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:16.254 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:16.261 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:16.329 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:16.332 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:16.341 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.343 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Fri Apr 26 19:28:16 CST 2024] <<<<<< START cdc server in kafka_column_selector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.1718917191.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:28:16.422 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:16.431 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.467 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:16.470 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.554 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.556 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:16.560 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:16.571 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + set +x + tso='449349129358016513 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349129358016513 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:28:16 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.98199821.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:28:16.645 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:16.656 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:16.662 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.670 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:16.737 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:16.745 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:16.749 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:16.754 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:16.859 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:16.934 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:16.963 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:16.968 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:17.062 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:17.064 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:17.065 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.127 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:17.156 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:17.226 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:17.240 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:17.243 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:17.252 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:17.335 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:17.336 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.345 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6fab4001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:10583, start at 2024-04-26 19:28:14.559434643 +0800 CST m=+5.399331790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:14.568 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:14.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:14.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6fab4001f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:10583, start at 2024-04-26 19:28:14.559434643 +0800 CST m=+5.399331790 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:14.568 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:14.559 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:14.559 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a6fdc40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:10667, start at 2024-04-26 19:28:14.746737318 +0800 CST m=+5.528170959 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:14.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:14.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:14.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:28:17.532 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.567 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:17.643 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.645 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:17.724 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.734 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:17.738 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:17.745 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:17.764 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:17.772 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.819 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:17.821 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:17.822 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:17.833 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:17.839 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:17.850 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table multi_rocks.finish_mark not exists for 4-th check, retry later [2024/04/26 19:28:17.946 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:18.145 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:17.191 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/04/26 19:28:17.193 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/04/26 19:28:17.193 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=77194287-2bf6-4f20-8ab1-33168ec7546c] [2024/04/26 19:28:17.193 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/04/26 19:28:17.194 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/04/26 19:28:17.393 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=e6facbd3-e8d8-4294-85fe-bd8056a02d5a] [captureID=2563d2d1-91d5-4213-b907-620ed6dd6720] [processorDetail="{\"table_ids\":[106,108]}"] [2024/04/26 19:28:17.592 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=e6facbd3-e8d8-4294-85fe-bd8056a02d5a] [captureID=6d4ae25d-b6ad-4c96-906d-83750a3f53d8] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:28:17.792 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=e6facbd3-e8d8-4294-85fe-bd8056a02d5a] [captureID=77194287-2bf6-4f20-8ab1-33168ec7546c] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:28:17.792 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"2563d2d1-91d5-4213-b907-620ed6dd6720\":[{\"ID\":106,\"Changefeed\":\"e6facbd3-e8d8-4294-85fe-bd8056a02d5a\"},{\"ID\":108,\"Changefeed\":\"e6facbd3-e8d8-4294-85fe-bd8056a02d5a\"}],\"6d4ae25d-b6ad-4c96-906d-83750a3f53d8\":[],\"77194287-2bf6-4f20-8ab1-33168ec7546c\":[]}"] [2024/04/26 19:28:17.792 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=e6facbd3-e8d8-4294-85fe-bd8056a02d5a&target-cp-id=77194287-2bf6-4f20-8ab1-33168ec7546c&table-id=106"] [2024/04/26 19:28:17.842 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/04/26 19:28:17.842 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=e6facbd3-e8d8-4294-85fe-bd8056a02d5a&target-cp-id=77194287-2bf6-4f20-8ab1-33168ec7546c&table-id=108"] [2024/04/26 19:28:17.893 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/04/26 19:28:17.893 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=2563d2d1-91d5-4213-b907-620ed6dd6720] [targetCapture=77194287-2bf6-4f20-8ab1-33168ec7546c] table move_table.check1 exists check diff successfully [2024/04/26 19:28:18.335 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:18.344 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:18.447 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:18.456 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:18.526 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:18.534 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:18.543 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:18.550 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:18.561 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:18.577 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:18.578 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:18.621 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:18.632 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:18.636 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:18.650 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:18.691 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:18.758 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:18.837 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info [2024/04/26 19:28:18.943 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:19.022 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.128 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:19.148 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:19.152 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:19 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f {"id":"77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a298bcac7 77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f /tidb/cdc/default/default/upstream/7362136088833274754 {"id":7362136088833274754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f {"id":"77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a298bcac7 77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f /tidb/cdc/default/default/upstream/7362136088833274754 {"id":7362136088833274754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f {"id":"77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a298bcac7 77ec0839-eb79-4fe9-af2c-dd6ba6e9c67f /tidb/cdc/default/default/upstream/7362136088833274754 {"id":7362136088833274754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.cli.17243.out cli changefeed create --start-ts=449349129597616129 '--sink-uri=kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json&partition-num=1&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/conf/changefeed.toml [2024/04/26 19:28:19.219 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:19.232 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:19.254 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.337 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:19.341 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:19.355 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:19.420 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:19.427 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.439 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:19.474 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.525 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:19.550 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.563 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:19.640 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:19.657 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:19.661 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.665 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Create changefeed successfully! ID: test Info: {"upstream_id":7362136088833274754,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json\u0026partition-num=1\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:28:19.588272137+08:00","start_ts":449349129597616129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.t1"],"columns":["a","b"]},{"matcher":["test.*"],"columns":["*","!b"]},{"matcher":["test1.t1"],"columns":["column*","!column1"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349129597616129,"checkpoint_ts":449349129597616129,"checkpoint_time":"2024-04-26 19:28:15.987"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:28:19.727 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:19.821 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:19.921 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:19.922 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table multi_rocks.finish_mark exists check diff successfully [Fri Apr 26 19:28:19 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.1198711989.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:28:19.943 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:20.046 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:20.048 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:20.128 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:20.161 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/04/26 19:28:20.222 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:20.242 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:20.259 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:20.277 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:20.325 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:20.327 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:20.334 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:20.336 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:20.358 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:20.444 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:20.449 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] wait process cdc.test exit for 1-th time... [2024/04/26 19:28:20.532 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:20.719 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:20.721 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:20.753 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:20.834 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:20.848 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:20.920 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:20.933 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:20.937 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:20.940 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:20.947 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:20.948 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:20.953 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31 {"id":"a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29873fd4 a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31 /tidb/cdc/default/default/upstream/7362136076637902543 {"id":7362136076637902543,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31 {"id":"a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29873fd4 a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31 /tidb/cdc/default/default/upstream/7362136076637902543 {"id":7362136076637902543,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31 {"id":"a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29873fd4 a6b1b68a-2e57-4c18-84a2-0eee7a7e3b31 /tidb/cdc/default/default/upstream/7362136076637902543 {"id":7362136076637902543,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9880.out cli changefeed create --start-ts=449349129358016513 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-6830?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:28:20 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> [2024/04/26 19:28:20.970 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:20.980 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:20.985 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:21.021 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:21.133 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:21.138 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + set +x Create changefeed successfully! ID: 90bfde17-4ec5-4038-a365-2807bbe0f4bd Info: {"upstream_id":7362136076637902543,"namespace":"default","id":"90bfde17-4ec5-4038-a365-2807bbe0f4bd","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-6830?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:28:21.194605852+08:00","start_ts":449349129358016513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349129358016513,"checkpoint_ts":449349129358016513,"checkpoint_time":"2024-04-26 19:28:15.073"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:28:21.228 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:21.340 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:21.354 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Verifying downstream PD is started... [2024/04/26 19:28:21.629 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:21.634 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:21.644 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:21.645 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:21.649 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:21.657 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:21.666 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:21.670 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:21.671 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:21.673 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:21.678 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:21.747 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:21.767 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:21.771 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:21.860 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:21.860 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:22.228 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:22.243 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:22.248 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:22.249 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:22.319 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.327 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:22.338 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:22.341 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:22.341 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:22.345 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:22.345 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:22.375 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.421 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.421 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:22.452 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.455 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:22 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/abe18e26-2026-4c22-833d-55ce6af8411d {"id":"abe18e26-2026-4c22-833d-55ce6af8411d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130899} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2993e1cb abe18e26-2026-4c22-833d-55ce6af8411d /tidb/cdc/default/default/upstream/7362136099293177449 {"id":7362136099293177449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/abe18e26-2026-4c22-833d-55ce6af8411d {"id":"abe18e26-2026-4c22-833d-55ce6af8411d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130899} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2993e1cb abe18e26-2026-4c22-833d-55ce6af8411d /tidb/cdc/default/default/upstream/7362136099293177449 {"id":7362136099293177449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/abe18e26-2026-4c22-833d-55ce6af8411d {"id":"abe18e26-2026-4c22-833d-55ce6af8411d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130899} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2993e1cb abe18e26-2026-4c22-833d-55ce6af8411d /tidb/cdc/default/default/upstream/7362136099293177449 {"id":7362136099293177449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.12045.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:28:22.743 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:22.758 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.761 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:22.762 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:22.861 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:22.919 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:22.928 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:22.935 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:22.943 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.948 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:22.967 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:22.967 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:22.974 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:23.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:23.032 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] + set +x [Fri Apr 26 19:28:22 CST 2024] <<<<<< START kafka consumer in ddl_sequence case >>>>>> [2024/04/26 19:28:23.155 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:23.226 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.227 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:22.527 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/04/26 19:28:22.530 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/04/26 19:28:22.530 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=77194287-2bf6-4f20-8ab1-33168ec7546c] [2024/04/26 19:28:22.531 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/04/26 19:28:22.531 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/04/26 19:28:22.692 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=e6facbd3-e8d8-4294-85fe-bd8056a02d5a] [captureID=2563d2d1-91d5-4213-b907-620ed6dd6720] [processorDetail="{\"table_ids\":[110]}"] [2024/04/26 19:28:22.892 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=e6facbd3-e8d8-4294-85fe-bd8056a02d5a] [captureID=6d4ae25d-b6ad-4c96-906d-83750a3f53d8] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:28:23.092 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=e6facbd3-e8d8-4294-85fe-bd8056a02d5a] [captureID=77194287-2bf6-4f20-8ab1-33168ec7546c] [processorDetail="{\"table_ids\":[108]}"] [2024/04/26 19:28:23.092 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"2563d2d1-91d5-4213-b907-620ed6dd6720\":[{\"ID\":110,\"Changefeed\":\"e6facbd3-e8d8-4294-85fe-bd8056a02d5a\"}],\"6d4ae25d-b6ad-4c96-906d-83750a3f53d8\":[],\"77194287-2bf6-4f20-8ab1-33168ec7546c\":[{\"ID\":108,\"Changefeed\":\"e6facbd3-e8d8-4294-85fe-bd8056a02d5a\"}]}"] [2024/04/26 19:28:23.092 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=e6facbd3-e8d8-4294-85fe-bd8056a02d5a&target-cp-id=77194287-2bf6-4f20-8ab1-33168ec7546c&table-id=110"] [2024/04/26 19:28:23.143 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/04/26 19:28:23.143 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=2563d2d1-91d5-4213-b907-620ed6dd6720] [targetCapture=77194287-2bf6-4f20-8ab1-33168ec7546c] check diff successfully table move_table.check2 not exists for 1-th check, retry later [2024/04/26 19:28:23.439 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:23.445 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:23.453 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:23.528 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.539 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:23.546 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:23.621 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.626 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:23.636 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:23.661 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.673 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:23.774 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.783 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:23.933 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.949 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:23.958 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:23.960 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:23.968 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:24.043 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:24.046 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:24.050 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:24.119 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:24.142 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:24.246 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:24.246 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449349131412176900 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349131412176900 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.12089.out cli changefeed create --start-ts=449349131412176900 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-2602?protocol=simple&encoding-format=avro' -c simple-handle-key-only-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml [2024/04/26 19:28:24.353 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:24.366 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:24.376 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:24.383 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:24.431 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6127fd38_5cbd_4dc2_822c_767f0123798d"] [2024/04/26 19:28:24.449 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:24.449 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:24.533 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:24.551 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:24.826 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:24.833 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] table ddl_sequence.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7362136099293177449,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-2602?protocol=simple\u0026encoding-format=avro","create_time":"2024-04-26T19:28:24.844640651+08:00","start_ts":449349131412176900,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349131412176900,"checkpoint_ts":449349131412176900,"checkpoint_time":"2024-04-26 19:28:22.909"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:28:24.958 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:24.963 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:24.966 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:24.969 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:24.980 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:24.999 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:25.042 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:25.048 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:25.241 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:25.257 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] Starting build checksum checker... go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 table move_table.check2 exists [2024/04/26 19:28:25.404 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/04/26 19:28:25.425 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/04/26 19:28:25.432 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:25.439 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:25.521 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5b20459f_83f1_4c65_9670_3214d162591b"] go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading google.golang.org/protobuf v1.33.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 check diff successfully [2024/04/26 19:28:25.617 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsda4949a2_329f_4455_9f73_43c5de50794d"] [2024/04/26 19:28:25.625 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs338f6810_f95d_4930_9125_0172a7d6c8af"] [2024/04/26 19:28:25.697 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5448f079_ac0a_44c5_9cea_528116c2ee6e"] [2024/04/26 19:28:25.719 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs262afb93_d377_4384_89a3_268ee4af4d6c"] [2024/04/26 19:28:25.732 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:25.738 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:25.816 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 [2024/04/26 19:28:25.870 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc74e5d11_d37e_4763_a3f2_2eefc011ac5b"] [2024/04/26 19:28:25.938 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:26.033 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:26.038 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:26.043 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8ac9b5e0_b026_4772_a9d3_dd4d2d357891"] [2024/04/26 19:28:26.045 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:26.231 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:26.243 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:26.248 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:26.258 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:26.322 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:26.323 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:26.335 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:26.363 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:28:26.439 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:26.446 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:26.466 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:26.545 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:26.553 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] wait process cdc.test exit for 2-th time... [2024/04/26 19:28:26.628 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:26.629 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:26.635 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:26.833 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table ddl_sequence.finish_mark not exists for 2-th check, retry later [Fri Apr 26 19:28:25 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 10 [Fri Apr 26 19:28:25 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 20 [Fri Apr 26 19:28:25 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... [2024/04/26 19:28:26.920 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:26.928 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:26.932 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:26.956 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:26.963 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:27.030 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:27.055 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:27.132 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:27.143 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.144 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:27.219 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:27.237 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:27.242 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:27.252 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.256 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:27.336 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table test.finish not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:28:27 CST 2024] <<<<<< run test case move_table success! >>>>>> [2024/04/26 19:28:27.445 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:27.445 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.527 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.553 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.554 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] go: downloading github.com/ardielle/ardielle-go v1.5.2 [2024/04/26 19:28:27.648 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:27.725 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:27.752 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:27.770 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:27.771 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.855 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:27.874 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:27.875 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:27.931 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:27.933 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:27.951 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:28.021 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:28.033 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:28.048 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:28.063 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:28.066 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:28.140 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/tklauser/numcpus v0.6.1 [2024/04/26 19:28:28.219 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:28.243 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:28.354 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:28.356 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 [2024/04/26 19:28:28.441 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:28.462 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:28.463 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:28.528 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:28.528 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:28.546 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:28.580 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:28.623 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:28.636 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:28.644 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_sequence.finish_mark not exists for 3-th check, retry later [2024/04/26 19:28:28.655 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:28.669 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:28.723 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:28.757 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:28.942 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:28.945 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.136 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:29.152 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:29.154 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:29.242 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.245 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:29.271 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:29.337 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.343 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:29.352 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.352 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.367 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:29.372 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:29.380 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:29.435 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:29.526 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.526 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:29.639 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 [2024/04/26 19:28:29.739 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:29.748 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:29.844 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:29.845 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:29.920 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table test.finish exists check diff successfully [2024/04/26 19:28:29.953 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:29.955 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:29.968 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:29.970 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:29.974 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:29.978 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:29.981 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:30.030 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:30.059 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:30.061 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:30.127 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:30.156 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a7e0680004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:19466, start at 2024-04-26 19:28:29.214673972 +0800 CST m=+5.085472424 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:29.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:29.210 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:29.210 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a7e0680004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:19466, start at 2024-04-26 19:28:29.214673972 +0800 CST m=+5.085472424 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:29.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:29.210 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:29.210 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a7e2980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:19542, start at 2024-04-26 19:28:29.389378473 +0800 CST m=+5.210380729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:29.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:29.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:29.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/04/26 19:28:30.228 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:30.346 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:30.348 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_sequence.finish_mark not exists for 4-th check, retry later [2024/04/26 19:28:30.462 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:30.561 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:30.562 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:30.635 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:30.635 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:30.635 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:30.638 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:30.660 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:30.663 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a7e1040016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:11764, start at 2024-04-26 19:28:29.290195208 +0800 CST m=+7.205053823 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:29.300 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:29.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:29.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a7e1040016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:11764, start at 2024-04-26 19:28:29.290195208 +0800 CST m=+7.205053823 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:29.300 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:29.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:29.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a7e1040014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:11800, start at 2024-04-26 19:28:29.280478654 +0800 CST m=+7.077860238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:29.290 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:29.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:29.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... [2024/04/26 19:28:30.753 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:30.753 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:30.756 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:30.771 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:30.830 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:30.945 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:30.951 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:31.044 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:31.136 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:31.142 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:31.165 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:31.167 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:31.169 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] wait process cdc.test exit for 3-th time... [2024/04/26 19:28:31.219 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:31.235 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:31.246 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:31.281 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:31.328 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:31.331 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:31.331 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:31.344 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:31.465 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:31.472 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:31.552 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:31.655 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:31.661 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:28:31 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> [2024/04/26 19:28:31.735 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:31.738 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:31.747 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:31.756 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:31.822 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:31.828 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:31.843 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:31.854 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:31.855 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:31.860 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:31.870 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:32.055 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:32.057 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:32.149 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:32.336 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:32.343 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:32.425 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:32.430 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:32.436 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:32.447 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.13177.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:28:32.464 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:32.472 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:32.527 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:32.535 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:32.538 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:28:32.541 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:32.542 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:28:32.628 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:32.632 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.12147.out cli changefeed pause -c simple-handle-key-only-avro [2024/04/26 19:28:32.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:32.845 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.21009.out cli tso query --pd=http://127.0.0.1:2379 table ddl_sequence.finish_mark not exists for 5-th check, retry later [2024/04/26 19:28:33.042 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:33.113 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:33.117 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.129 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:33.142 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:33.150 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:33.184 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [2024/04/26 19:28:33.230 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.233 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.237 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:33.238 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:33.239 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:28:33.266 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:33.271 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 19:28:33.319 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.342 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2024/04/26 19:28:33.737 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.741 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:33.749 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:33.835 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:33.852 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.852 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:33.950 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:33.953 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:33.957 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:33.957 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:33.957 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 19:28:33.967 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:33.968 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 19:28:33.971 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:33.986 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2024/04/26 19:28:34.097 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:34.164 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:34.230 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:34.231 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [Pipeline] // stage + set +x + tso='449349134018150401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349134018150401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:28:34 CST 2024] <<<<<< START cdc server in cli case >>>>>> [Pipeline] } [2024/04/26 19:28:34.368 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:34.368 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:34.369 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:34.374 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 19:28:34.380 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:34.384 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 19:28:34.419 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:34.435 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:34.495 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:34.556 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:34.567 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:34.633 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.1323513237.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.12184.out cli changefeed update -c simple-handle-key-only-avro '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-2602?protocol=simple&encoding-format=avro&max-message-bytes=650' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml --no-confirm [2024/04/26 19:28:34.852 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:34.852 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:34.860 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 19:28:34.865 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:34.875 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:34.922 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:34.929 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 19:28:34.941 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:34.968 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:34.999 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:35.007 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] + set +x + tso='449349134086307842 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349134086307842 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:28:34 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2104821050.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_sequence.finish_mark exists check diff successfully [2024/04/26 19:28:35.019 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:35.238 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 19:28:35.255 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:28:35 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> [2024/04/26 19:28:35.382 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 19:28:35.435 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:35.526 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 19:28:35.564 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb37e308a_d4f9_4cd9_a195_de8339d5fca8"] [2024/04/26 19:28:35.620 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:35.661 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf4530b5f_3c10_4886_91ae_103dac2fbc0d"] [2024/04/26 19:28:35.719 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 19:28:35.896 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:35.902 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:35.959 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:36.024 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:36.063 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd955bdb6_734d_4739_8d22_1d8b66c101b5"] [2024/04/26 19:28:36.196 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:36.207 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:36.251 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:36.261 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2d611caf_a6fd_4a4e_ac26_5bbff992d608"] [2024/04/26 19:28:36.319 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:36.320 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs79aeb48d_f67a_4c39_adf3_913cc9a9d196"] [2024/04/26 19:28:36.357 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:36.363 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:36.573 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:36.588 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:36.652 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:36.660 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:36.661 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:36.691 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:28:36.693 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:36.695 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:28:36.719 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:36.728 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-avro-2602?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/simple-handle-key-only-avro-2602?protocol=simple&encoding-format=avro&max-message-bytes=650} {Type:update Path:[Config SyncPointInterval] From: To:0xc003958db8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003958dc8} {Type:update Path:[Config Consistent] From: To:0xc00133ee70} [2024/04/26 19:28:36.854 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:36.877 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:36.973 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:36.985 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:36.988 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Update changefeed config successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7362136099293177449,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-2602?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=650","create_time":"2024-04-26T19:28:24.844640651+08:00","start_ts":449349131412176900,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":0,"checkpoint_ts":449349133915652101,"checkpoint_time":"2024-04-26 19:28:32.459"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:28:37.046 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:28:37.055 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:37.058 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:28:37.075 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:37.089 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:37.182 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:37.221 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:37.282 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:37.292 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:37.296 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:37.350 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:28:37.363 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:28:37.367 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:37.381 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:37.422 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:37.549 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:37.621 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:37.680 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:37.688 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:37.693 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:37.734 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:28:37.754 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:28:37.758 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:37.759 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:37.770 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:37.851 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:37.942 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:38.045 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:37 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f41d045d-9d92-4ad9-b065-866103523460 {"id":"f41d045d-9d92-4ad9-b065-866103523460","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29c7fa40 f41d045d-9d92-4ad9-b065-866103523460 /tidb/cdc/default/default/upstream/7362136150943869283 {"id":7362136150943869283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f41d045d-9d92-4ad9-b065-866103523460 {"id":"f41d045d-9d92-4ad9-b065-866103523460","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29c7fa40 f41d045d-9d92-4ad9-b065-866103523460 /tidb/cdc/default/default/upstream/7362136150943869283 {"id":7362136150943869283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f41d045d-9d92-4ad9-b065-866103523460 {"id":"f41d045d-9d92-4ad9-b065-866103523460","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29c7fa40 f41d045d-9d92-4ad9-b065-866103523460 /tidb/cdc/default/default/upstream/7362136150943869283 {"id":7362136150943869283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.13293.out cli changefeed create --start-ts=449349134018150401 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-31013?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:28:37 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/941c2be8-a550-4cff-b416-3b73a2750758 {"id":"941c2be8-a550-4cff-b416-3b73a2750758","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29d03deb 941c2be8-a550-4cff-b416-3b73a2750758 /tidb/cdc/default/default/upstream/7362136159568258656 {"id":7362136159568258656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/941c2be8-a550-4cff-b416-3b73a2750758 {"id":"941c2be8-a550-4cff-b416-3b73a2750758","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29d03deb 941c2be8-a550-4cff-b416-3b73a2750758 /tidb/cdc/default/default/upstream/7362136159568258656 {"id":7362136159568258656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/941c2be8-a550-4cff-b416-3b73a2750758 {"id":"941c2be8-a550-4cff-b416-3b73a2750758","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a29d03deb 941c2be8-a550-4cff-b416-3b73a2750758 /tidb/cdc/default/default/upstream/7362136159568258656 {"id":7362136159568258656,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/04/26 19:28:38.059 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:38.072 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:38.132 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:28:38.160 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:38.160 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:28:38.163 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:38.175 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:38.215 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:38.261 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7362136150943869283,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-31013?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:28:38.139868282+08:00","start_ts":449349134018150401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349134018150401,"checkpoint_ts":449349134018150401,"checkpoint_time":"2024-04-26 19:28:32.850"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.12218.out cli changefeed resume -c simple-handle-key-only-avro [2024/04/26 19:28:38.334 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:38.350 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:38.375 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:38.453 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:28:38.482 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:38.491 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:38.491 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:28:38.519 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:38.556 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:38.600 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:38.645 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:38.660 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:38.684 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:38.747 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:28:38.774 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:38.785 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:38.786 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:28:38.816 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:38.845 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:38.928 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:38.942 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:38.981 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:39.029 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:28:39.057 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Fri Apr 26 19:28:38 CST 2024] <<<<<< START kafka consumer in capture_session_done_during_task case >>>>>> lease 22318f1a29d03deb revoked [2024/04/26 19:28:39.078 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:39.081 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:28:39.108 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:39.221 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:28:39.265 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:39.299 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:28:39.327 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:39.363 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:28:39.394 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:39.499 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:28:39.548 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:39.567 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + set +x [Fri Apr 26 19:28:39 CST 2024] <<<<<< START kafka consumer in cli case >>>>>> table test.simple not exists for 1-th check, retry later [2024/04/26 19:28:39.644 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:28:39.827 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v46.623874417s"] [2024/04/26 19:28:39.905 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 46.701061859s"] table capture_session_done_during_task.t exists check diff failed 1-th time, retry later table mark.finish_mark_1 not exists for 1-th check, retry later + set +x table test.finish_mark not exists for 1-th check, retry later table test.simple not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later table mark.finish_mark_1 not exists for 2-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349134437318688, "checkpoint_time": "2024-04-26 19:28:34.449", "error": null }' + echo '{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349134437318688, "checkpoint_time": "2024-04-26 19:28:34.449", "error": null }' { "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349134437318688, "checkpoint_time": "2024-04-26 19:28:34.449", "error": null } ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449349134437318688, '"checkpoint_time":' '"2024-04-26' '19:28:34.449",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449349134437318688, '"checkpoint_time":' '"2024-04-26' '19:28:34.449",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 check diff failed 3-th time, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 table test.finish_mark exists check diff successfully start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc004193810} {Type:update Path:[Config SyncPointRetention] From: To:0xc004193818} {Type:update Path:[Config Consistent] From: To:0xc00125e1c0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.13750.out cli changefeed --changefeed-id custom-changefeed-name pause wait process cdc.test exit for 1-th time... PASS table mark.finish_mark_1 not exists for 3-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2715 0 --:--:-- --:--:-- --:--:-- 2728 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:28:32.805","puller_resolved_ts":"2024-04-26 19:28:25.806","last_synced_ts":"2024-04-26 19:26:16.906","now_ts":"2024-04-26 19:28:34.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:28:32.805","puller_resolved_ts":"2024-04-26' '19:28:25.806","last_synced_ts":"2024-04-26' '19:26:16.906","now_ts":"2024-04-26' '19:28:34.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9909 7.6 0.0 13849360 141156 ? Sl 19:25 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9976 5.2 0.0 13456848 136472 ? Sl 19:25 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714130919.775137,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002d2b340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130919.775197,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130919.8375962,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021e01c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130919.83767,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130919.914387,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003c5afc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714130919.914486,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:28:44.691685+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:28:44.695361+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:28:44.766956+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... check diff successfully check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=kafka + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run kafka + '[' kafka '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri Apr 26 19:28:46 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri Apr 26 19:28:46 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:28:46 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later + set +x check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table mark.finish_mark_2 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:28:49 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:28:50 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449349137320902660, "checkpoint_time": "2024-04-26 19:28:45.449", "error": null }' + echo '{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449349137320902660, "checkpoint_time": "2024-04-26 19:28:45.449", "error": null }' { "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449349137320902660, "checkpoint_time": "2024-04-26 19:28:45.449", "error": null } ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449349137320902660, '"checkpoint_time":' '"2024-04-26' '19:28:45.449",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449349137320902660, '"checkpoint_time":' '"2024-04-26' '19:28:45.449",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.13843.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name {"level":"warn","ts":"2024-04-26T19:28:50.69268+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:28:50.696565+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:28:50.767953+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc001520c88} {Type:update Path:[Config SyncPointRetention] From: To:0xc001520c98} {Type:update Path:[Config Consistent] From: To:0xc001468310} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7362136150943869283,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-31013?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:28:38.139868282+08:00","start_ts":449349134018150401,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":0,"checkpoint_ts":449349137320902660,"checkpoint_time":"2024-04-26 19:28:45.449"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_2 not exists for 3-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.13880.out cli changefeed --changefeed-id custom-changefeed-name resume ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:28:52 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> table mark.finish_mark_2 not exists for 4-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table mark.finish_mark_2 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:28:56 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:28:56.693853+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:28:56.697422+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:28:56.769758+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a97ef00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:30450, start at 2024-04-26 19:28:55.744091424 +0800 CST m=+5.104672196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:55.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:55.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:55.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a97ef00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:30450, start at 2024-04-26 19:28:55.744091424 +0800 CST m=+5.104672196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:55.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:55.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:55.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a981240009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:30527, start at 2024-04-26 19:28:55.892003281 +0800 CST m=+5.199728365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:55.899 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:55.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:55.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_2 not exists for 6-th check, retry later Verifying downstream PD is started... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349140466630661, "checkpoint_time": "2024-04-26 19:28:57.449", "error": null }' + echo '{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349140466630661, "checkpoint_time": "2024-04-26 19:28:57.449", "error": null }' { "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349140466630661, "checkpoint_time": "2024-04-26 19:28:57.449", "error": null } ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449349140466630661, '"checkpoint_time":' '"2024-04-26' '19:28:57.449",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449349140466630661, '"checkpoint_time":' '"2024-04-26' '19:28:57.449",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.13975.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449349140729036805 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-31013?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Fri Apr 26 19:28:58 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3193031932.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 not exists for 7-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh: line 1: 12251 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_2 not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:02 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ccfef900-0580-4171-a8d4-3e16339a614c {"id":"ccfef900-0580-4171-a8d4-3e16339a614c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130939} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a35ecc5 ccfef900-0580-4171-a8d4-3e16339a614c /tidb/cdc/default/default/upstream/7362136276123539870 {"id":7362136276123539870,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' Starting Upstream TiDB... + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ccfef900-0580-4171-a8d4-3e16339a614c {"id":"ccfef900-0580-4171-a8d4-3e16339a614c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130939} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a35ecc5 ccfef900-0580-4171-a8d4-3e16339a614c /tidb/cdc/default/default/upstream/7362136276123539870 {"id":7362136276123539870,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ccfef900-0580-4171-a8d4-3e16339a614c {"id":"ccfef900-0580-4171-a8d4-3e16339a614c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130939} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a35ecc5 ccfef900-0580-4171-a8d4-3e16339a614c /tidb/cdc/default/default/upstream/7362136276123539870 {"id":7362136276123539870,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 63c837e7-c904-45fb-ab99-834f65f2e6ea Info: {"upstream_id":7362136276123539870,"namespace":"default","id":"63c837e7-c904-45fb-ab99-834f65f2e6ea","sink_uri":"kafka://127.0.0.1:9092/ticdc-autorandom-test-111?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:29:02.261130606+08:00","start_ts":449349141683240965,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349141683240965,"checkpoint_ts":449349141683240965,"checkpoint_time":"2024-04-26 19:29:02.090"} [Fri Apr 26 19:29:02 CST 2024] <<<<<< START kafka consumer in autorandom case >>>>>> Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check Starting Upstream PD... 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:29:02.69494+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:02.699008+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:02.771027+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table autorandom_test.table_a not exists for 1-th check, retry later table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.14062.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-31013?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7362136150943869283,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-31013?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:29:04.404664939+08:00","start_ts":449349142262317057,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349142262317057,"checkpoint_ts":449349142262317057,"checkpoint_time":"2024-04-26 19:29:04.299"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:29:04.681365+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:29:04.681412+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:29:04.686024+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:29:04.686075+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:29:04.757349+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:29:04.757405+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [Pipeline] // withCredentials start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table mark.finish_mark_3 not exists for 2-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } + set +x [Pipeline] // stage [Pipeline] } table autorandom_test.table_a exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 3-th check, retry later check_changefeed_state http://127.0.0.1:2379 b4371248-ce9b-4df9-8375-6b35a6c0ac1c finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=b4371248-ce9b-4df9-8375-6b35a6c0ac1c + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b4371248-ce9b-4df9-8375-6b35a6c0ac1c -s + info='{ "upstream_id": 7362135867920361408, "namespace": "default", "id": "b4371248-ce9b-4df9-8375-6b35a6c0ac1c", "state": "finished", "checkpoint_tso": 449349140287062019, "checkpoint_time": "2024-04-26 19:28:56.764", "error": null }' + echo '{ "upstream_id": 7362135867920361408, "namespace": "default", "id": "b4371248-ce9b-4df9-8375-6b35a6c0ac1c", "state": "finished", "checkpoint_tso": 449349140287062019, "checkpoint_time": "2024-04-26 19:28:56.764", "error": null }' { "upstream_id": 7362135867920361408, "namespace": "default", "id": "b4371248-ce9b-4df9-8375-6b35a6c0ac1c", "state": "finished", "checkpoint_tso": 449349140287062019, "checkpoint_time": "2024-04-26 19:28:56.764", "error": null } ++ echo '{' '"upstream_id":' 7362135867920361408, '"namespace":' '"default",' '"id":' '"b4371248-ce9b-4df9-8375-6b35a6c0ac1c",' '"state":' '"finished",' '"checkpoint_tso":' 449349140287062019, '"checkpoint_time":' '"2024-04-26' '19:28:56.764",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7362135867920361408, '"namespace":' '"default",' '"id":' '"b4371248-ce9b-4df9-8375-6b35a6c0ac1c",' '"state":' '"finished",' '"checkpoint_tso":' 449349140287062019, '"checkpoint_time":' '"2024-04-26' '19:28:56.764",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:29:04 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aa32100013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:11043, start at 2024-04-26 19:29:07.236284343 +0800 CST m=+5.134930584 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:07.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:07.204 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:07.204 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aa32100013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:11043, start at 2024-04-26 19:29:07.236284343 +0800 CST m=+5.134930584 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:07.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:07.204 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:07.204 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aa34440008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-4h6bf-dcq76, pid:11133, start at 2024-04-26 19:29:07.352509531 +0800 CST m=+5.196779568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:07.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:07.345 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:07.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:29:08.6954+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:08.700705+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:08.772202+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff successfully wait process cdc.test exit for 1-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349143350476806, "checkpoint_time": "2024-04-26 19:29:08.450", "error": null }' + echo '{ "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349143350476806, "checkpoint_time": "2024-04-26 19:29:08.450", "error": null }' { "upstream_id": 7362136150943869283, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449349143350476806, "checkpoint_time": "2024-04-26 19:29:08.450", "error": null } ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449349143350476806, '"checkpoint_time":' '"2024-04-26' '19:29:08.450",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362136150943869283, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449349143350476806, '"checkpoint_time":' '"2024-04-26' '19:29:08.450",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... table mark.finish_mark_3 not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:29:09 CST 2024] <<<<<< run test case autorandom success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.12588.out cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.14189.out cli changefeed create --start-ts=449349134018150401 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-31803?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai [WARN] --tz is deprecated in changefeed settings. ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: ba982d58-6857-4fb6-abd5-643ce3b831d8 Info: {"upstream_id":7362136150943869283,"namespace":"default","id":"ba982d58-6857-4fb6-abd5-643ce3b831d8","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-31803?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-04-26T19:29:10.865676356+08:00","start_ts":449349134018150401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349134018150401,"checkpoint_ts":449349134018150401,"checkpoint_time":"2024-04-26 19:28:32.850"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 5-th check, retry later + set +x + tso='449349143941349377 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349143941349377 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:29:12 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1262812630.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.14223.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.14273.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:29:14.696805+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:14.702572+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:14.774139+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714130954.776121,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002d2b340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130954.7761626,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130954.8392591,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021e01c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130954.839324,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130954.9160287,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003c5afc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714130954.9160895,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe90690-36d9-4b60-b8ef-2917dbb76063 {"id":"ffe90690-36d9-4b60-b8ef-2917dbb76063","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130952} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a5e34d3 ffe90690-36d9-4b60-b8ef-2917dbb76063 /tidb/cdc/default/default/upstream/7362136311845199440 {"id":7362136311845199440,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe90690-36d9-4b60-b8ef-2917dbb76063 {"id":"ffe90690-36d9-4b60-b8ef-2917dbb76063","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130952} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a5e34d3 ffe90690-36d9-4b60-b8ef-2917dbb76063 /tidb/cdc/default/default/upstream/7362136311845199440 {"id":7362136311845199440,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe90690-36d9-4b60-b8ef-2917dbb76063 {"id":"ffe90690-36d9-4b60-b8ef-2917dbb76063","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130952} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a5e34d3 ffe90690-36d9-4b60-b8ef-2917dbb76063 /tidb/cdc/default/default/upstream/7362136311845199440 {"id":7362136311845199440,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.12690.out cli changefeed create --start-ts=449349143941349377 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-17039?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table mark.finish_mark_3 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aa922c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:13478, start at 2024-04-26 19:29:13.383378166 +0800 CST m=+5.190375071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:13.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:13.355 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:13.355 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aa922c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:13478, start at 2024-04-26 19:29:13.383378166 +0800 CST m=+5.190375071 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:13.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:13.355 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:13.355 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aa93a40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:13552, start at 2024-04-26 19:29:13.454241795 +0800 CST m=+5.199891540 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:13.459 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:13.449 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:13.449 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: 0cbad273-edc6-47a7-9caa-896b261ff707 Info: {"upstream_id":7362136311845199440,"namespace":"default","id":"0cbad273-edc6-47a7-9caa-896b261ff707","sink_uri":"kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-17039?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:29:16.581284137+08:00","start_ts":449349143941349377,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349143941349377,"checkpoint_ts":449349143941349377,"checkpoint_time":"2024-04-26 19:29:10.704"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aac0300009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:22513, start at 2024-04-26 19:29:16.309589547 +0800 CST m=+5.168168890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:16.317 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:16.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:16.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aac0300009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:22513, start at 2024-04-26 19:29:16.309589547 +0800 CST m=+5.168168890 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:16.317 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:16.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:16.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aac0d40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-jb3h1-89ss8, pid:22587, start at 2024-04-26 19:29:16.384747701 +0800 CST m=+5.197647088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:16.391 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:16.391 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:16.391 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [Fri Apr 26 19:29:18 CST 2024] <<<<<< START kafka consumer in resourcecontrol case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... table resourcecontrol.finish_mark not exists for 1-th check, retry later [Fri Apr 26 19:29:18 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.1490314905.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.14407.out cli unsafe resolve-lock --region=144 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.23902.out cli tso query --pd=http://127.0.0.1:2379 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 9-th check, retry later Verifying downstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:29:20.698531+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f60380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:20.704247+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aea000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:29:20.775999+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00125a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table resourcecontrol.finish_mark exists check diff successfully + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.14441.out cli unsafe resolve-lock --region=144 --ts=449349145355616259 wait process cdc.test exit for 1-th time... + set +x + tso='449349146312966145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349146312966145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:29:21 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2393723939.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... table mark.finish_mark_3 not exists for 10-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:21 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f17e-c16f-4c94-bf73-5588441448f4 {"id":"4838f17e-c16f-4c94-bf73-5588441448f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a7b4fd1 4838f17e-c16f-4c94-bf73-5588441448f4 /tidb/cdc/default/default/upstream/7362136342326570475 {"id":7362136342326570475,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f17e-c16f-4c94-bf73-5588441448f4 {"id":"4838f17e-c16f-4c94-bf73-5588441448f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a7b4fd1 4838f17e-c16f-4c94-bf73-5588441448f4 /tidb/cdc/default/default/upstream/7362136342326570475 {"id":7362136342326570475,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f17e-c16f-4c94-bf73-5588441448f4 {"id":"4838f17e-c16f-4c94-bf73-5588441448f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a7b4fd1 4838f17e-c16f-4c94-bf73-5588441448f4 /tidb/cdc/default/default/upstream/7362136342326570475 {"id":7362136342326570475,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.14961.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:29:22 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 161k 4687 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" table mark.finish_mark_3 not exists for 11-th check, retry later + set +x + tso='449349146890731523 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349146890731523 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.15003.out cli changefeed create --start-ts=449349146890731523 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-18313?protocol=simple' -c kafka-simple-claim-check --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml Create changefeed successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7362136342326570475,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-18313?protocol=simple","create_time":"2024-04-26T19:29:23.860179458+08:00","start_ts":449349146890731523,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349146890731523,"checkpoint_ts":449349146890731523,"checkpoint_time":"2024-04-26 19:29:21.955"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:24 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4eb23f2-921d-4004-a930-e6b7c203752b {"id":"a4eb23f2-921d-4004-a930-e6b7c203752b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130961} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a8417ce a4eb23f2-921d-4004-a930-e6b7c203752b /tidb/cdc/default/default/upstream/7362136366655502331 {"id":7362136366655502331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4eb23f2-921d-4004-a930-e6b7c203752b {"id":"a4eb23f2-921d-4004-a930-e6b7c203752b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130961} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a8417ce a4eb23f2-921d-4004-a930-e6b7c203752b /tidb/cdc/default/default/upstream/7362136366655502331 {"id":7362136366655502331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4eb23f2-921d-4004-a930-e6b7c203752b {"id":"a4eb23f2-921d-4004-a930-e6b7c203752b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130961} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a8417ce a4eb23f2-921d-4004-a930-e6b7c203752b /tidb/cdc/default/default/upstream/7362136366655502331 {"id":7362136366655502331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.23991.out cli changefeed create --start-ts=449349146312966145 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-3016?protocol=open-protocol&partition-num=4&kafka-version=2.4.1' wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: 625fb706-ea78-47a3-86d2-2cd9226d342c Info: {"upstream_id":7362136366655502331,"namespace":"default","id":"625fb706-ea78-47a3-86d2-2cd9226d342c","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-3016?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:29:24.755106+08:00","start_ts":449349146312966145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349146312966145,"checkpoint_ts":449349146312966145,"checkpoint_time":"2024-04-26 19:29:19.751"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... + set +x table mark.finish_mark_3 not exists for 12-th check, retry later wait process cdc.test exit for 3-th time... } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 192k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-37-gcf8946a3a", "git_hash": "cf8946a3a9cb6bf28577ea67d4848a4fad982d67", "id": "d512f58a-9cbc-4052-a1bc-e93d028e5998", "pid": 13240, "is_owner": true + set +x [Fri Apr 26 19:29:26 CST 2024] <<<<<< START kafka consumer in ddl_attributes case >>>>>> }wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:29:26 CST 2024] <<<<<< run test case cli success! >>>>>> table mark.finish_mark_3 not exists for 13-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 14-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ab8da40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh, pid:4447, start at 2024-04-26 19:29:29.456811589 +0800 CST m=+5.176290627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:29.463 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:29.449 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:29.449 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table mark.finish_mark_3 not exists for 15-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.15051.out cli changefeed pause -c kafka-simple-claim-check PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ab8da40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh, pid:4447, start at 2024-04-26 19:29:29.456811589 +0800 CST m=+5.176290627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:29.463 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:29.449 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:29.449 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ab8d940013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-c3p4d-nlwlh, pid:4531, start at 2024-04-26 19:29:29.466818213 +0800 CST m=+5.130154881 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:29.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:29.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:29.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_3 not exists for 16-th check, retry later [2024/04/26 19:29:30.491 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 13.750689351s"] + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Fri Apr 26 19:29:34 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.59125914.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 17-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.15086.out cli changefeed update -c kafka-simple-claim-check '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-18313?protocol=simple&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml --no-confirm Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-18313?protocol=simple To:kafka://127.0.0.1:9092/kafka-simple-claim-check-18313?protocol=simple&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0015a9e68} {Type:update Path:[Config SyncPointRetention] From: To:0xc0015a9e78} {Type:update Path:[Config Consistent] From: To:0xc000eff260} Update changefeed config successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7362136342326570475,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-18313?protocol=simple\u0026max-message-bytes=2048","create_time":"2024-04-26T19:29:23.860179458+08:00","start_ts":449349146890731523,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":0,"checkpoint_ts":449349149328670725,"checkpoint_time":"2024-04-26 19:29:31.255"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } table test1.finishmark exists [2024/04/26 19:29:29.809 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/04/26 19:29:29.814 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/04/26 19:29:29.818 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/04/26 19:29:29.823 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/04/26 19:29:29.825 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/04/26 19:29:29.825 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=22.541385ms] [2024/04/26 19:29:29.828 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/04/26 19:29:29.830 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/04/26 19:29:29.831 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/04/26 19:29:29.836 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/04/26 19:29:29.838 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/04/26 19:29:29.838 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=13.015572ms] [2024/04/26 19:29:29.838 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:29:31 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> [Pipeline] // stage [Pipeline] } [2024/04/26 19:29:29.279 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/04/26 19:29:29.483 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:29:29.486 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:29:29.664 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:29:29.665 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:29:29.665 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:29:29.667 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:29:29.840 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:29:29.841 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:29:31.041 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:29:31.043 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:29:31.043 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:29:31.044 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:29:31.219 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:29:31.249 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:29:31.404 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:29:31.407 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:29:31.447 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:29:31.449 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:29:32.609 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:29:32.672 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:29:32.780 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:29:32.782 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:29:32.866 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:29:32.868 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:29:32.955 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:29:33.067 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:29:33.129 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:29:33.132 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:29:34.277 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:29:34.279 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:37 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/591bd659-468b-4c68-ab26-e2d933c38f0a {"id":"591bd659-468b-4c68-ab26-e2d933c38f0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ab935d1 591bd659-468b-4c68-ab26-e2d933c38f0a /tidb/cdc/default/default/upstream/7362136410216467600 {"id":7362136410216467600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/591bd659-468b-4c68-ab26-e2d933c38f0a {"id":"591bd659-468b-4c68-ab26-e2d933c38f0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ab935d1 591bd659-468b-4c68-ab26-e2d933c38f0a /tidb/cdc/default/default/upstream/7362136410216467600 {"id":7362136410216467600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/591bd659-468b-4c68-ab26-e2d933c38f0a {"id":"591bd659-468b-4c68-ab26-e2d933c38f0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ab935d1 591bd659-468b-4c68-ab26-e2d933c38f0a /tidb/cdc/default/default/upstream/7362136410216467600 {"id":7362136410216467600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.15119.out cli changefeed resume -c kafka-simple-claim-check ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... Create changefeed successfully! ID: 6a87c6c1-9ef2-4d40-b9f9-5ad42d5ab928 Info: {"upstream_id":7362136410216467600,"namespace":"default","id":"6a87c6c1-9ef2-4d40-b9f9-5ad42d5ab928","sink_uri":"kafka://127.0.0.1:9092/ticdc-force_replicate_table-test-28098?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:29:37.993913478+08:00","start_ts":449349150218125314,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349150218125314,"checkpoint_ts":449349150218125314,"checkpoint_time":"2024-04-26 19:29:34.648"} [Fri Apr 26 19:29:38 CST 2024] <<<<<< START kafka consumer in force_replicate_table case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml table mark.finish_mark_3 not exists for 18-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] // container =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table force_replicate_table.t0 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 19-th check, retry later [2024/04/26 19:29:39.795 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/04/26 19:29:40.012 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:29:40.020 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:29:40.200 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:29:40.202 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:29:40.204 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:29:40.206 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] table test.finish_mark not exists for 1-th check, retry later [2024/04/26 19:29:40.397 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:29:40.405 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:29:40.654 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:29:40.660 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:29:40.672 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:29:40.678 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:29:40.836 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:29:40.862 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:29:41.029 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:29:41.034 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:29:41.058 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:29:41.064 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:29:41.210 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:29:41.245 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:29:41.389 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:29:41.394 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:29:41.447 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:29:41.453 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:29:41.567 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ac3d780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:33272, start at 2024-04-26 19:29:40.72765442 +0800 CST m=+5.055476115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:40.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:40.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:40.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ac3d780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:33272, start at 2024-04-26 19:29:40.72765442 +0800 CST m=+5.055476115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:40.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:40.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:40.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later [2024/04/26 19:29:41.640 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:29:41.743 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:29:41.748 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:29:41.836 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:29:41.842 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] table mark.finish_mark_3 not exists for 20-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ac407c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-2q1jv-p18ld, pid:33344, start at 2024-04-26 19:29:40.905082944 +0800 CST m=+5.184340465 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:40.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:40.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:40.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:29:41.915 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:29:42.034 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:29:42.097 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:29:42.102 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [2024/04/26 19:29:42.210 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:29:42.215 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:29:42.269 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:29:42.386 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:29:42.435 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:29:42.444 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:29:42.557 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:29:42.562 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:29:42.600 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:29:42.727 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:29:42.760 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 19:29:42.772 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] table test.finish_mark not exists for 2-th check, retry later [2024/04/26 19:29:42.900 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 19:29:42.906 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 19:29:42.928 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 19:29:43.070 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 19:29:43.086 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 19:29:43.093 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 19:29:43.239 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 19:29:43.244 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 19:29:43.248 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/04/26 19:29:43.414 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 19:29:43.418 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 19:29:43.429 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/04/26 19:29:43.579 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/04/26 19:29:43.579 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/04/26 19:29:43.584 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 21-th check, retry later [2024/04/26 19:29:43.743 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/04/26 19:29:43.756 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/04/26 19:29:43.760 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/04/26 19:29:43.909 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/04/26 19:29:43.939 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/04/26 19:29:43.944 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/04/26 19:29:44.088 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/04/26 19:29:44.100 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/04/26 19:29:44.126 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:29:44.260 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/04/26 19:29:44.301 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/04/26 19:29:44.306 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/04/26 19:29:44.433 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34683.out cli tso query --pd=http://127.0.0.1:2379 table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/04/26 19:29:44.445 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/04/26 19:29:44.477 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/04/26 19:29:44.598 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/04/26 19:29:44.651 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/04/26 19:29:44.657 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] table test.finish_mark exists [2024/04/26 19:29:44.774 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/04/26 19:29:44.789 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] check diff successfully wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 22-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449349152932626433 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349152932626433 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:29:46 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3472034722.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:29:46 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> table ddl_attributes.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 23-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh: line 1: 17273 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 (wd: /tmp/tidb_cdc_test/kafka_column_selector) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb9d3b30-706a-455a-9f2d-7057591bbb22 {"id":"cb9d3b30-706a-455a-9f2d-7057591bbb22","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130986} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ae141d6 cb9d3b30-706a-455a-9f2d-7057591bbb22 /tidb/cdc/default/default/upstream/7362136457858796804 {"id":7362136457858796804,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb9d3b30-706a-455a-9f2d-7057591bbb22 {"id":"cb9d3b30-706a-455a-9f2d-7057591bbb22","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130986} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ae141d6 cb9d3b30-706a-455a-9f2d-7057591bbb22 /tidb/cdc/default/default/upstream/7362136457858796804 {"id":7362136457858796804,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb9d3b30-706a-455a-9f2d-7057591bbb22 {"id":"cb9d3b30-706a-455a-9f2d-7057591bbb22","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130986} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ae141d6 cb9d3b30-706a-455a-9f2d-7057591bbb22 /tidb/cdc/default/default/upstream/7362136457858796804 {"id":7362136457858796804,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34780.out cli changefeed create --start-ts=449349152932626433 '--sink-uri=kafka://127.0.0.1:9092/ticdc-generate-column-test-23218?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table mark.finish_mark_3 not exists for 24-th check, retry later table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 514ad232-9ac8-447c-8bce-2af38ef7e28b Info: {"upstream_id":7362136457858796804,"namespace":"default","id":"514ad232-9ac8-447c-8bce-2af38ef7e28b","sink_uri":"kafka://127.0.0.1:9092/ticdc-generate-column-test-23218?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:29:50.01651265+08:00","start_ts":449349152932626433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349152932626433,"checkpoint_ts":449349152932626433,"checkpoint_time":"2024-04-26 19:29:45.003"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aca17c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:13649, start at 2024-04-26 19:29:47.125668448 +0800 CST m=+5.177243359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:47.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:47.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:47.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aca17c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:13649, start at 2024-04-26 19:29:47.125668448 +0800 CST m=+5.177243359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:47.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:47.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:47.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68aca6d40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-j4c34-z1r1z, pid:13722, start at 2024-04-26 19:29:47.494437845 +0800 CST m=+5.495913706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:47.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:47.495 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:47.495 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_attributes.finish_mark not exists for 5-th check, retry later [2024/04/26 19:29:50.705 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/04/26 19:29:50.910 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:29:50.916 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:29:51.074 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:29:51.076 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:29:51.082 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:29:51.084 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:29:51.237 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:29:51.244 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:29:51.408 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:29:51.410 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:29:51.419 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:29:51.421 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] + set +x [Fri Apr 26 19:29:51 CST 2024] <<<<<< START kafka consumer in generate_column case >>>>>> [2024/04/26 19:29:51.576 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:29:51.587 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] table generate_column.t not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 25-th check, retry later table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later [2024/04/26 19:29:51.740 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:29:51.742 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:29:51.756 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:29:51.758 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:29:51.907 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:29:51.920 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:29:52.089 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:29:52.091 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:29:52.100 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:29:52.102 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:52 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 379 395 --:--:-- --:--:-- --:--:-- 396 {"compatibility":"NONE"}The 1 times to try to start tidb cluster... + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.15055.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:29:52.263 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:29:52.263 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:29:52.423 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:29:52.425 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:29:52.426 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:29:52.428 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] table ddl_attributes.finish_mark not exists for 6-th check, retry later [2024/04/26 19:29:52.597 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:29:52.601 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:29:52.761 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:29:52.763 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 19:29:52.784 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:29:52.787 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:29:52.927 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:29:52.960 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:29:53.088 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:29:53.090 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:29:53.143 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:29:53.145 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:29:53.250 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:29:53.316 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:29:53.425 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 19:29:53.427 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 19:29:53.494 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 19:29:53.496 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 19:29:53.593 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 19:29:53.674 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 19:29:53.773 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 19:29:53.775 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] table generate_column.t not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 26-th check, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:29:53.852 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 19:29:53.854 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 19:29:53.947 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/04/26 19:29:54.040 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 19:29:54.134 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 19:29:54.135 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/04/26 19:29:54.216 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/04/26 19:29:54.218 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] + set +x + tso='449349154924920834 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349154924920834 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349154924920834 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 26 19:29:54 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1509415096.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:29:54.300 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/04/26 19:29:54.389 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/04/26 19:29:54.465 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/04/26 19:29:54.467 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] table ddl_attributes.finish_mark exists [2024/04/26 19:29:54.555 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/04/26 19:29:54.557 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/04/26 19:29:54.628 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/04/26 19:29:54.718 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/04/26 19:29:54.794 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/04/26 19:29:54.796 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] check diff successfully [2024/04/26 19:29:54.892 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/04/26 19:29:54.894 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/04/26 19:29:54.956 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/04/26 19:29:55.054 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/04/26 19:29:55.117 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/04/26 19:29:55.119 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/04/26 19:29:55.215 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/04/26 19:29:55.217 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/04/26 19:29:55.281 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] wait process cdc.test exit for 1-th time... [2024/04/26 19:29:55.378 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/04/26 19:29:55.447 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/04/26 19:29:55.448 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/04/26 19:29:55.544 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/04/26 19:29:55.546 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] wait process cdc.test exit for 2-th time... [2024/04/26 19:29:55.613 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ad00500013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:15599, start at 2024-04-26 19:29:53.204016574 +0800 CST m=+5.281734145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:53.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:53.222 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:53.222 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ad00500013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:15599, start at 2024-04-26 19:29:53.204016574 +0800 CST m=+5.281734145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:53.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:53.222 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:53.222 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ad01dc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:15696, start at 2024-04-26 19:29:53.315686506 +0800 CST m=+5.344316926 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:31:53.322 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:29:53.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:19:53.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table generate_column.t exists table generate_column.t1 exists check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit table mark.finish_mark_3 not exists for 27-th check, retry later check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector_avro Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... [Fri Apr 26 19:29:56 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:57 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00fc6d87-e8db-4113-ad8e-3ffc637aa2bd {"id":"00fc6d87-e8db-4113-ad8e-3ffc637aa2bd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130994} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b0438d4 00fc6d87-e8db-4113-ad8e-3ffc637aa2bd /tidb/cdc/default/default/upstream/7362136491736698645 {"id":7362136491736698645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00fc6d87-e8db-4113-ad8e-3ffc637aa2bd {"id":"00fc6d87-e8db-4113-ad8e-3ffc637aa2bd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130994} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b0438d4 00fc6d87-e8db-4113-ad8e-3ffc637aa2bd /tidb/cdc/default/default/upstream/7362136491736698645 {"id":7362136491736698645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00fc6d87-e8db-4113-ad8e-3ffc637aa2bd {"id":"00fc6d87-e8db-4113-ad8e-3ffc637aa2bd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714130994} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b0438d4 00fc6d87-e8db-4113-ad8e-3ffc637aa2bd /tidb/cdc/default/default/upstream/7362136491736698645 {"id":7362136491736698645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349154924920834 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.15151.out cli changefeed create --start-ts=449349154924920834 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362136491736698645,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:29:57.620130375+08:00","start_ts":449349154924920834,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349154924920834,"checkpoint_ts":449349154924920834,"checkpoint_time":"2024-04-26 19:29:52.603"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff successfully table mark.finish_mark_3 not exists for 28-th check, retry later wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.17025.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:29:59 CST 2024] <<<<<< run test case generate_column success! >>>>>> + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 29-th check, retry later + set +x + tso='449349156489658369 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349156489658369 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:30:00 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1705917061.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh: line 1: 15157 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 30-th check, retry later [2024/04/26 19:30:01.614 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/04/26 19:30:01.814 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:30:01.817 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:30:02.009 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:30:02.010 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:30:02.011 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:30:02.011 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:30:02.187 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:30:02.202 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:30:02.356 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:30:02.358 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] table test.t1 exists + sleep 5 [2024/04/26 19:30:02.379 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:30:02.381 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:30:02.542 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:30:02.557 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:30:02.738 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:30:02.740 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:30:02.751 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:30:02.753 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:30:02.938 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:30:02.939 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:30:03.122 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:30:03.124 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:30:03.143 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check_avro Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... [2024/04/26 19:30:03.146 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:30:03.312 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:30:03.350 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:30:03 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264 {"id":"a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131000} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b13b8d4 a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264 /tidb/cdc/default/default/upstream/7362136519249516002 {"id":7362136519249516002,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264 {"id":"a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131000} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b13b8d4 a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264 /tidb/cdc/default/default/upstream/7362136519249516002 {"id":7362136519249516002,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264 {"id":"a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131000} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b13b8d4 a92bcc7d-7d2f-4ee1-ae9f-b273a96ea264 /tidb/cdc/default/default/upstream/7362136519249516002 {"id":7362136519249516002,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.17112.out cli changefeed create --start-ts=449349156489658369 '--sink-uri=kafka://127.0.0.1:9092/ticdc-savepoint-test-19778?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/04/26 19:30:03.500 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:30:03.502 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:30:03.555 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:30:03.558 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] Create changefeed successfully! ID: 967ea1cb-521e-4174-a8f4-ac9c0d625b58 Info: {"upstream_id":7362136519249516002,"namespace":"default","id":"967ea1cb-521e-4174-a8f4-ac9c0d625b58","sink_uri":"kafka://127.0.0.1:9092/ticdc-savepoint-test-19778?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:30:03.60896733+08:00","start_ts":449349156489658369,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349156489658369,"checkpoint_ts":449349156489658369,"checkpoint_time":"2024-04-26 19:29:58.572"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 31-th check, retry later [2024/04/26 19:30:03.687 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:30:03.766 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:30:03.870 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:30:03.872 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:30:03.971 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:30:03.974 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 19:30:04.055 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:30:04.178 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:30:04.239 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:30:04.241 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:30:04.387 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:30:04.389 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:30:04.424 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:30:04.585 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:30:04.607 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 19:30:04.609 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] Verifying downstream PD is started... [2024/04/26 19:30:04.783 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 19:30:04.786 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 19:30:04.788 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 19:30:04.959 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 19:30:04.961 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 19:30:04.964 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 19:30:05.126 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 19:30:05.141 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 19:30:05.143 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 19:30:05.298 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/04/26 19:30:05.300 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/04/26 19:30:05.340 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/04/26 19:30:05.477 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] + set +x [Fri Apr 26 19:30:05 CST 2024] <<<<<< START kafka consumer in savepoint case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:30:05.530 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 19:30:05.532 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/04/26 19:30:05.650 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/04/26 19:30:05.652 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/04/26 19:30:05.718 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/04/26 19:30:05.832 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/04/26 19:30:05.906 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/04/26 19:30:05.909 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/04/26 19:30:06.028 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/04/26 19:30:06.030 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/04/26 19:30:06.106 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/04/26 19:30:06.203 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] table mark.finish_mark_3 not exists for 32-th check, retry later [2024/04/26 19:30:02.441 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column name not equal, one is X8EhY another is c"] [2024/04/26 19:30:06.297 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/04/26 19:30:06.299 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/04/26 19:30:06.393 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/04/26 19:30:06.395 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/04/26 19:30:06.475 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/04/26 19:30:06.575 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] table savepoint.finish_mark not exists for 1-th check, retry later check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 4-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12890 21.7 0.5 4696436 2227312 ? Sl 19:29 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12891 29.4 0.5 4730224 2273468 ? Sl 19:29 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12892 21.7 0.5 4688756 2200468 ? Sl 19:29 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12894 28.4 0.5 4729200 2264652 ? Sl 19:29 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2307 0 --:--:-- --:--:-- --:--:-- 2314 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:30:05.003","puller_resolved_ts":"2024-04-26 19:29:59.403","last_synced_ts":"2024-04-26 19:29:59.852","now_ts":"2024-04-26 19:30:07.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:30:05.003","puller_resolved_ts":"2024-04-26' '19:29:59.403","last_synced_ts":"2024-04-26' '19:29:59.852","now_ts":"2024-04-26' '19:30:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:30:05.003","puller_resolved_ts":"2024-04-26' '19:29:59.403","last_synced_ts":"2024-04-26' '19:29:59.852","now_ts":"2024-04-26' '19:30:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 33-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) table savepoint.finish_mark not exists for 2-th check, retry later [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68add3cc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:31240, start at 2024-04-26 19:30:06.744918329 +0800 CST m=+5.179318991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:06.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:06.756 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:06.756 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68add3cc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:31240, start at 2024-04-26 19:30:06.744918329 +0800 CST m=+5.179318991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:06.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:06.756 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:06.756 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68add5f4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:31322, start at 2024-04-26 19:30:06.861435096 +0800 CST m=+5.247664969 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:06.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:06.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:06.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 34-th check, retry later table savepoint.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table mark.finish_mark_3 not exists for 35-th check, retry later [Pipeline] // dir wait process cdc.test exit for 2-th time... [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Fri Apr 26 19:30:11 CST 2024] <<<<<< START cdc server in kafka_column_selector_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.3269932701.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // node [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:30:12 CST 2024] <<<<<< run test case savepoint success! >>>>>> [2024/04/26 19:30:12.513 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/04/26 19:30:12.708 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:30:12.720 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:30:12.884 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:30:12.884 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:30:12.898 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:30:12.900 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:30:13.062 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:30:13.068 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:30:13.239 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:30:13.240 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:30:13.245 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:30:13.247 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:30:13.409 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:30:13.422 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:30:13.584 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:30:13.585 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:30:13.604 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:30:13.606 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:30:13.777 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:30:13.808 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:30:13.970 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:30:13.971 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:30:14.012 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:30:14.014 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:30:14.156 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:30:14.211 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:30:14.346 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:30:14.347 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] table mark.finish_mark_3 not exists for 36-th check, retry later [2024/04/26 19:30:14.410 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:30:14.412 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:30:14.533 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:30:14.599 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:30:14.722 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:30:14.723 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:30:14.794 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:30:14.796 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/04/26 19:30:14.909 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:30:14.991 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:30:15.099 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:30:15.100 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 5-th time, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:30:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0193c4f3-1df0-4b01-9067-8886340e839d {"id":"0193c4f3-1df0-4b01-9067-8886340e839d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b4af0cd 0193c4f3-1df0-4b01-9067-8886340e839d /tidb/cdc/default/default/upstream/7362136579596930620 {"id":7362136579596930620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0193c4f3-1df0-4b01-9067-8886340e839d {"id":"0193c4f3-1df0-4b01-9067-8886340e839d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b4af0cd 0193c4f3-1df0-4b01-9067-8886340e839d /tidb/cdc/default/default/upstream/7362136579596930620 {"id":7362136579596930620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0193c4f3-1df0-4b01-9067-8886340e839d {"id":"0193c4f3-1df0-4b01-9067-8886340e839d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b4af0cd 0193c4f3-1df0-4b01-9067-8886340e839d /tidb/cdc/default/default/upstream/7362136579596930620 {"id":7362136579596930620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.cli.32758.out cli changefeed create --start-ts=449349159998193665 '--sink-uri=kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 [2024/04/26 19:30:15.177 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:30:15.179 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:30:15.275 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:30:15.363 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ae44a80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:16371, start at 2024-04-26 19:30:13.943524025 +0800 CST m=+5.121155025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:13.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:13.930 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:13.930 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ae44a80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:16371, start at 2024-04-26 19:30:13.943524025 +0800 CST m=+5.121155025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:13.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:13.930 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:13.930 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ae46e40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:16442, start at 2024-04-26 19:30:14.075216782 +0800 CST m=+5.202852393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:14.083 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:14.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:14.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/error.log [2024/04/26 19:30:15.445 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 19:30:15.446 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 19:30:15.548 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 19:30:15.550 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 19:30:15.622 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] Create changefeed successfully! ID: test Info: {"upstream_id":7362136579596930620,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true","create_time":"2024-04-26T19:30:15.566687957+08:00","start_ts":449349159998193665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.*"],"columns":["*","!b"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349159998193665,"checkpoint_ts":449349159998193665,"checkpoint_time":"2024-04-26 19:30:11.956"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:30:15.729 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 19:30:15.797 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 19:30:15.798 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 19:30:15.908 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 19:30:15.910 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 19:30:15.977 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 19:30:16.094 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/04/26 19:30:16.152 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/04/26 19:30:16.153 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/04/26 19:30:16.273 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 19:30:16.275 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/04/26 19:30:16.318 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] table mark.finish_mark_3 not exists for 37-th check, retry later [2024/04/26 19:30:16.454 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/04/26 19:30:16.482 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/04/26 19:30:16.482 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/04/26 19:30:16.637 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/04/26 19:30:16.639 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/04/26 19:30:16.649 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/04/26 19:30:16.818 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/04/26 19:30:16.820 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/04/26 19:30:16.821 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/04/26 19:30:16.988 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/04/26 19:30:16.999 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/04/26 19:30:17.001 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/04/26 19:30:17.156 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/04/26 19:30:17.157 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] + set +x [Fri Apr 26 19:30:17 CST 2024] <<<<<< START kafka consumer in kafka_column_selector_avro case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml schema registry uri found: http://127.0.0.1:8088 [2024/04/26 19:30:17.181 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/04/26 19:30:17.323 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/04/26 19:30:17.357 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/04/26 19:30:17.359 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/04/26 19:30:17.495 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/04/26 19:30:17.496 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 38-th check, retry later [Fri Apr 26 19:30:19 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.1779517797.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting build checksum checker... table test.finishmark not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 39-th check, retry later table test.finishmark not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 40-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:30:22 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1a4ff52-c93a-4d6a-8567-ca9724cc5956 {"id":"d1a4ff52-c93a-4d6a-8567-ca9724cc5956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b6727d3 d1a4ff52-c93a-4d6a-8567-ca9724cc5956 /tidb/cdc/default/default/upstream/7362136602735040881 {"id":7362136602735040881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1a4ff52-c93a-4d6a-8567-ca9724cc5956 {"id":"d1a4ff52-c93a-4d6a-8567-ca9724cc5956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b6727d3 d1a4ff52-c93a-4d6a-8567-ca9724cc5956 /tidb/cdc/default/default/upstream/7362136602735040881 {"id":7362136602735040881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1a4ff52-c93a-4d6a-8567-ca9724cc5956 {"id":"d1a4ff52-c93a-4d6a-8567-ca9724cc5956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b6727d3 d1a4ff52-c93a-4d6a-8567-ca9724cc5956 /tidb/cdc/default/default/upstream/7362136602735040881 {"id":7362136602735040881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.17853.out cli tso query --pd=http://127.0.0.1:2379 table test.finishmark exists [2024/04/26 19:30:23.170 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/04/26 19:30:23.172 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/04/26 19:30:23.172 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=10.387213ms] [2024/04/26 19:30:23.178 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/04/26 19:30:23.180 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/04/26 19:30:23.180 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=8.208137ms] [2024/04/26 19:30:23.180 +08:00] [INFO] [main.go:95] ["compare checksum passed"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:30:23 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> [2024/04/26 19:30:23.414 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/04/26 19:30:23.648 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:30:23.658 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:30:23.845 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:30:23.846 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:30:23.876 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:30:23.877 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:30:24.043 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:30:24.087 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] + set +x + tso='449349162783211522 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349162783211522 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.17887.out cli changefeed create --start-ts=449349162783211522 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-20670?protocol=simple&encoding-format=avro' -c kafka-simple-claim-check-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml [2024/04/26 19:30:24.238 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:30:24.239 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:30:24.276 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:30:24.277 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:30:24.450 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] table mark.finish_mark_3 not exists for 41-th check, retry later [2024/04/26 19:30:21.647 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:30:24 CST 2024] <<<<<< run test case cdc success! >>>>>> Create changefeed successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7362136602735040881,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-20670?protocol=simple\u0026encoding-format=avro","create_time":"2024-04-26T19:30:24.490276545+08:00","start_ts":449349162783211522,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349162783211522,"checkpoint_ts":449349162783211522,"checkpoint_time":"2024-04-26 19:30:22.580"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:30:24.482 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:30:24.665 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:30:24.666 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:30:24.689 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:30:24.690 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:30:24.949 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:30:24.982 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 6-th time, retry later [2024/04/26 19:30:25.199 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:30:25.200 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:30:25.239 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:30:25.240 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:30:25.459 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:30:25.503 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:30:25.723 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:30:25.725 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:30:25.773 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:30:25.774 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] + set +x [2024/04/26 19:30:26.024 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:30:26.069 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:30:23 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> table mark.finish_mark_3 not exists for 42-th check, retry later [2024/04/26 19:30:26.350 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:30:26.352 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:30:26.388 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 19:30:26.390 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:30:26.758 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:30:26.825 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:30:26.994 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:30:26.995 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:30:27.037 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:30:27.038 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:30:27.187 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:30:27.230 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:30:27.370 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 19:30:27.371 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 19:30:27.409 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 19:30:27.410 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 19:30:27.551 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 19:30:27.594 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 19:30:27.782 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 19:30:27.783 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 19:30:27.863 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 19:30:27.864 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] table mark.finish_mark_3 not exists for 43-th check, retry later [2024/04/26 19:30:28.081 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 19:30:28.167 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 44-th check, retry later table mark.finish_mark_3 not exists for 45-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.17937.out cli changefeed pause -c kafka-simple-claim-check-avro Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.17975.out cli changefeed update -c kafka-simple-claim-check-avro '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-20670?protocol=simple&encoding-format=avro&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml --no-confirm Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 46-th check, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-20670?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-20670?protocol=simple&encoding-format=avro&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0013ece28} {Type:update Path:[Config SyncPointRetention] From: To:0xc0013ece38} {Type:update Path:[Config Consistent] From: To:0xc001392230} Update changefeed config successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7362136602735040881,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-20670?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=2048","create_time":"2024-04-26T19:30:24.490276545+08:00","start_ts":449349162783211522,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":0,"checkpoint_ts":449349165208043526,"checkpoint_time":"2024-04-26 19:30:31.830"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:30:35.881 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m6.601281447s"] table mark.finish_mark_3 not exists for 47-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.18020.out cli changefeed resume -c kafka-simple-claim-check-avro table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 7-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 48-th check, retry later table mark.finish_mark not exists for 2-th check, retry later + set +x table test.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 2-th check, retry later table mark.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: kafka... <<================= table mark.finish_mark exists wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:30:42 CST 2024] <<<<<< run test case multi_source success! >>>>>> check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68afd50c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:18276, start at 2024-04-26 19:30:39.569778095 +0800 CST m=+5.469446807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:39.578 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:39.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:39.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68afd50c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:18276, start at 2024-04-26 19:30:39.569778095 +0800 CST m=+5.469446807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:39.578 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:39.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:39.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68afd50c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-ljzmt-2nk64, pid:18351, start at 2024-04-26 19:30:39.566244965 +0800 CST m=+5.414678558 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:39.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:39.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:39.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... table test.finish_mark exists wait process cdc.test exit for 2-th time... check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:30:43 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1415 100 678 100 737 1330 1445 --:--:-- --:--:-- --:--:-- 1447 HTTP/1.1 201 Created Date: Fri, 26 Apr 2024 11:30:43 GMT Location: http://localhost:8083/connectors/my-connector Content-Type: application/json Content-Length: 678 Server: Jetty(9.4.51.v20230217) {"name":"my-connector","config":{"connector.class":"io.debezium.connector.mysql.MySqlConnector","tasks.max":"1","database.hostname":"127.0.0.1","database.port":"3310","database.user":"debezium","database.password":"dbz","database.server.id":"184054","topic.prefix":"dbserver1","schema.history.internal.kafka.bootstrap.servers":"127.0.0.1:9092","schema.history.internal.kafka.topic":"schemahistory.test","transforms":"x","transforms.x.type":"org.apache.kafka.connect.transforms.RegexRouter","transforms.x.regex":"(.*)","transforms.x.replacement":"output_debezium","binary.handling.mode":"base64","decimal.handling.mode":"double","name":"my-connector"},"tasks":[],"type":"source"}The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:30:44 CST 2024] <<<<<< run test case default_value success! >>>>>> + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19797.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449349168609361921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349168609361921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349168609361921 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:30:46 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1983719839.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/debezium Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:30:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d74c0756-35a8-4c68-8492-1d2cd4eff27e {"id":"d74c0756-35a8-4c68-8492-1d2cd4eff27e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bc81fd6 d74c0756-35a8-4c68-8492-1d2cd4eff27e /tidb/cdc/default/default/upstream/7362136721666921919 {"id":7362136721666921919,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d74c0756-35a8-4c68-8492-1d2cd4eff27e {"id":"d74c0756-35a8-4c68-8492-1d2cd4eff27e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bc81fd6 d74c0756-35a8-4c68-8492-1d2cd4eff27e /tidb/cdc/default/default/upstream/7362136721666921919 {"id":7362136721666921919,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d74c0756-35a8-4c68-8492-1d2cd4eff27e {"id":"d74c0756-35a8-4c68-8492-1d2cd4eff27e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bc81fd6 d74c0756-35a8-4c68-8492-1d2cd4eff27e /tidb/cdc/default/default/upstream/7362136721666921919 {"id":7362136721666921919,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349168609361921 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19897.out cli changefeed create --start-ts=449349168609361921 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362136721666921919,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:30:49.902748927+08:00","start_ts":449349168609361921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349168609361921,"checkpoint_ts":449349168609361921,"checkpoint_time":"2024-04-26 19:30:44.805"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3079 0 --:--:-- --:--:-- --:--:-- 3112 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:30:51.155","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:30:51.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:30:51.155","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:30:51.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:30:51.155","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:30:51.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-04-26 19:30:51.155' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:30:51.155","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:30:51.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:30:51.155","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:30:51.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-04-26 19:30:51' + echo 'sink_checkpoint_ts is 2024-04-26' 19:30:51.155 sink_checkpoint_ts is 2024-04-26 19:30:51.155 ++ date -d '2024-04-26 19:30:51.155' +%s + checkpoint_timestamp=1714131051 ++ date -d '2024-04-26 19:30:51' +%s + current_timestamp=1714131051 + '[' 0 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b074480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:7151, start at 2024-04-26 19:30:49.769583522 +0800 CST m=+5.178435253 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:49.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:49.746 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:49.746 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b074480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:7151, start at 2024-04-26 19:30:49.769583522 +0800 CST m=+5.178435253 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:49.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:49.746 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:49.746 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b076840013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-llwp6-lbfd3, pid:7228, start at 2024-04-26 19:30:49.910624483 +0800 CST m=+5.263840730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:49.919 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:49.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:49.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 8-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b099b4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:24648, start at 2024-04-26 19:30:52.149636739 +0800 CST m=+5.114486745 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:52.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:52.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:52.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b099b4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:24648, start at 2024-04-26 19:30:52.149636739 +0800 CST m=+5.114486745 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:52.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:52.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:52.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b09c0c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-wrgs8-1m0vk, pid:24728, start at 2024-04-26 19:30:52.299729691 +0800 CST m=+5.211079709 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:52.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:52.291 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:52.291 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Fri Apr 26 19:30:53 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.85498551.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26047.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh: line 1: 18056 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:30:56 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f1c1fbb-6aea-43ef-b906-4a5e36af2380 {"id":"4f1c1fbb-6aea-43ef-b906-4a5e36af2380","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131054} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bf502c8 4f1c1fbb-6aea-43ef-b906-4a5e36af2380 /tidb/cdc/default/default/upstream/7362136765697972718 {"id":7362136765697972718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f1c1fbb-6aea-43ef-b906-4a5e36af2380 {"id":"4f1c1fbb-6aea-43ef-b906-4a5e36af2380","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131054} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bf502c8 4f1c1fbb-6aea-43ef-b906-4a5e36af2380 /tidb/cdc/default/default/upstream/7362136765697972718 {"id":7362136765697972718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f1c1fbb-6aea-43ef-b906-4a5e36af2380 {"id":"4f1c1fbb-6aea-43ef-b906-4a5e36af2380","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131054} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bf502c8 4f1c1fbb-6aea-43ef-b906-4a5e36af2380 /tidb/cdc/default/default/upstream/7362136765697972718 {"id":7362136765697972718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8614.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-only-block-related-table + set +x + tso='449349171450216449 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349171450216449 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7362136765697972718,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:30:57.203904839+08:00","start_ts":449349171818528773,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349171818528773,"checkpoint_ts":449349171818528773,"checkpoint_time":"2024-04-26 19:30:57.047"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Fri Apr 26 19:30:57 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2609226094.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:30:58 CST 2024] <<<<<< START kafka consumer in ddl_only_block_related_table case >>>>>> table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 5429 0 --:--:-- --:--:-- --:--:-- 5522 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:30:58.155","puller_resolved_ts":"2024-04-26 19:30:51.655","last_synced_ts":"2024-04-26 19:30:52.105","now_ts":"2024-04-26 19:30:59.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:30:58.155","puller_resolved_ts":"2024-04-26' '19:30:51.655","last_synced_ts":"2024-04-26' '19:30:52.105","now_ts":"2024-04-26' '19:30:59.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:30:58.155","puller_resolved_ts":"2024-04-26' '19:30:51.655","last_synced_ts":"2024-04-26' '19:30:52.105","now_ts":"2024-04-26' '19:30:59.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b102fc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:34068, start at 2024-04-26 19:30:58.893653164 +0800 CST m=+5.171159371 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:58.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:58.879 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:58.879 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:00 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78e009da-62d2-40e6-adf9-cfecad12f53a {"id":"78e009da-62d2-40e6-adf9-cfecad12f53a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131057} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c027af5 78e009da-62d2-40e6-adf9-cfecad12f53a /tidb/cdc/default/default/upstream/7362136774641815066 {"id":7362136774641815066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78e009da-62d2-40e6-adf9-cfecad12f53a {"id":"78e009da-62d2-40e6-adf9-cfecad12f53a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131057} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c027af5 78e009da-62d2-40e6-adf9-cfecad12f53a /tidb/cdc/default/default/upstream/7362136774641815066 {"id":7362136774641815066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78e009da-62d2-40e6-adf9-cfecad12f53a {"id":"78e009da-62d2-40e6-adf9-cfecad12f53a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131057} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c027af5 78e009da-62d2-40e6-adf9-cfecad12f53a /tidb/cdc/default/default/upstream/7362136774641815066 {"id":7362136774641815066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26160.out cli changefeed create --start-ts=449349171450216449 '--sink-uri=kafka://127.0.0.1:9092/ticdc-split-region-test-8442?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Create changefeed successfully! ID: 23740f12-2ef8-41bd-b6a9-ede118a6fd29 Info: {"upstream_id":7362136774641815066,"namespace":"default","id":"23740f12-2ef8-41bd-b6a9-ede118a6fd29","sink_uri":"kafka://127.0.0.1:9092/ticdc-split-region-test-8442?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:31:00.939885904+08:00","start_ts":449349171450216449,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349171450216449,"checkpoint_ts":449349171450216449,"checkpoint_time":"2024-04-26 19:30:55.642"} PASS table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b102fc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:34068, start at 2024-04-26 19:30:58.893653164 +0800 CST m=+5.171159371 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:58.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:58.879 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:58.879 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1054c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-09rd0-1r27c, pid:34159, start at 2024-04-26 19:30:59.03513394 +0800 CST m=+5.246656050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:59.041 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:59.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:59.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/debezium/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/debezium/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/debezium/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [Fri Apr 26 19:31:02 CST 2024] <<<<<< START kafka consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_adapter_compatibility Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later table split_region.test1 not exists for 2-th check, retry later [Fri Apr 26 19:31:04 CST 2024] <<<<<< START cdc server in debezium case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.3549435496.out server --log-file /tmp/tidb_cdc_test/debezium/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/debezium/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release *************************** 1. row *************************** count(distinct region_id): 1 split_and_random_merge scale: 80 table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table split_region.test1 exists table split_region.test2 exists check diff failed 1-th time, retry later check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 9-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24721370-906b-45fa-9c26-65c2eb2f6f45 {"id":"24721370-906b-45fa-9c26-65c2eb2f6f45","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131064} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c1504ce 24721370-906b-45fa-9c26-65c2eb2f6f45 /tidb/cdc/default/default/upstream/7362136804790906831 {"id":7362136804790906831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24721370-906b-45fa-9c26-65c2eb2f6f45 {"id":"24721370-906b-45fa-9c26-65c2eb2f6f45","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131064} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c1504ce 24721370-906b-45fa-9c26-65c2eb2f6f45 /tidb/cdc/default/default/upstream/7362136804790906831 {"id":7362136804790906831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24721370-906b-45fa-9c26-65c2eb2f6f45 {"id":"24721370-906b-45fa-9c26-65c2eb2f6f45","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131064} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c1504ce 24721370-906b-45fa-9c26-65c2eb2f6f45 /tidb/cdc/default/default/upstream/7362136804790906831 {"id":7362136804790906831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.cli.35553.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/output_ticdc?protocol=debezium&kafka-version=2.4.0' table ddl_only_block_related_table.finish_mark exists Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 7a5d1e98-954b-40e7-92af-a75773df73e7 Info: {"upstream_id":7362136804790906831,"namespace":"default","id":"7a5d1e98-954b-40e7-92af-a75773df73e7","sink_uri":"kafka://127.0.0.1:9092/output_ticdc?protocol=debezium\u0026kafka-version=2.4.0","create_time":"2024-04-26T19:31:07.636073367+08:00","start_ts":449349174566322178,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"debezium","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349174566322178,"checkpoint_ts":449349174566322178,"checkpoint_time":"2024-04-26 19:31:07.529"} PASS wait process 8554 exit for 1-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process 8554 exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8554) - No such process wait process 8554 exit for 3-th time... process 8554 already exit [Fri Apr 26 19:31:08 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.87318733.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + set +x go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873 go: downloading github.com/google/go-cmp v0.6.0 go: downloading github.com/alecthomas/chroma v0.10.0 go: downloading github.com/google/uuid v1.3.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4 go: downloading github.com/thessem/zap-prettyconsole v0.3.0 go: downloading github.com/segmentio/kafka-go v0.4.45 go: downloading go.uber.org/zap v1.26.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/Code-Hex/dd v1.1.0 go: downloading github.com/klauspost/compress v1.17.1 go: downloading github.com/pierrec/lz4/v4 v4.1.15 go: downloading github.com/dlclark/regexp2 v1.4.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 go: downloading github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 go: downloading golang.org/x/exp v0.0.0-20231006140011-7918f672742d go: downloading github.com/golang/protobuf v1.5.3 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e go: downloading github.com/shirou/gopsutil/v3 v3.23.10 go: downloading github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 go: downloading github.com/prometheus/client_model v0.5.0 go: downloading github.com/prometheus/client_golang v1.17.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading google.golang.org/protobuf v1.31.0 go: downloading github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 go: downloading google.golang.org/grpc v1.59.0 go: downloading github.com/prometheus/common v0.45.0 go: downloading github.com/prometheus/procfs v0.12.0 go: downloading github.com/cespare/xxhash/v2 v2.2.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/rogpeppe/go-internal v1.11.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.10 go: downloading github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 go: downloading go.etcd.io/etcd/api/v3 v3.5.10 go: downloading golang.org/x/sync v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.10 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b go: downloading golang.org/x/net v0.18.0 *************************** 1. row *************************** count(distinct region_id): 32 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362136765697972718 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:30:57.203904839 +0800 CST StartTs:449349171818528773 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025b2750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349171857850370} {CheckpointTs:449349175383425028 MinTableBarrierTs:449349175645569028 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6802d5b4-2351-4419-bec9-ceefcb5ad8a8 {"id":"6802d5b4-2351-4419-bec9-ceefcb5ad8a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bf5039a 6802d5b4-2351-4419-bec9-ceefcb5ad8a8 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362136765697972718,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:30:57.203904839+08:00","start-ts":449349171818528773,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349171857850370} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349175383425028,"min-table-barrier-ts":449349175645569028,"admin-job-type":0} /tidb/cdc/default/default/task/position/6802d5b4-2351-4419-bec9-ceefcb5ad8a8/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362136765697972718 {"id":7362136765697972718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362136765697972718 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:30:57.203904839 +0800 CST StartTs:449349171818528773 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025b2750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349171857850370} {CheckpointTs:449349175383425028 MinTableBarrierTs:449349175645569028 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6802d5b4-2351-4419-bec9-ceefcb5ad8a8 {"id":"6802d5b4-2351-4419-bec9-ceefcb5ad8a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bf5039a 6802d5b4-2351-4419-bec9-ceefcb5ad8a8 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362136765697972718,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:30:57.203904839+08:00","start-ts":449349171818528773,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"syn+ grep -q 'failed to get info:' ced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349171857850370} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349175383425028,"min-table-barrier-ts":449349175645569028,"admin-job-type":0} /tidb/cdc/default/default/task/position/6802d5b4-2351-4419-bec9-ceefcb5ad8a8/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362136765697972718 {"id":7362136765697972718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362136765697972718 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:30:57.203904839 +0800 CST StartTs:449349171818528773 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025b2750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-37-gcf8946a3a Epoch:449349171857850370} {CheckpointTs:449349175383425028 MinTableBarrierTs:449349175645569028 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349175645569029, checkpointTs: 449349175383425028, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6802d5b4-2351-4419-bec9-ceefcb5ad8a8 {"id":"6802d5b4-2351-4419-bec9-ceefcb5ad8a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bf5039a 6802d5b4-2351-4419-bec9-ceefcb5ad8a8 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362136765697972718,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22963?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:30:57.203904839+08:00","start-ts":449349171818528773,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"syn+ grep -q 'etcd info' ced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-37-gcf8946a3a","epoch":449349171857850370} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349175383425028,"min-table-barrier-ts":449349175645569028,"admin-job-type":0} /tidb/cdc/default/default/task/position/6802d5b4-2351-4419-bec9-ceefcb5ad8a8/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362136765697972718 {"id":7362136765697972718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_ts_not_forward ddl-only-block-related-table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1d30c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:19187, start at 2024-04-26 19:31:12.238608454 +0800 CST m=+5.102425555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:12.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:12.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:12.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1d30c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:19187, start at 2024-04-26 19:31:12.238608454 +0800 CST m=+5.102425555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:12.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:12.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:12.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1d4840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-96drt-3vwvp, pid:19270, start at 2024-04-26 19:31:12.328617895 +0800 CST m=+5.142737963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:12.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:12.339 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:12.339 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1e18c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:21951, start at 2024-04-26 19:31:13.12950565 +0800 CST m=+5.114627126 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:13.135 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:13.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:13.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1e18c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:21951, start at 2024-04-26 19:31:13.12950565 +0800 CST m=+5.114627126 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:13.135 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:13.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:13.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b1e30c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:22034, start at 2024-04-26 19:31:13.236476137 +0800 CST m=+5.165117046 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:13.243 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:13.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:13.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.20668.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.23431.out cli tso query --pd=http://127.0.0.1:2379 table region_merge.t1 exists check diff failed 1-th time, retry later + set +x + tso='449349176720097281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349176720097281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:31:17 CST 2024] <<<<<< START cdc server in canal_json_adapter_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.2070920711.out server --log-file /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task failed 1-th time, retry later + set +x + tso='449349176963366913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349176963366913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:31:18 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2348923491.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... check_ts_not_forward ddl-only-block-related-table wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:31:19 CST 2024] <<<<<< run test case region_merge success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bdb74fd-6409-44ca-91dc-4f387768b933 {"id":"7bdb74fd-6409-44ca-91dc-4f387768b933","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131077} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c4b25d0 7bdb74fd-6409-44ca-91dc-4f387768b933 /tidb/cdc/default/default/upstream/7362136863634269884 {"id":7362136863634269884,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bdb74fd-6409-44ca-91dc-4f387768b933 {"id":"7bdb74fd-6409-44ca-91dc-4f387768b933","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131077} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c4b25d0 7bdb74fd-6409-44ca-91dc-4f387768b933 /tidb/cdc/default/default/upstream/7362136863634269884 {"id":7362136863634269884,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bdb74fd-6409-44ca-91dc-4f387768b933 {"id":"7bdb74fd-6409-44ca-91dc-4f387768b933","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131077} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c4b25d0 7bdb74fd-6409-44ca-91dc-4f387768b933 /tidb/cdc/default/default/upstream/7362136863634269884 {"id":7362136863634269884,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.20766.out cli changefeed create --start-ts=449349176720097281 '--sink-uri=kafka://127.0.0.1:9092/test?protocol=canal-json&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 8594b3b8-5829-463e-8c76-08c23c30cd4b Info: {"upstream_id":7362136863634269884,"namespace":"default","id":"8594b3b8-5829-463e-8c76-08c23c30cd4b","sink_uri":"kafka://127.0.0.1:9092/test?protocol=canal-json\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:31:20.748147255+08:00","start_ts":449349176720097281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349176720097281,"checkpoint_ts":449349176720097281,"checkpoint_time":"2024-04-26 19:31:15.745"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:21 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c240517-5ace-46d6-8b1e-57a567bdef21 {"id":"9c240517-5ace-46d6-8b1e-57a567bdef21","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c49e3e9 9c240517-5ace-46d6-8b1e-57a567bdef21 /tidb/cdc/default/default/upstream/7362136857804632742 {"id":7362136857804632742,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c240517-5ace-46d6-8b1e-57a567bdef21 {"id":"9c240517-5ace-46d6-8b1e-57a567bdef21","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c49e3e9 9c240517-5ace-46d6-8b1e-57a567bdef21 /tidb/cdc/default/default/upstream/7362136857804632742 {"id":7362136857804632742,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9c240517-5ace-46d6-8b1e-57a567bdef21 {"id":"9c240517-5ace-46d6-8b1e-57a567bdef21","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c49e3e9 9c240517-5ace-46d6-8b1e-57a567bdef21 /tidb/cdc/default/default/upstream/7362136857804632742 {"id":7362136857804632742,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.23547.out cli changefeed create --start-ts=449349176963366913 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-simple-test-24468?protocol=open-protocol&partition-num=4&kafka-client-id=cdc_test_simple&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 25028b41-094b-4ac3-9ed9-5f261eeb6cef Info: {"upstream_id":7362136857804632742,"namespace":"default","id":"25028b41-094b-4ac3-9ed9-5f261eeb6cef","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-simple-test-24468?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=cdc_test_simple\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:31:21.871733357+08:00","start_ts":449349176963366913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349176963366913,"checkpoint_ts":449349176963366913,"checkpoint_time":"2024-04-26 19:31:16.673"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check diff failed 2-th time, retry later + set +x [Fri Apr 26 19:31:23 CST 2024] <<<<<< START kafka consumer in simple case >>>>>> succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 id=4,a=2 doesn't exist in downstream table force_replicate_table.t0 run task failed 10-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:31:26 CST 2024] <<<<<< run test case split_region success! >>>>>> table test.binary_columns not exists for 1-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later table test.binary_columns not exists for 2-th check, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.binary_columns not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:31:31 CST 2024] <<<<<< run test case simple success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.binary_columns not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... table test.binary_columns not exists for 5-th check, retry later table test.binary_columns not exists for 6-th check, retry later 7:31PM INF > Info cdc.mysql=kafka://127.0.0.1:9092/output_debezium cdc.tidb=kafka://127.0.0.1:9092/output_ticdc db.mysql=root@tcp(127.0.0.1:3310)/{db}?allowNativePasswords=true db.tidb=root@tcp(127.0.0.1:4000)/{db}?allowNativePasswords=true 7:31PM INF > Run case=sql/data_types.sql \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1732/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.binary_columns not exists for 7-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.binary_columns exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task failed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b3e4340017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:7269, start at 2024-04-26 19:31:46.079197145 +0800 CST m=+5.128271059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:46.085 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:46.061 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:46.061 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b3e4340017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:7269, start at 2024-04-26 19:31:46.079197145 +0800 CST m=+5.128271059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:46.085 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:46.061 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:46.061 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b3e5a80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-0g9sx-0qt5m, pid:7362, start at 2024-04-26 19:31:46.194795241 +0800 CST m=+5.190661988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:33:46.201 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:31:46.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:21:46.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Starting Downstream PD... Release Version: v8.2.0-alpha-10-g50c80407c Edition: Community Git Commit Hash: 50c80407cd57e96a85452dd1601fcb41c1f263cf Git Branch: master UTC Build Time: 2024-04-26 02:17:39 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.8758.out cli tso query --pd=http://127.0.0.1:2379 table test.binary_columns exists check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 2118288ceb3f52a4a8431acba4fc2c560def0732 Git Commit Branch: master UTC Build Time: 2024-04-25 21:05:46 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449349186122153985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449349186122153985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:31:53 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.87958797.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-51-g944fff519c Edition: Community Git Commit Hash: 944fff519c90039747affb94067439ff9541f2fb Git Branch: master UTC Build Time: 2024-04-26 09:54:14 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:56 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6eb541b9-f092-48db-ad17-595dea19808e {"id":"6eb541b9-f092-48db-ad17-595dea19808e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131113} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ccd45da 6eb541b9-f092-48db-ad17-595dea19808e /tidb/cdc/default/default/upstream/7362137005110325270 {"id":7362137005110325270,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6eb541b9-f092-48db-ad17-595dea19808e {"id":"6eb541b9-f092-48db-ad17-595dea19808e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131113} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ccd45da 6eb541b9-f092-48db-ad17-595dea19808e /tidb/cdc/default/default/upstream/7362137005110325270 {"id":7362137005110325270,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6eb541b9-f092-48db-ad17-595dea19808e {"id":"6eb541b9-f092-48db-ad17-595dea19808e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-37-gcf8946a3a","git-hash":"cf8946a3a9cb6bf28577ea67d4848a4fad982d67","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714131113} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2ccd45da 6eb541b9-f092-48db-ad17-595dea19808e /tidb/cdc/default/default/upstream/7362137005110325270 {"id":7362137005110325270,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 210040d0-81d1-4848-9e65-ea4521e3e65e Info: {"upstream_id":7362137005110325270,"namespace":"default","id":"210040d0-81d1-4848-9e65-ea4521e3e65e","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-2347?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:31:56.384956247+08:00","start_ts":449349186122153985,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-37-gcf8946a3a","resolved_ts":449349186122153985,"checkpoint_ts":449349186122153985,"checkpoint_time":"2024-04-26 19:31:51.611"} [Fri Apr 26 19:31:56 CST 2024] <<<<<< START kafka consumer in common_1 case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache table common_1.v1 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // dir Post stage [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + ls /tmp/tidb_cdc_test/ changefeed_finish cov.changefeed_finish.28052807.out force_replicate_table sql_res.changefeed_finish.txt sql_res.force_replicate_table.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G14.tar.gz /tmp/tidb_cdc_test/changefeed_finish/tikv_down.log /tmp/tidb_cdc_test/changefeed_finish/down_pd.log /tmp/tidb_cdc_test/changefeed_finish/cdc.log /tmp/tidb_cdc_test/changefeed_finish/stdout.log /tmp/tidb_cdc_test/changefeed_finish/tidb-slow.log /tmp/tidb_cdc_test/changefeed_finish/tikv1.log /tmp/tidb_cdc_test/changefeed_finish/tidb.log /tmp/tidb_cdc_test/changefeed_finish/tidb_down.log /tmp/tidb_cdc_test/changefeed_finish/pd1.log /tmp/tidb_cdc_test/changefeed_finish/tidb_other.log /tmp/tidb_cdc_test/changefeed_finish/sync_diff_inspector.log /tmp/tidb_cdc_test/changefeed_finish/tikv3.log /tmp/tidb_cdc_test/changefeed_finish/cdc_kafka_consumer.log /tmp/tidb_cdc_test/changefeed_finish/tikv2.log /tmp/tidb_cdc_test/changefeed_finish/cdc_kafka_consumer_stdout.log /tmp/tidb_cdc_test/force_replicate_table/tikv3/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/tikv_down.log /tmp/tidb_cdc_test/force_replicate_table/down_pd.log /tmp/tidb_cdc_test/force_replicate_table/tikv2/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/tikv_down/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/cdc.log /tmp/tidb_cdc_test/force_replicate_table/stdout.log /tmp/tidb_cdc_test/force_replicate_table/tidb-slow.log /tmp/tidb_cdc_test/force_replicate_table/tikv1.log /tmp/tidb_cdc_test/force_replicate_table/tidb.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log /tmp/tidb_cdc_test/force_replicate_table/tidb_down.log /tmp/tidb_cdc_test/force_replicate_table/tikv1/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/pd1.log /tmp/tidb_cdc_test/force_replicate_table/tidb_other.log /tmp/tidb_cdc_test/force_replicate_table/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/force_replicate_table/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/force_replicate_table/tikv3.log /tmp/tidb_cdc_test/force_replicate_table/cdc_kafka_consumer.log /tmp/tidb_cdc_test/force_replicate_table/pd1/hot-region/000001.log /tmp/tidb_cdc_test/force_replicate_table/pd1/region-meta/000001.log /tmp/tidb_cdc_test/force_replicate_table/tikv2.log /tmp/tidb_cdc_test/force_replicate_table/cdc_kafka_consumer_stdout.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0002/000002.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/changefeed_finish/tikv_down.log /tmp/tidb_cdc_test/changefeed_finish/down_pd.log /tmp/tidb_cdc_test/changefeed_finish/cdc.log /tmp/tidb_cdc_test/changefeed_finish/stdout.log /tmp/tidb_cdc_test/changefeed_finish/tidb-slow.log /tmp/tidb_cdc_test/changefeed_finish/tikv1.log /tmp/tidb_cdc_test/changefeed_finish/tidb.log /tmp/tidb_cdc_test/changefeed_finish/tidb_down.log /tmp/tidb_cdc_test/changefeed_finish/pd1.log /tmp/tidb_cdc_test/changefeed_finish/tidb_other.log /tmp/tidb_cdc_test/changefeed_finish/sync_diff_inspector.log /tmp/tidb_cdc_test/changefeed_finish/tikv3.log /tmp/tidb_cdc_test/changefeed_finish/cdc_kafka_consumer.log /tmp/tidb_cdc_test/changefeed_finish/tikv2.log /tmp/tidb_cdc_test/changefeed_finish/cdc_kafka_consumer_stdout.log /tmp/tidb_cdc_test/force_replicate_table/tikv3/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/tikv_down.log check diff failed 4-th time, retry later /tmp/tidb_cdc_test/force_replicate_table/down_pd.log /tmp/tidb_cdc_test/force_replicate_table/tikv2/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/tikv_down/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/cdc.log /tmp/tidb_cdc_test/force_replicate_table/stdout.log /tmp/tidb_cdc_test/force_replicate_table/tidb-slow.log /tmp/tidb_cdc_test/force_replicate_table/tikv1.log /tmp/tidb_cdc_test/force_replicate_table/tidb.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log /tmp/tidb_cdc_test/force_replicate_table/tidb_down.log /tmp/tidb_cdc_test/force_replicate_table/tikv1/db/000005.log /tmp/tidb_cdc_test/force_replicate_table/pd1.log /tmp/tidb_cdc_test/force_replicate_table/tidb_other.log /tmp/tidb_cdc_test/force_replicate_table/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/force_replicate_table/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/force_replicate_table/tikv3.log table common_1.v1 not exists for 2-th check, retry later /tmp/tidb_cdc_test/force_replicate_table/cdc_kafka_consumer.log /tmp/tidb_cdc_test/force_replicate_table/pd1/hot-region/000001.log /tmp/tidb_cdc_test/force_replicate_table/pd1/region-meta/000001.log /tmp/tidb_cdc_test/force_replicate_table/tikv2.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b4c05c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:24805, start at 2024-04-26 19:32:00.17470569 +0800 CST m=+5.181682575 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:34:00.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:32:00.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:22:00.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /tmp/tidb_cdc_test/force_replicate_table/cdc_kafka_consumer_stdout.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/force_replicate_table/cdc_data/tmp/sorter/0002/000002.log + ls -alh log-G14.tar.gz -rw-r--r--. 1 jenkins jenkins 7.9M Apr 26 19:32 log-G14.tar.gz [Pipeline] archiveArtifacts Archiving artifacts Recording fingerprints [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed 5-th time, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' Sending interrupt signal to process Killing processes VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b4c05c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:24805, start at 2024-04-26 19:32:00.17470569 +0800 CST m=+5.181682575 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:34:00.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:32:00.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:22:00.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b4c1d80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1732-whfsf-l7x63, pid:24889, start at 2024-04-26 19:32:00.272686985 +0800 CST m=+5.226462773 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:34:00.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:32:00.246 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:22:00.246 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-13-g37701038f Edition: Community Git Commit Hash: 37701038fbd30f5eafac11c3ad180f14c6dcab9e Git Branch: HEAD UTC Build Time: 2024-04-26 10:25:26 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-26 10:30:15 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["37701038fbd30f5eafac11c3ad180f14c6dcab9e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-13-g37701038f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 table common_1.recover_and_insert not exists for 2-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714131124.8967056,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0023a7dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache {"level":"warn","ts":1714131125.1698804,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00249efc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } {"level":"warn","ts":1714131126.0764685,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022a6c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} [Pipeline] // timeout script returned exit code 143 [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } ++ stop_tidb_cluster script returned exit code 143 [Pipeline] } [Pipeline] // dir [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] // withCredentials [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // cache [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // timeout [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G01' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] // node [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' ++ stop_tidb_cluster /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh: line 1: 15239 Terminated sleep 130 script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE