INFO logs
Log path is: /opt/yuga/data/yb-data/master/logs/yb-master.INFO
Showing last 1048576 bytes of log
{ permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 10:34:49.069291 913 xcluster_safe_time_service.cc:109] ProcessTaskPeriodically: Going into idle mode due to master leader change: Illegal state (yb/master/catalog_manager.cc:2190): Not the leader. Local UUID: 55d29a1415af4e44ac228b2c8a82f866, Consensus state: current_term: 29 config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 10:34:49.069304 913 xcluster_safe_time_service.cc:109] ProcessTaskPeriodically: Going into idle mode due to master leader change: Illegal state (yb/master/catalog_manager.cc:2190): Not the leader. Local UUID: 55d29a1415af4e44ac228b2c8a82f866, Consensus state: current_term: 29 config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 10:34:49.069650 3811424 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 28.28864. Preceding OpId from leader: 29.28877. (index mismatch)
I0821 10:34:49.069667 3811424 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 28.28864. Preceding OpId from leader: 29.28877. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:34:49.069825 3811424 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 28.28864. Preceding OpId from leader: 29.28877. (index mismatch)
I0821 10:34:49.069832 3811424 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 28.28864. Preceding OpId from leader: 29.28877. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:34:49.088868 3811424 consensus_meta.cc:361] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866: Updating active role from FOLLOWER to FOLLOWER. Consensus state: current_term: 29 leader_uuid: "a5110a40aa5b46eaa183f0022d8d9063" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }, has_pending_config = 0
I0821 10:34:49.088915 3811424 raft_consensus.cc:3580] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Calling mark dirty synchronously for reason code NEW_LEADER_ELECTED
I0821 10:34:49.089673 3811424 sys_catalog.cc:447] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: SysCatalogTable state changed. Locked=1. Reason: New leader a5110a40aa5b46eaa183f0022d8d9063 elected. Latest consensus state: current_term: 29 leader_uuid: "a5110a40aa5b46eaa183f0022d8d9063" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 10:34:49.090675 3811424 sys_catalog.cc:451] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: This master's current role is: FOLLOWER
I0821 10:34:49.094102 3811424 replica_state.cc:1215] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Reset our lease: 334.584s
I0821 10:34:49.094122 3811424 replica_state.cc:1227] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Reset our ht lease: { physical: 1787308154511914 }
I0821 10:34:49.097319 3811424 raft_consensus.cc:3580] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Calling mark dirty synchronously for reason code FOLLOWER_NO_OP_COMPLETE
I0821 10:34:49.097344 3811424 sys_catalog.cc:447] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: SysCatalogTable state changed. Locked=1. Reason: Replicate of NO_OP complete on follower. Latest consensus state: current_term: 29 leader_uuid: "a5110a40aa5b46eaa183f0022d8d9063" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 10:34:49.097378 3811424 sys_catalog.cc:451] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: This master's current role is: FOLLOWER
W0821 10:34:53.884385 912 periodic.cc:174] PeriodicTimer callback delayed by 0.307s
W0821 10:34:56.473448 911 periodic.cc:174] PeriodicTimer callback delayed by 1.255s
I0821 10:34:58.867182 3811535 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:34:58.926867 911 thread.cc:844] raft_19656_worker (raft) Time spent starting thread: real 0.831s user 0.000s sys 0.027s
I0821 10:35:00.514474 3811535 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:35:02.594507 3811535 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.232s
W0821 10:35:06.381697 913 periodic.cc:174] PeriodicTimer callback delayed by 5.204s
W0821 10:35:08.114876 911 thread.cc:844] CatalogManagerBGTasks_5755_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.169s user 0.000s sys 0.029s
W0821 10:35:11.762732 910 periodic.cc:174] PeriodicTimer callback delayed by 1.681s
W0821 10:35:17.619629 911 periodic.cc:174] PeriodicTimer callback delayed by 1.484s
W0821 10:35:23.246719 910 thread.cc:844] CatalogManagerBGTasks_5756_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.701s user 0.000s sys 0.049s
W0821 10:35:23.438257 912 periodic.cc:174] PeriodicTimer callback delayed by 1.963s
W0821 10:35:29.682785 910 periodic.cc:174] PeriodicTimer callback delayed by 2.008s
W0821 10:35:25.913522 916 yb_rpc.cc:553] Connection (0x000050403a907520) client 10.0.0.2:37067 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.885s, timeout: 15.000s, now: 96213.969s, last_read_time_: 96198.084s
W0821 10:35:31.237375 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403bf19c00: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312979, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96198.075s (29.404s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96213.055s (14.424s ago), now: 96227.480s, connection: Connection (0x000050403a907de0) client 10.0.0.2:44611 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:35:37.362248 911 periodic.cc:174] PeriodicTimer callback delayed by 3.193s
W0821 10:35:38.861804 914 thread.cc:850] Master_2312_worker (Master) Time spent creating pthread: real 0.746s user 0.000s sys 0.012s
W0821 10:35:40.623932 914 thread.cc:844] Master_2312_worker (Master) Time spent starting thread: real 3.725s user 0.000s sys 0.058s
W0821 10:35:43.718165 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403dbb8380: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312976, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96198.075s (40.391s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96213.056s (25.410s ago), now: 96238.466s, connection: Connection (0x000050403a911360) client 10.0.0.2:34045 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:35:41.726001 3811554 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf19c00: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312979, state: TIMED_OUT, transfer_state: PENDING took 0.983s
W0821 10:35:44.165234 913 periodic.cc:174] PeriodicTimer callback delayed by 2.062s
W0821 10:35:49.554363 917 thread.cc:844] Master_2313_worker (Master) Time spent starting thread: real 1.148s user 0.000s sys 0.015s
W0821 10:35:50.506305 911 periodic.cc:174] PeriodicTimer callback delayed by 2.299s
W0821 10:35:51.258993 3811558 outbound_call.cc:526] RPC callback for RPC call 0x000050403dbb8380: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312976, state: TIMED_OUT, transfer_state: PENDING took 2.460s
W0821 10:35:59.784121 910 periodic.cc:174] PeriodicTimer callback delayed by 4.695s
W0821 10:36:01.583042 913 thread.cc:850] CatalogManagerBGTasks_5757_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.614s user 0.000s sys 0.008s
W0821 10:36:05.610208 913 thread.cc:844] CatalogManagerBGTasks_5757_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.054s user 0.000s sys 0.055s
W0821 10:36:08.624146 913 periodic.cc:174] PeriodicTimer callback delayed by 4.716s
I0821 10:35:02.884724 3811535 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:36:16.392220 912 periodic.cc:174] PeriodicTimer callback delayed by 2.515s
I0821 10:36:16.443619 3811535 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:36:20.702723 3811424 long_operation_tracker.cc:155] UpdateReplica took a long time: 78.615s
W0821 10:36:21.449836 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:36:22.719902 3811535 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:36:24.069314 912 periodic.cc:174] PeriodicTimer callback delayed by 2.526s
W0821 10:36:23.792435 3811424 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:40119 => 10.0.0.2:7100 (request call id 24626) took 87312ms (client timeout 3000ms).
W0821 10:36:30.832377 912 periodic.cc:174] PeriodicTimer callback delayed by 2.012s
W0821 10:36:30.166306 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:36:30.166257 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:36:32.514587 912 thread.cc:888] raft_19657_worker (raft) Time spent waiting for new thread to publish its TID: real 0.624s user 0.000s sys 0.015s
I0821 10:36:32.186034 3811585 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:36:33.727861 912 thread.cc:844] raft_19657_worker (raft) Time spent starting thread: real 2.644s user 0.000s sys 0.050s
I0821 10:36:34.343782 3811585 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:36:36.670432 3811585 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.091s
W0821 10:36:39.342959 912 periodic.cc:174] PeriodicTimer callback delayed by 4.850s
W0821 10:36:41.068955 913 thread.cc:844] CatalogManagerBGTasks_5758_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.952s user 0.000s sys 0.018s
W0821 10:36:43.959884 912 periodic.cc:174] PeriodicTimer callback delayed by 2.357s
W0821 10:36:50.835469 912 periodic.cc:174] PeriodicTimer callback delayed by 1.970s
W0821 10:36:50.985891 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 10:36:57.317626 911 periodic.cc:174] PeriodicTimer callback delayed by 2.278s
W0821 10:37:04.043347 910 periodic.cc:174] PeriodicTimer callback delayed by 2.080s
I0821 10:36:38.616901 3811585 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:37:10.438979 913 periodic.cc:174] PeriodicTimer callback delayed by 1.850s
I0821 10:37:11.764791 3811585 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:37:15.899012 910 thread.cc:850] CatalogManagerBGTasks_5759_worker (CatalogManagerBGTasks) Time spent creating pthread: real 1.198s user 0.000s sys 0.021s
W0821 10:37:16.548311 913 periodic.cc:174] PeriodicTimer callback delayed by 1.871s
W0821 10:37:17.322188 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403be26380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312980, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96287.863s (41.610s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96300.116s (29.356s ago), now: 96329.473s, connection: Connection (0x000050403a910560) client 10.0.0.2:41567 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:37:15.886628 917 yb_rpc.cc:553] Connection (0x000050403a9348e0) client 10.0.0.2:37775 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 30.927s, timeout: 15.000s, now: 96238.023s, last_read_time_: 96207.095s
W0821 10:37:21.112867 910 thread.cc:844] CatalogManagerBGTasks_5759_worker (CatalogManagerBGTasks) Time spent starting thread: real 6.387s user 0.000s sys 0.100s
W0821 10:37:25.169700 911 periodic.cc:174] PeriodicTimer callback delayed by 3.464s
W0821 10:37:26.568574 914 thread.cc:844] Master-high-pri_459_worker (Master-high-pri) Time spent starting thread: real 3.256s user 0.000s sys 0.053s
W0821 10:37:25.072073 3811613 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312980) to 10.0.0.3:7100 timed out after 3.232s
W0821 10:37:29.539899 3811613 outbound_call.cc:526] RPC callback for RPC call 0x000050403be26380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312980, state: TIMED_OUT, transfer_state: PENDING took 4.315s
W0821 10:37:33.998883 912 periodic.cc:174] PeriodicTimer callback delayed by 2.741s
I0821 10:37:33.219758 3811585 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:37:41.680253 913 periodic.cc:174] PeriodicTimer callback delayed by 3.241s
W0821 10:37:49.543658 912 periodic.cc:174] PeriodicTimer callback delayed by 2.340s
W0821 10:37:46.746709 916 yb_rpc.cc:553] Connection (0x000050403a9078a0) client 10.0.0.2:36303 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 102.826s, timeout: 15.000s, now: 96306.396s, last_read_time_: 96203.570s
I0821 10:37:53.087563 3811625 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:37:54.760578 912 thread.cc:888] raft_19658_worker (raft) Time spent waiting for new thread to publish its TID: real 0.518s user 0.000s sys 0.023s
W0821 10:37:54.760159 913 thread.cc:888] CatalogManagerBGTasks_5760_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.518s user 0.000s sys 0.023s
W0821 10:37:56.537379 912 thread.cc:844] raft_19658_worker (raft) Time spent starting thread: real 4.337s user 0.000s sys 0.061s
W0821 10:37:57.497709 913 thread.cc:844] CatalogManagerBGTasks_5760_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.893s user 0.000s sys 0.065s
I0821 10:37:55.403088 3811625 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:38:03.296435 910 periodic.cc:174] PeriodicTimer callback delayed by 9.176s
I0821 10:38:00.499809 3811625 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.033s
W0821 10:37:57.998189 917 yb_rpc.cc:553] Connection (0x000050403a911de0) client 10.0.0.2:45909 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 40.014s, timeout: 15.000s, now: 96238.023s, last_read_time_: 96198.009s
W0821 10:38:11.837761 910 periodic.cc:174] PeriodicTimer callback delayed by 3.004s
W0821 10:38:15.746145 912 periodic.cc:174] PeriodicTimer callback delayed by 0.425s
W0821 10:38:18.600878 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403da31c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312981, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96294.466s (106.848s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96310.943s (90.370s ago), now: 96401.314s, connection: Connection (0x000050403a9358a0) client 10.0.0.2:33551 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:38:23.310240 913 periodic.cc:174] PeriodicTimer callback delayed by 2.230s
W0821 10:38:24.274139 915 thread.cc:844] Master-high-pri_460_worker (Master-high-pri) Time spent starting thread: real 1.406s user 0.000s sys 0.025s
W0821 10:38:24.113278 3811655 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312981) to 10.0.0.4:7100 timed out after 3.232s
I0821 10:38:26.286157 3811655 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:38:27.772858 910 thread.cc:888] CatalogManagerBGTasks_5761_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.701s user 0.000s sys 0.030s
W0821 10:38:29.727708 910 thread.cc:844] CatalogManagerBGTasks_5761_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.817s user 0.000s sys 0.070s
W0821 10:38:30.742831 911 periodic.cc:174] PeriodicTimer callback delayed by 2.514s
W0821 10:38:30.947037 3811655 thread.cc:844] raft_19659_worker (raft) Time spent starting thread: real 0.966s user 0.000s sys 0.023s
I0821 10:38:05.845970 3811625 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:38:34.620690 3811625 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:38:34.737190 3811664 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.108s
I0821 10:38:37.277604 3811664 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:38:38.550516 911 periodic.cc:174] PeriodicTimer callback delayed by 2.543s
W0821 10:38:43.057753 910 periodic.cc:174] PeriodicTimer callback delayed by 2.637s
W0821 10:38:51.597081 911 periodic.cc:174] PeriodicTimer callback delayed by 2.890s
W0821 10:38:50.792337 3811655 outbound_call.cc:526] RPC callback for RPC call 0x000050403da31c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312981, state: TIMED_OUT, transfer_state: PENDING took 26.555s
W0821 10:38:57.559679 910 periodic.cc:174] PeriodicTimer callback delayed by 1.713s
W0821 10:39:01.326124 913 thread.cc:850] CatalogManagerBGTasks_5762_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.776s user 0.000s sys 0.010s
W0821 10:39:05.876294 913 thread.cc:844] CatalogManagerBGTasks_5762_worker (CatalogManagerBGTasks) Time spent starting thread: real 5.498s user 0.000s sys 0.059s
I0821 10:39:06.768236 3811625 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:39:07.071165 910 periodic.cc:174] PeriodicTimer callback delayed by 3.488s
W0821 10:39:13.307753 910 periodic.cc:174] PeriodicTimer callback delayed by 1.931s
I0821 10:39:18.330592 3811690 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:39:20.222175 3811690 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:39:22.134943 910 thread.cc:844] raft_19660_worker (raft) Time spent starting thread: real 2.470s user 0.000s sys 0.032s
I0821 10:39:22.653515 3811690 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.294s
W0821 10:39:28.900812 910 periodic.cc:174] PeriodicTimer callback delayed by 10.270s
W0821 10:39:33.211429 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403be92380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312983, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96365.876s (112.615s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96408.460s (70.031s ago), now: 96478.491s, connection: Connection (0x000050403a910e20) client 10.0.0.2:41277 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:39:33.521615 912 periodic.cc:174] PeriodicTimer callback delayed by 0.532s
W0821 10:39:34.315480 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312982, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96342.825s (138.678s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96421.402s (60.100s ago), now: 96481.503s, connection: Connection (0x000050403b9d3de0) client 10.0.0.2:45385 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:39:34.780699 3811703 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312983) to 10.0.0.4:7100 timed out after 3.091s
W0821 10:39:34.884040 3811705 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312982) to 10.0.0.3:7100 timed out after 3.091s
I0821 10:39:34.946785 3811703 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:39:24.258562 3811690 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:39:36.026758 3811690 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:39:37.190104 3811703 thread.cc:850] raft_19661_worker (raft) Time spent creating pthread: real 1.090s user 0.000s sys 0.017s
W0821 10:39:35.348501 3811705 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312982, state: TIMED_OUT, transfer_state: PENDING took 0.369s
W0821 10:39:37.881178 3811703 thread.cc:844] raft_19661_worker (raft) Time spent starting thread: real 1.938s user 0.001s sys 0.029s
W0821 10:39:38.446101 910 thread.cc:844] CatalogManagerBGTasks_5763_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.903s user 0.000s sys 0.024s
I0821 10:39:37.758654 3811710 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.353s
W0821 10:39:40.119176 910 periodic.cc:174] PeriodicTimer callback delayed by 1.928s
I0821 10:39:41.636452 3811690 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 10:39:42.080142 3811710 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:39:42.171358 3811703 outbound_call.cc:526] RPC callback for RPC call 0x000050403be92380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312983, state: TIMED_OUT, transfer_state: PENDING took 7.315s
W0821 10:39:46.227731 913 periodic.cc:174] PeriodicTimer callback delayed by 1.858s
W0821 10:39:49.229673 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf19c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312984, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96426.106s (65.200s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96464.048s (27.258s ago), now: 96491.306s, connection: Connection (0x000050403b9b58a0) client 10.0.0.2:41807 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:39:51.484743 3811703 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312984) to 10.0.0.3:7100 timed out after 3.033s
W0821 10:39:52.477977 910 periodic.cc:174] PeriodicTimer callback delayed by 1.583s
W0821 10:39:53.768689 3811703 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf19c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312984, state: TIMED_OUT, transfer_state: PENDING took 2.238s
I0821 10:39:56.266402 3811721 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:39:56.629463 910 thread.cc:844] raft_19662_worker (raft) Time spent starting thread: real 1.068s user 0.000s sys 0.027s
I0821 10:39:57.981451 3811721 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:40:00.715955 3811721 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.381s
W0821 10:40:03.872272 912 periodic.cc:174] PeriodicTimer callback delayed by 7.123s
W0821 10:40:08.111413 913 periodic.cc:174] PeriodicTimer callback delayed by 2.040s
W0821 10:40:11.901114 911 thread.cc:844] CatalogManagerBGTasks_5764_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.508s user 0.000s sys 0.032s
W0821 10:40:14.798743 910 periodic.cc:174] PeriodicTimer callback delayed by 2.208s
W0821 10:40:20.982499 913 periodic.cc:174] PeriodicTimer callback delayed by 2.091s
I0821 10:40:03.007162 3811721 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:40:28.608577 913 periodic.cc:174] PeriodicTimer callback delayed by 3.256s
I0821 10:40:31.440709 3811721 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:40:36.901772 911 periodic.cc:174] PeriodicTimer callback delayed by 2.973s
W0821 10:40:44.420020 911 thread.cc:844] CatalogManagerBGTasks_5765_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.613s user 0.000s sys 0.027s
W0821 10:40:44.467969 910 periodic.cc:174] PeriodicTimer callback delayed by 3.031s
I0821 10:40:48.408150 3811721 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:40:48.424840 917 tcp_stream.cc:226] { local: 10.0.0.2:7100 remote: 10.0.0.3:40119 }: Send failed: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:40:48.427572 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:40:48.428680 917 inbound_call.cc:111] 0x000050403968c800 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:40119 => 10.0.0.2:7100 (request call id 24626): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:40119 => 10.0.0.2:7100 (request call id 24626) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:40:48.482334 3811793 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)
I0821 10:40:48.482987 3811793 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:40:48.484571 3811777 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)
I0821 10:40:48.484582 3811777 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:40:48.502633 3811793 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356944.939s
I0821 10:40:48.502730 3811777 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)
I0821 10:40:48.504977 3811777 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:40:48.508204 3811777 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)
I0821 10:40:48.508375 3811777 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28885. Preceding OpId from leader: 29.28914. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:40:48.514174 3811777 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356944.927s
W0821 10:40:52.453071 912 periodic.cc:174] PeriodicTimer callback delayed by 2.391s
W0821 10:40:55.732067 910 periodic.cc:174] PeriodicTimer callback delayed by 1.673s
W0821 10:40:54.605765 3811793 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312989) to 10.0.0.4:7100 timed out after 3.381s
W0821 10:40:54.605840 3811777 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312985) to 10.0.0.4:7100 timed out after 3.033s
I0821 10:40:56.956522 3811793 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:40:56.903661 3811899 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312987) to 10.0.0.4:7100 timed out after 3.294s
I0821 10:40:57.017786 3811777 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:40:57.653566 914 thread.cc:844] Master-high-pri_465_worker (Master-high-pri) Time spent starting thread: real 1.280s user 0.000s sys 0.035s
W0821 10:40:57.791188 916 thread.cc:850] Master-high-pri_466_worker (Master-high-pri) Time spent creating pthread: real 0.672s user 0.000s sys 0.017s
W0821 10:40:57.539357 912 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33427 => 10.0.0.2:7100 (request call id 25012) took 4101ms (client timeout 3000ms).
I0821 10:40:58.346171 3811899 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:40:58.906767 910 thread.cc:844] raft_19663_worker (raft) Time spent starting thread: real 0.880s user 0.000s sys 0.049s
I0821 10:40:58.945750 3811901 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:40:59.367889 916 thread.cc:844] Master-high-pri_466_worker (Master-high-pri) Time spent starting thread: real 2.191s user 0.000s sys 0.056s
I0821 10:40:59.648479 3811903 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.507s
W0821 10:41:00.680449 3811793 thread.cc:844] raft_19664_worker (raft) Time spent starting thread: real 1.061s user 0.000s sys 0.021s
W0821 10:41:00.262053 3811777 thread.cc:844] raft_19665_worker (raft) Time spent starting thread: real 1.063s user 0.000s sys 0.022s
I0821 10:41:01.096514 3811903 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:41:01.096542 3811902 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.730s
W0821 10:41:03.265403 3811899 thread.cc:888] raft_19666_worker (raft) Time spent waiting for new thread to publish its TID: real 1.251s user 0.004s sys 0.065s
W0821 10:41:04.259586 912 periodic.cc:174] PeriodicTimer callback delayed by 5.057s
I0821 10:41:04.231290 3811902 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:41:04.237142 3811901 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:41:04.709491 3811899 thread.cc:844] raft_19666_worker (raft) Time spent starting thread: real 3.500s user 0.005s sys 0.113s
W0821 10:41:02.910832 3811777 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312985, state: TIMED_OUT, transfer_state: PENDING took 8.292s
W0821 10:41:02.837440 3811793 outbound_call.cc:526] RPC callback for RPC call 0x000050403be92380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312989, state: TIMED_OUT, transfer_state: PENDING took 8.221s
I0821 10:41:06.325484 3811901 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.067s
W0821 10:41:07.815291 913 periodic.cc:174] PeriodicTimer callback delayed by 1.269s
W0821 10:41:07.149492 3811899 outbound_call.cc:526] RPC callback for RPC call 0x000050403ce36a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312987, state: TIMED_OUT, transfer_state: PENDING took 10.242s
W0821 10:41:09.798062 913 periodic.cc:174] PeriodicTimer callback delayed by 1.855s
W0821 10:41:13.410300 910 periodic.cc:174] PeriodicTimer callback delayed by 1.365s
W0821 10:41:17.910576 912 periodic.cc:174] PeriodicTimer callback delayed by 1.395s
W0821 10:41:18.041128 913 thread.cc:844] CatalogManagerBGTasks_5766_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.096s user 0.000s sys 0.032s
W0821 10:41:22.245090 912 periodic.cc:174] PeriodicTimer callback delayed by 0.810s
W0821 10:41:23.760504 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:41:28.974982 910 periodic.cc:174] PeriodicTimer callback delayed by 2.340s
W0821 10:41:32.466120 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:41:35.190462 910 periodic.cc:174] PeriodicTimer callback delayed by 2.012s
W0821 10:41:39.774746 910 periodic.cc:174] PeriodicTimer callback delayed by 1.352s
W0821 10:41:35.982700 917 yb_rpc.cc:553] Connection (0x0000504038f8ba60) client 10.0.0.2:41565 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 39.207s, timeout: 15.000s, now: 96603.830s, last_read_time_: 96564.623s
W0821 10:41:46.371660 911 periodic.cc:174] PeriodicTimer callback delayed by 2.036s
W0821 10:41:54.984761 910 periodic.cc:174] PeriodicTimer callback delayed by 3.222s
W0821 10:42:02.601413 913 periodic.cc:174] PeriodicTimer callback delayed by 2.384s
W0821 10:42:06.708204 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:08.196130 912 periodic.cc:174] PeriodicTimer callback delayed by 1.751s
I0821 10:41:09.686020 3811901 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:42:12.786401 912 periodic.cc:174] PeriodicTimer callback delayed by 1.447s
I0821 10:42:13.913434 3811901 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:42:14.142841 3811904 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.696s
I0821 10:42:16.983454 3811904 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:42:17.213706 3811901 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:42:17.270121 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:17.270143 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:17.271278 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:20.001773 911 periodic.cc:174] PeriodicTimer callback delayed by 2.334s
W0821 10:42:23.082168 913 periodic.cc:174] PeriodicTimer callback delayed by 1.463s
I0821 10:42:25.132720 3811985 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:42:25.130496 912 thread.cc:844] CatalogManagerBGTasks_5768_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.987s user 0.000s sys 0.030s
W0821 10:42:23.549928 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:42:25.142161 3811985 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:42:25.151721 3811985 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.540s
I0821 10:42:25.154103 3811985 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:42:25.399873 3811985 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:42:25.320286 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:27.074667 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:42:27.133751 3811985 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:42:27.188078 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:29.138358 911 periodic.cc:174] PeriodicTimer callback delayed by 2.166s
I0821 10:42:29.371876 3812022 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:42:31.770668 3812022 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:42:33.333841 910 periodic.cc:174] PeriodicTimer callback delayed by 0.541s
W0821 10:42:31.269667 914 yb_rpc.cc:553] Connection (0x0000504039f4ec60) client 10.0.0.2:40925 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 67.103s, timeout: 15.000s, now: 96635.017s, last_read_time_: 96567.913s
W0821 10:42:35.393290 915 thread.cc:850] Master_2318_worker (Master) Time spent creating pthread: real 1.138s user 0.000s sys 0.021s
I0821 10:42:33.349272 3812022 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.481s
W0821 10:42:36.758884 910 periodic.cc:174] PeriodicTimer callback delayed by 0.750s
W0821 10:42:39.568357 915 thread.cc:844] Master_2318_worker (Master) Time spent starting thread: real 4.260s user 0.000s sys 0.061s
W0821 10:42:43.115768 910 periodic.cc:174] PeriodicTimer callback delayed by 2.137s
W0821 10:42:44.037384 913 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:35691 => 10.0.0.2:7100 (request call id 25014) took 10107ms (client timeout 3000ms).
W0821 10:42:45.970036 917 thread.cc:850] Master-high-pri_468_worker (Master-high-pri) Time spent creating pthread: real 0.504s user 0.000s sys 0.007s
W0821 10:42:49.811683 917 thread.cc:888] Master-high-pri_468_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 1.068s user 0.000s sys 0.017s
W0821 10:42:51.097718 911 periodic.cc:174] PeriodicTimer callback delayed by 3.037s
W0821 10:42:49.149108 915 inbound_call.cc:111] 0x000050403aed4c00 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:44615 => 10.0.0.2:7100 (request call id 8569404): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:44615 => 10.0.0.2:7100 (request call id 8569404) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:42:54.860858 917 thread.cc:844] Master-high-pri_468_worker (Master-high-pri) Time spent starting thread: real 7.646s user 0.000s sys 0.118s
W0821 10:42:57.812182 911 thread.cc:888] CatalogManagerBGTasks_5769_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.762s user 0.000s sys 0.030s
W0821 10:42:58.986632 910 periodic.cc:174] PeriodicTimer callback delayed by 3.012s
W0821 10:42:59.108444 911 thread.cc:844] CatalogManagerBGTasks_5769_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.536s user 0.000s sys 0.053s
W0821 10:42:59.700120 917 inbound_call.cc:111] 0x000050403e200000 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:35691 => 10.0.0.2:7100 (request call id 25014): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:35691 => 10.0.0.2:7100 (request call id 25014) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:43:02.870669 911 periodic.cc:174] PeriodicTimer callback delayed by 0.732s
W0821 10:43:03.121227 3812027 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:35691 => 10.0.0.2:7100 (request call id 25040) took 6145ms (client timeout 3000ms).
W0821 10:43:06.828402 912 periodic.cc:174] PeriodicTimer callback delayed by 0.611s
W0821 10:43:12.680415 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403be26380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312991, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96646.236s (45.851s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96675.116s (16.971s ago), now: 96692.087s, connection: Connection (0x000050403b9b36e0) client 10.0.0.2:32843 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:43:14.814281 910 periodic.cc:174] PeriodicTimer callback delayed by 3.223s
W0821 10:43:16.145449 3812027 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312991) to 10.0.0.4:7100 timed out after 3.067s
W0821 10:43:20.478679 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf19c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312992, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96654.775s (50.365s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96677.432s (27.708s ago), now: 96705.141s, connection: Connection (0x000050403ede8020) client 10.0.0.2:38465 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:43:19.393401 3812027 outbound_call.cc:526] RPC callback for RPC call 0x000050403be26380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312991, state: TIMED_OUT, transfer_state: PENDING took 3.092s
W0821 10:43:22.236335 910 periodic.cc:174] PeriodicTimer callback delayed by 1.925s
W0821 10:43:20.820333 917 inbound_call.cc:111] 0x000050403deb9c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:35691 => 10.0.0.2:7100 (request call id 25040): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:35691 => 10.0.0.2:7100 (request call id 25040) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:42:39.281808 3812022 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:43:22.869009 3812027 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312992) to 10.0.0.3:7100 timed out after 3.540s
I0821 10:43:27.740062 3812022 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:43:28.953178 3812023 long_operation_tracker.cc:155] UpdateReplica took a long time: 55.484s
W0821 10:43:30.492342 910 periodic.cc:174] PeriodicTimer callback delayed by 3.770s
W0821 10:43:27.945964 3812027 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf19c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312992, state: TIMED_OUT, transfer_state: PENDING took 4.887s
W0821 10:43:32.609668 912 thread.cc:850] CatalogManagerBGTasks_5770_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.519s user 0.000s sys 0.006s
W0821 10:43:28.867301 3812015 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:40945 => 10.0.0.2:7100 (request call id 8568786) took 59414ms (client timeout 29993ms).
W0821 10:43:29.215875 3812026 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:40945 => 10.0.0.2:7100 (request call id 8568819) took 57531ms (client timeout 14999ms).
W0821 10:43:34.237248 912 thread.cc:844] CatalogManagerBGTasks_5770_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.341s user 0.000s sys 0.029s
W0821 10:43:34.299329 910 periodic.cc:174] PeriodicTimer callback delayed by 2.795s
W0821 10:43:31.804337 3812023 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33427 => 10.0.0.2:7100 (request call id 25039) took 63183ms (client timeout 3000ms).
I0821 10:43:35.889212 3812022 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:43:40.587090 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403f921c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312990, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96645.783s (65.344s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96680.074s (31.054s ago), now: 96711.128s, connection: Connection (0x000050403bb57de0) client 10.0.0.2:37051 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:43:41.238920 910 periodic.cc:174] PeriodicTimer callback delayed by 2.653s
W0821 10:43:42.930500 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 10:43:44.239073 3812023 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312990) to 10.0.0.3:7100 timed out after 3.067s
I0821 10:43:46.724298 3812023 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:43:46.914988 915 inbound_call.cc:111] 0x000050403ca5d800 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:40945 => 10.0.0.2:7100 (request call id 8568786): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:40945 => 10.0.0.2:7100 (request call id 8568786) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:43:48.833966 913 periodic.cc:174] PeriodicTimer callback delayed by 2.283s
W0821 10:43:52.279744 3812023 thread.cc:850] raft_19669_worker (raft) Time spent creating pthread: real 0.555s user 0.000s sys 0.010s
W0821 10:43:53.945966 913 thread.cc:844] raft_19670_worker (raft) Time spent starting thread: real 1.646s user 0.000s sys 0.021s
W0821 10:43:54.442420 3812023 thread.cc:844] raft_19669_worker (raft) Time spent starting thread: real 4.003s user 0.001s sys 0.055s
I0821 10:43:53.460041 3812053 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:43:54.257997 3812052 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.819s
I0821 10:43:58.628773 3812053 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:43:58.602377 3812052 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:44:01.318338 912 periodic.cc:174] PeriodicTimer callback delayed by 7.766s
I0821 10:44:00.730744 3812053 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 5.201s
W0821 10:43:58.340036 3812023 outbound_call.cc:526] RPC callback for RPC call 0x000050403f921c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312990, state: TIMED_OUT, transfer_state: PENDING took 13.852s
W0821 10:44:04.057185 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x00005040379eea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312993, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96656.195s (74.922s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96698.573s (32.544s ago), now: 96731.117s, connection: Connection (0x000050403bbbec60) client 10.0.0.2:33935 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:44:04.271499 913 periodic.cc:174] PeriodicTimer callback delayed by 0.857s
I0821 10:44:02.092358 3812053 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:44:05.329700 3812053 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:44:05.332773 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403d99d880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312995, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96728.491s (23.753s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96746.894s (5.351s ago), now: 96752.245s, connection: Connection (0x0000504038f3e560) client 10.0.0.2:34491 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:44:05.332909 3812066 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312995) to 10.0.0.4:7100 timed out after 4.481s
W0821 10:44:05.327915 3812023 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312993) to 10.0.0.4:7100 timed out after 3.540s
I0821 10:44:05.333000 3812023 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:44:05.333554 3812067 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.936s
I0821 10:44:05.333580 3812067 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:44:05.336818 3812053 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:44:05.333869 916 inbound_call.cc:111] 0x000050403e5f6c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33427 => 10.0.0.2:7100 (request call id 25039): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33427 => 10.0.0.2:7100 (request call id 25039) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:44:05.414667 3812066 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356748.024s
I0821 10:44:05.416105 3812066 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:44:05.416225 3812066 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356748.023s
I0821 10:44:05.416356 3812053 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 5.214s
I0821 10:44:05.416371 3812053 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:44:07.576920 913 periodic.cc:174] PeriodicTimer callback delayed by 1.097s
W0821 10:44:10.022867 912 periodic.cc:174] PeriodicTimer callback delayed by 0.182s
W0821 10:44:13.743345 912 periodic.cc:174] PeriodicTimer callback delayed by 1.463s
W0821 10:44:13.554085 3812066 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312997) to 10.0.0.4:7100 timed out after 5.201s
I0821 10:44:16.605695 3812066 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:44:19.519742 912 thread.cc:888] raft_19672_worker (raft) Time spent waiting for new thread to publish its TID: real 1.490s user 0.000s sys 0.063s
I0821 10:44:19.610769 3812239 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:44:20.970192 912 thread.cc:844] raft_19672_worker (raft) Time spent starting thread: real 3.656s user 0.000s sys 0.097s
I0821 10:44:21.326467 3812239 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:44:21.844377 3812066 thread.cc:844] raft_19673_worker (raft) Time spent starting thread: real 0.915s user 0.000s sys 0.031s
I0821 10:44:22.871692 3812239 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.931s
W0821 10:44:27.701663 911 periodic.cc:174] PeriodicTimer callback delayed by 8.899s
W0821 10:44:27.080762 3812066 outbound_call.cc:526] RPC callback for RPC call 0x000050403e2ac700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312997, state: TIMED_OUT, transfer_state: PENDING took 13.119s
W0821 10:44:33.123006 910 periodic.cc:174] PeriodicTimer callback delayed by 3.162s
W0821 10:44:40.895076 912 periodic.cc:174] PeriodicTimer callback delayed by 2.315s
W0821 10:44:43.923770 911 thread.cc:888] CatalogManagerBGTasks_5772_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 2.816s user 0.000s sys 0.115s
W0821 10:44:43.641634 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:44:46.275563 911 thread.cc:844] CatalogManagerBGTasks_5772_worker (CatalogManagerBGTasks) Time spent starting thread: real 6.513s user 0.000s sys 0.178s
W0821 10:44:47.686687 910 periodic.cc:174] PeriodicTimer callback delayed by 2.108s
W0821 10:44:54.400422 912 periodic.cc:174] PeriodicTimer callback delayed by 2.092s
W0821 10:44:54.857609 917 yb_rpc.cc:553] Connection (0x000050403b12e560) client 10.0.0.2:34237 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 30.242s, timeout: 15.000s, now: 96799.535s, last_read_time_: 96769.293s
W0821 10:44:57.583297 914 yb_rpc.cc:553] Connection (0x000050403b93e020) client 10.0.0.2:44479 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 19.182s, timeout: 15.000s, now: 96773.595s, last_read_time_: 96754.412s
W0821 10:45:00.262063 910 periodic.cc:174] PeriodicTimer callback delayed by 1.499s
W0821 10:45:02.772409 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:06.257033 911 periodic.cc:174] PeriodicTimer callback delayed by 2.177s
W0821 10:45:14.062886 913 periodic.cc:174] PeriodicTimer callback delayed by 2.960s
W0821 10:45:12.765132 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:21.649947 912 periodic.cc:174] PeriodicTimer callback delayed by 2.997s
W0821 10:45:22.379604 911 thread.cc:844] CatalogManagerBGTasks_5773_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.120s user 0.000s sys 0.022s
W0821 10:45:22.512857 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:27.748555 913 periodic.cc:174] PeriodicTimer callback delayed by 1.789s
W0821 10:45:25.915861 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:28.407764 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:35.568682 912 periodic.cc:174] PeriodicTimer callback delayed by 2.215s
W0821 10:45:35.812815 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:39.226568 915 thread.cc:850] Master-high-pri_470_worker (Master-high-pri) Time spent creating pthread: real 1.077s user 0.000s sys 0.022s
W0821 10:45:42.886236 911 periodic.cc:174] PeriodicTimer callback delayed by 3.478s
W0821 10:45:42.991011 915 thread.cc:844] Master-high-pri_470_worker (Master-high-pri) Time spent starting thread: real 5.324s user 0.000s sys 0.089s
W0821 10:45:38.020272 913 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33797 => 10.0.0.2:7100 (request call id 25258) took 9971ms (client timeout 3000ms).
W0821 10:45:46.566432 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:49.567567 910 periodic.cc:174] PeriodicTimer callback delayed by 2.179s
W0821 10:45:46.795519 3812263 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38659 => 10.0.0.2:7100 (request call id 25269) took 4110ms (client timeout 3000ms).
I0821 10:44:25.677578 3812239 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:45:55.520763 3812239 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:45:55.526887 3812242 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.000s
W0821 10:45:57.283015 910 thread.cc:844] CatalogManagerBGTasks_5774_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.596s user 0.000s sys 0.041s
W0821 10:45:57.555735 912 periodic.cc:174] PeriodicTimer callback delayed by 3.100s
I0821 10:45:59.711673 3812242 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:45:58.907050 915 inbound_call.cc:111] 0x000050403cbce000 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33797 => 10.0.0.2:7100 (request call id 25258): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33797 => 10.0.0.2:7100 (request call id 25258) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:45:57.046746 3812263 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44907 => 10.0.0.2:7100 (request call id 25260) took 6220ms (client timeout 3000ms).
I0821 10:46:01.114416 3812239 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:46:02.539752 916 thread.cc:888] Master-high-pri_471_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.880s user 0.000s sys 0.045s
W0821 10:46:04.010684 910 periodic.cc:174] PeriodicTimer callback delayed by 2.218s
W0821 10:46:05.267295 916 thread.cc:844] Master-high-pri_471_worker (Master-high-pri) Time spent starting thread: real 3.960s user 0.000s sys 0.076s
W0821 10:46:02.605261 913 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44907 => 10.0.0.2:7100 (request call id 25280) took 6098ms (client timeout 3000ms).
W0821 10:46:09.006471 910 periodic.cc:174] PeriodicTimer callback delayed by 3.802s
W0821 10:46:09.784801 916 inbound_call.cc:111] 0x000050403ca3dc00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38659 => 10.0.0.2:7100 (request call id 25269): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38659 => 10.0.0.2:7100 (request call id 25269) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:46:15.599656 913 periodic.cc:174] PeriodicTimer callback delayed by 2.298s
W0821 10:46:20.236136 913 thread.cc:844] raft_19674_worker (raft) Time spent starting thread: real 1.615s user 0.000s sys 0.031s
I0821 10:46:19.717113 3812291 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:46:21.040059 3812291 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:46:23.314548 3812291 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.036s
W0821 10:46:26.203738 913 periodic.cc:174] PeriodicTimer callback delayed by 5.861s
W0821 10:46:32.150341 913 thread.cc:850] CatalogManagerBGTasks_5775_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.642s user 0.000s sys 0.011s
W0821 10:46:34.848119 910 periodic.cc:174] PeriodicTimer callback delayed by 3.170s
W0821 10:46:33.089658 916 inbound_call.cc:111] 0x000050403e3f3400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44907 => 10.0.0.2:7100 (request call id 25260): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44907 => 10.0.0.2:7100 (request call id 25260) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:46:36.965052 913 thread.cc:844] CatalogManagerBGTasks_5775_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.894s user 0.000s sys 0.061s
W0821 10:46:43.041603 912 periodic.cc:174] PeriodicTimer callback delayed by 2.433s
I0821 10:46:29.142243 3812291 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:46:47.297462 3812291 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:46:47.297536 3812291 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:46:49.086580 913 periodic.cc:174] PeriodicTimer callback delayed by 1.818s
W0821 10:46:54.952371 913 thread.cc:888] raft_19675_worker (raft) Time spent waiting for new thread to publish its TID: real 3.281s user 0.000s sys 0.122s
I0821 10:46:54.961622 3812366 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:46:56.872298 913 thread.cc:844] raft_19675_worker (raft) Time spent starting thread: real 5.369s user 0.000s sys 0.157s
I0821 10:46:57.735599 3812366 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:47:03.404554 911 periodic.cc:174] PeriodicTimer callback delayed by 9.856s
I0821 10:47:00.998579 3812366 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.198s
W0821 10:47:08.152976 910 periodic.cc:174] PeriodicTimer callback delayed by 1.115s
W0821 10:47:07.806259 911 thread.cc:844] CatalogManagerBGTasks_5776_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.061s user 0.000s sys 0.020s
W0821 10:47:14.417711 912 periodic.cc:174] PeriodicTimer callback delayed by 2.211s
W0821 10:47:15.676940 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bff8700: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.2:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312999, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96870.948s (59.744s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96920.177s (10.515s ago), now: 96930.692s, connection: Connection (0x000050403b957520) client 10.0.0.2:35953 => 10.0.0.2:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:47:23.092854 912 periodic.cc:174] PeriodicTimer callback delayed by 2.961s
W0821 10:47:23.627086 915 thread.cc:888] Master_2320_worker (Master) Time spent waiting for new thread to publish its TID: real 0.519s user 0.000s sys 0.025s
W0821 10:47:26.192793 915 thread.cc:844] Master_2320_worker (Master) Time spent starting thread: real 3.776s user 0.000s sys 0.085s
W0821 10:47:26.206562 3812264 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.2:7100]) id (55d29a1415af4e44ac228b2c8a82f866): GetMasterRegistration RPC (request call id 312999) to 10.0.0.2:7100 timed out after 1.500s
W0821 10:47:30.320192 911 periodic.cc:174] PeriodicTimer callback delayed by 2.378s
W0821 10:47:34.169790 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312998, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96866.697s (89.988s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96921.241s (35.445s ago), now: 96956.686s, connection: Connection (0x0000504038ef6560) client 10.0.0.2:32893 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:47:38.667263 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf27500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313000, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96872.227s (85.759s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96936.061s (21.924s ago), now: 96957.986s, connection: Connection (0x000050403b99f6e0) client 10.0.0.2:44763 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:47:40.482254 912 periodic.cc:174] PeriodicTimer callback delayed by 4.804s
W0821 10:47:43.990365 911 thread.cc:850] CatalogManagerBGTasks_5777_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.719s user 0.000s sys 0.014s
W0821 10:47:44.179714 915 thread.cc:844] Master-high-pri_472_worker (Master-high-pri) Time spent starting thread: real 2.624s user 0.000s sys 0.034s
W0821 10:47:44.205763 911 thread.cc:844] CatalogManagerBGTasks_5777_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.823s user 0.000s sys 0.028s
W0821 10:47:44.086144 3812376 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 312998) to 10.0.0.3:7100 timed out after 4.931s
I0821 10:47:04.374717 3812366 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:47:44.230863 3812379 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313000) to 10.0.0.4:7100 timed out after 4.931s
W0821 10:47:44.289988 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x00005040379eea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313001, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 96916.303s (56.942s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 96937.735s (35.510s ago), now: 96973.246s, connection: Connection (0x000050403b9a1a60) client 10.0.0.2:42683 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:47:44.233664 3812376 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 312998, state: TIMED_OUT, transfer_state: PENDING took 0.167s
I0821 10:47:44.292268 3812366 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:47:44.293669 3812382 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313001) to 10.0.0.3:7100 timed out after 4.036s
I0821 10:47:44.294683 3812366 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 10:47:44.314703 3812379 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:47:44.314790 3812379 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf27500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313000, state: TIMED_OUT, transfer_state: PENDING took 0.089s
I0821 10:47:44.315230 3812366 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.244s
I0821 10:47:44.315263 3812366 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:47:44.368361 3812382 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)
I0821 10:47:44.368450 3812382 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:47:44.371811 3812379 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)
I0821 10:47:44.371827 3812379 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:47:44.372999 3812379 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)
I0821 10:47:44.373015 3812379 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:47:44.373196 3812382 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)
I0821 10:47:44.373203 3812382 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28951. Preceding OpId from leader: 29.28965. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:47:44.384312 3812379 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356529.055s
W0821 10:47:45.898350 3812264 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.4:7100]) id (64fedbe39ec148ab8f9e982b6d19f6d2): GetMasterRegistration RPC (request call id 313006) to 10.0.0.4:7100 timed out after 1.500s
W0821 10:47:45.901136 3812264 master-path-handlers.cc:237] Not found (yb/master/master-path-handlers.cc:277): Unable to locate leader master to redirect this request: /
W0821 10:47:47.584573 3812382 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313005) to 10.0.0.4:7100 timed out after 3.198s
I0821 10:47:47.596505 3812382 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:47:47.608505 3812514 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.295s
I0821 10:47:47.632431 3812514 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:47:48.713224 3812382 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313002) to 10.0.0.4:7100 timed out after 4.036s
W0821 10:47:48.883366 912 periodic.cc:174] PeriodicTimer callback delayed by 2.482s
I0821 10:47:48.843219 3812382 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:47:52.464466 910 periodic.cc:174] PeriodicTimer callback delayed by 1.497s
W0821 10:47:52.781152 3812382 thread.cc:844] raft_19677_worker (raft) Time spent starting thread: real 1.456s user 0.001s sys 0.018s
I0821 10:47:51.937971 3812529 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.708s
I0821 10:47:54.877421 3812529 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:47:52.611699 3812379 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46391 => 10.0.0.2:7100 (request call id 25463) took 3621ms (client timeout 3000ms).
W0821 10:47:57.673558 910 thread.cc:844] raft_19678_worker (raft) Time spent starting thread: real 1.505s user 0.000s sys 0.029s
I0821 10:47:57.590761 3812532 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:47:57.358793 3812382 outbound_call.cc:526] RPC callback for RPC call 0x000050403e5dae00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313002, state: TIMED_OUT, transfer_state: PENDING took 8.544s
I0821 10:48:00.708806 3812532 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:48:03.699366 3812532 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 7.822s
W0821 10:48:05.166239 912 periodic.cc:174] PeriodicTimer callback delayed by 8.052s
I0821 10:48:07.309723 3812532 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:48:11.105265 912 periodic.cc:174] PeriodicTimer callback delayed by 1.628s
I0821 10:48:12.405716 3812532 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:48:13.887518 913 periodic.cc:174] PeriodicTimer callback delayed by 0.530s
W0821 10:48:13.731892 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:48:15.340222 3812532 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:48:14.309273 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:15.359297 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:15.358043 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:15.824206 911 periodic.cc:174] PeriodicTimer callback delayed by 1.317s
W0821 10:48:21.920265 913 periodic.cc:174] PeriodicTimer callback delayed by 1.806s
W0821 10:48:19.428947 914 yb_rpc.cc:553] Connection (0x000050403b89d360) client 10.0.0.2:36303 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 16.243s, timeout: 15.000s, now: 96989.608s, last_read_time_: 96973.365s
W0821 10:48:23.141520 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:25.157740 913 thread.cc:850] raft_19679_worker (raft) Time spent creating pthread: real 0.523s user 0.000s sys 0.009s
W0821 10:48:27.255419 913 thread.cc:844] raft_19679_worker (raft) Time spent starting thread: real 2.580s user 0.000s sys 0.036s
I0821 10:48:26.678409 3812584 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:48:29.426273 3812584 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:48:30.945964 3812584 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.258s
W0821 10:48:31.716588 916 thread.cc:844] Master-high-pri_475_worker (Master-high-pri) Time spent starting thread: real 1.052s user 0.000s sys 0.025s
W0821 10:48:32.269462 911 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39715 => 10.0.0.2:7100 (request call id 25465) took 4466ms (client timeout 3000ms).
W0821 10:48:33.897013 913 periodic.cc:174] PeriodicTimer callback delayed by 7.586s
W0821 10:48:36.030196 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:37.720096 913 periodic.cc:174] PeriodicTimer callback delayed by 1.566s
I0821 10:48:32.685086 3812584 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:48:37.885998 917 yb_rpc.cc:553] Connection (0x0000504038e3ae20) client 10.0.0.2:34737 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 19.329s, timeout: 15.000s, now: 97001.590s, last_read_time_: 96982.260s
I0821 10:48:37.899215 3812584 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:48:41.020167 912 periodic.cc:174] PeriodicTimer callback delayed by 1.036s
W0821 10:48:42.048192 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:48:43.413492 3812584 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:48:44.314801 911 periodic.cc:174] PeriodicTimer callback delayed by 2.317s
W0821 10:48:47.569473 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:48.958829 910 thread.cc:850] CatalogManagerBGTasks_5779_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.810s user 0.000s sys 0.014s
W0821 10:48:50.160588 910 thread.cc:844] CatalogManagerBGTasks_5779_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.367s user 0.000s sys 0.043s
W0821 10:48:50.064504 911 periodic.cc:174] PeriodicTimer callback delayed by 1.885s
W0821 10:48:50.153467 916 inbound_call.cc:111] 0x000050403bd87c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39715 => 10.0.0.2:7100 (request call id 25465): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39715 => 10.0.0.2:7100 (request call id 25465) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:51.629472 911 thread.cc:888] raft_19680_worker (raft) Time spent waiting for new thread to publish its TID: real 0.854s user 0.000s sys 0.077s
W0821 10:48:51.878939 915 thread.cc:888] Master-high-pri_476_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.929s user 0.000s sys 0.084s
W0821 10:48:53.886133 911 thread.cc:844] raft_19680_worker (raft) Time spent starting thread: real 3.332s user 0.000s sys 0.098s
W0821 10:48:54.136143 915 thread.cc:844] Master-high-pri_476_worker (Master-high-pri) Time spent starting thread: real 3.812s user 0.000s sys 0.109s
I0821 10:48:51.879658 3812597 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:48:54.431653 3812597 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:48:53.506506 3812599 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46193 => 10.0.0.2:7100 (request call id 25473) took 5832ms (client timeout 3000ms).
I0821 10:48:57.428819 3812597 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 7.062s
W0821 10:48:57.295861 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:48:58.449950 910 periodic.cc:174] PeriodicTimer callback delayed by 4.718s
W0821 10:49:01.500910 915 inbound_call.cc:111] 0x000050403e5b1800 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46193 => 10.0.0.2:7100 (request call id 25473): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46193 => 10.0.0.2:7100 (request call id 25473) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:49:04.711800 913 periodic.cc:174] PeriodicTimer callback delayed by 1.483s
W0821 10:49:04.694898 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:49:09.400666 912 periodic.cc:174] PeriodicTimer callback delayed by 2.628s
W0821 10:49:12.722293 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403be92a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313008, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97004.365s (47.000s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97035.160s (16.204s ago), now: 97051.365s, connection: Connection (0x000050403b076020) client 10.0.0.2:34725 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:49:16.794982 914 thread.cc:844] Master_2323_worker (Master) Time spent starting thread: real 1.659s user 0.000s sys 0.019s
W0821 10:49:17.600273 911 periodic.cc:174] PeriodicTimer callback delayed by 2.565s
W0821 10:49:20.786324 916 thread.cc:844] Master-high-pri_477_worker (Master-high-pri) Time spent starting thread: real 1.888s user 0.000s sys 0.025s
W0821 10:49:20.344223 3812617 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313008) to 10.0.0.4:7100 timed out after 7.822s
W0821 10:49:24.650100 910 thread.cc:888] CatalogManagerBGTasks_5780_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.552s user 0.000s sys 0.010s
W0821 10:49:24.966209 911 periodic.cc:174] PeriodicTimer callback delayed by 2.458s
W0821 10:49:27.012176 910 thread.cc:844] CatalogManagerBGTasks_5780_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.964s user 0.000s sys 0.048s
I0821 10:49:00.438858 3812597 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:49:27.016875 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:49:24.494762 3812617 outbound_call.cc:526] RPC callback for RPC call 0x000050403be92a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313008, state: TIMED_OUT, transfer_state: PENDING took 4.048s
W0821 10:49:27.669773 911 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43127 => 10.0.0.2:7100 (request call id 25467) took 4657ms (client timeout 3000ms).
I0821 10:49:28.807965 3812597 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:49:29.090593 914 thread.cc:844] Master-high-pri_478_worker (Master-high-pri) Time spent starting thread: real 1.187s user 0.000s sys 0.016s
I0821 10:49:30.805986 3812616 yb_rpc.cc:395] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:38225 => 10.0.0.2:7100 (request call id 8588310) took 20807ms.
W0821 10:49:31.143589 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403ddf2e00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313007, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97002.966s (74.134s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97027.940s (49.159s ago), now: 97077.100s, connection: Connection (0x000050403bbf9c20) client 10.0.0.2:40851 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 10:49:31.140232 3812597 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:49:31.146107 3812624 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313007) to 10.0.0.3:7100 timed out after 7.822s
I0821 10:49:31.171342 3812624 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:49:32.044596 911 periodic.cc:174] PeriodicTimer callback delayed by 2.076s
W0821 10:49:33.062628 914 inbound_call.cc:111] 0x000050403875a400 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:38225 => 10.0.0.2:7100 (request call id 8588310): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:38225 => 10.0.0.2:7100 (request call id 8588310) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:49:33.296924 3812643 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 8.126s
W0821 10:49:35.207101 911 thread.cc:850] raft_19682_worker (raft) Time spent creating pthread: real 0.563s user 0.000s sys 0.009s
I0821 10:49:36.059947 3812644 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:49:37.254621 911 thread.cc:844] raft_19682_worker (raft) Time spent starting thread: real 2.686s user 0.000s sys 0.046s
I0821 10:49:35.425235 3812643 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:49:35.729260 3812624 outbound_call.cc:526] RPC callback for RPC call 0x000050403ddf2e00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313007, state: TIMED_OUT, transfer_state: PENDING took 4.536s
I0821 10:49:37.382587 3812644 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:49:39.301951 3812644 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.364s
W0821 10:49:42.358384 911 periodic.cc:174] PeriodicTimer callback delayed by 5.636s
W0821 10:49:43.075312 914 inbound_call.cc:111] 0x000050403bd9c000 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43127 => 10.0.0.2:7100 (request call id 25467): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43127 => 10.0.0.2:7100 (request call id 25467) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:49:46.141736 911 periodic.cc:174] PeriodicTimer callback delayed by 1.582s
W0821 10:49:49.510995 910 periodic.cc:174] PeriodicTimer callback delayed by 1.750s
I0821 10:49:41.365660 3812644 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:49:49.565596 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313009, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97028.555s (63.502s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97053.624s (38.434s ago), now: 97092.058s, connection: Connection (0x000050403bbe4fe0) client 10.0.0.2:44277 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:49:49.572654 3812624 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313009) to 10.0.0.3:7100 timed out after 9.258s
I0821 10:49:50.459894 3812644 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:49:51.596128 3812617 long_operation_tracker.cc:155] UpdateReplica took a long time: 12.743s
W0821 10:49:50.528962 3812624 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313009, state: TIMED_OUT, transfer_state: PENDING took 0.906s
W0821 10:49:53.634039 913 periodic.cc:174] PeriodicTimer callback delayed by 0.457s
I0821 10:49:54.457520 3812644 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:49:53.354912 3812617 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46391 => 10.0.0.2:7100 (request call id 25479) took 20371ms (client timeout 3000ms).
W0821 10:49:55.398494 913 periodic.cc:174] PeriodicTimer callback delayed by 1.488s
W0821 10:49:57.843134 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313010, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97034.467s (64.101s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97089.405s (9.163s ago), now: 97098.568s, connection: Connection (0x000050403b0768e0) client 10.0.0.2:41397 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:50:00.813112 911 periodic.cc:174] PeriodicTimer callback delayed by 1.655s
W0821 10:50:01.844224 913 thread.cc:844] CatalogManagerBGTasks_5781_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.410s user 0.000s sys 0.025s
W0821 10:50:01.775509 3812617 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313010) to 10.0.0.4:7100 timed out after 9.258s
I0821 10:50:02.048799 3812644 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:50:02.071861 3812617 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:50:03.646899 3812644 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:50:03.567644 916 inbound_call.cc:111] 0x000050403bd87c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46391 => 10.0.0.2:7100 (request call id 25479): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46391 => 10.0.0.2:7100 (request call id 25479) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:50:05.398456 3812644 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.181s
W0821 10:50:06.462877 3812617 thread.cc:844] raft_19683_worker (raft) Time spent starting thread: real 1.602s user 0.002s sys 0.029s
W0821 10:50:08.665220 911 periodic.cc:174] PeriodicTimer callback delayed by 3.297s
W0821 10:50:14.524129 911 periodic.cc:174] PeriodicTimer callback delayed by 3.641s
W0821 10:50:19.515611 912 periodic.cc:174] PeriodicTimer callback delayed by 2.648s
W0821 10:50:17.094591 3812617 outbound_call.cc:526] RPC callback for RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313010, state: TIMED_OUT, transfer_state: PENDING took 15.288s
W0821 10:50:22.666599 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x00005040379eea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313011, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97078.853s (38.784s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97100.648s (16.988s ago), now: 97117.637s, connection: Connection (0x000050403bbf4e20) client 10.0.0.2:32827 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:50:24.364171 3812617 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313011) to 10.0.0.3:7100 timed out after 7.062s
W0821 10:50:25.515383 913 periodic.cc:174] PeriodicTimer callback delayed by 1.652s
I0821 10:50:07.902734 3812644 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:50:28.610287 3812644 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:50:28.614853 3812682 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.446s
I0821 10:50:28.623121 3812682 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:50:28.619771 3812644 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:50:28.651640 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403bf27500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313012, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97080.155s (57.347s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97134.030s (3.472s ago), now: 97137.503s, connection: Connection (0x000050403bb99de0) client 10.0.0.2:36637 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:50:28.705677 3812617 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313012) to 10.0.0.4:7100 timed out after 7.062s
I0821 10:50:28.722855 3812617 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:50:28.731952 3812644 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.047s
I0821 10:50:28.735654 3812644 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:50:28.741285 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403be92a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313013, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97101.201s (36.511s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97136.305s (1.406s ago), now: 97137.712s, connection: Connection (0x000050403bb98aa0) client 10.0.0.2:36075 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:50:28.744553 3812617 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313013) to 10.0.0.3:7100 timed out after 6.364s
I0821 10:50:28.791678 3812617 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)
I0821 10:50:28.791739 3812617 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:50:28.798638 3812729 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)
I0821 10:50:28.798660 3812729 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:50:28.800387 3812729 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356364.638s
I0821 10:50:28.816422 3812729 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)
I0821 10:50:28.816442 3812729 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 10:50:28.818014 3812729 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)
I0821 10:50:28.818032 3812729 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.28969. Preceding OpId from leader: 29.29000. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 10:50:30.212893 910 periodic.cc:174] PeriodicTimer callback delayed by 1.194s
W0821 10:50:32.558333 912 periodic.cc:174] PeriodicTimer callback delayed by 0.083s
W0821 10:50:34.605316 911 thread.cc:844] CatalogManagerBGTasks_5782_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.674s user 0.000s sys 0.019s
W0821 10:50:35.911964 913 periodic.cc:174] PeriodicTimer callback delayed by 1.588s
I0821 10:50:39.357471 3812828 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:50:40.060161 913 thread.cc:844] raft_19684_worker (raft) Time spent starting thread: real 1.324s user 0.000s sys 0.042s
I0821 10:50:40.936632 3812828 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:50:41.054419 3812729 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:41339 => 10.0.0.2:7100 (request call id 25691) took 7128ms (client timeout 3000ms).
I0821 10:50:43.716886 3812828 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.316s
W0821 10:50:45.054814 912 periodic.cc:174] PeriodicTimer callback delayed by 5.722s
W0821 10:50:51.038347 911 periodic.cc:174] PeriodicTimer callback delayed by 1.433s
W0821 10:50:54.688916 911 periodic.cc:174] PeriodicTimer callback delayed by 1.404s
W0821 10:50:57.645236 913 periodic.cc:174] PeriodicTimer callback delayed by 1.307s
W0821 10:51:01.271121 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313014, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97104.840s (57.532s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97144.129s (18.243s ago), now: 97162.372s, connection: Connection (0x000050403b06c1e0) client 10.0.0.2:45217 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:51:02.173619 911 periodic.cc:174] PeriodicTimer callback delayed by 1.290s
W0821 10:51:05.962702 915 thread.cc:844] Master-high-pri_480_worker (Master-high-pri) Time spent starting thread: real 1.030s user 0.000s sys 0.020s
W0821 10:51:05.708299 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:51:05.930382 3812844 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313014) to 10.0.0.4:7100 timed out after 6.364s
W0821 10:51:07.374855 912 periodic.cc:174] PeriodicTimer callback delayed by 1.470s
W0821 10:51:07.770025 911 thread.cc:844] CatalogManagerBGTasks_5783_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.820s user 0.000s sys 0.020s
I0821 10:51:08.262997 3812844 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:51:11.721058 3812844 thread.cc:888] raft_19685_worker (raft) Time spent waiting for new thread to publish its TID: real 0.520s user 0.003s sys 0.028s
W0821 10:51:12.630134 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:51:13.221902 912 periodic.cc:174] PeriodicTimer callback delayed by 1.495s
W0821 10:51:13.417579 3812844 thread.cc:844] raft_19685_worker (raft) Time spent starting thread: real 2.751s user 0.006s sys 0.052s
W0821 10:51:20.386648 913 periodic.cc:174] PeriodicTimer callback delayed by 1.866s
W0821 10:51:17.938434 3812844 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313014, state: TIMED_OUT, transfer_state: PENDING took 11.786s
I0821 10:50:46.506366 3812828 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:51:25.154505 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:51:26.870766 913 periodic.cc:174] PeriodicTimer callback delayed by 1.652s
W0821 10:51:27.849242 917 thread.cc:888] Master_2328_worker (Master) Time spent waiting for new thread to publish its TID: real 0.863s user 0.000s sys 0.035s
W0821 10:51:27.850379 917 thread.cc:844] Master_2328_worker (Master) Time spent starting thread: real 1.555s user 0.000s sys 0.046s
I0821 10:51:26.855970 3812828 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:51:26.893507 3812850 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 5.545s
I0821 10:51:28.284073 3812850 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:51:29.805727 3812828 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:51:33.181979 912 periodic.cc:174] PeriodicTimer callback delayed by 1.782s
W0821 10:51:31.227667 3812858 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:45301 => 10.0.0.2:7100 (request call id 8597236) took 6889ms (client timeout 8635ms).
W0821 10:51:34.870167 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:51:35.620383 911 periodic.cc:174] PeriodicTimer callback delayed by 1.780s
W0821 10:51:40.017295 911 thread.cc:888] raft_19686_worker (raft) Time spent waiting for new thread to publish its TID: real 0.808s user 0.000s sys 0.027s
I0821 10:51:39.815809 3812866 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:51:41.802212 911 thread.cc:844] raft_19686_worker (raft) Time spent starting thread: real 3.790s user 0.000s sys 0.067s
W0821 10:51:41.410310 913 thread.cc:850] CatalogManagerBGTasks_5784_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.989s user 0.000s sys 0.014s
W0821 10:51:40.018306 914 yb_rpc.cc:553] Connection (0x000050403bb98aa0) client 10.0.0.2:36075 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.658s, timeout: 15.000s, now: 97170.613s, last_read_time_: 97144.954s
I0821 10:51:42.118383 3812866 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:51:45.081139 913 thread.cc:844] CatalogManagerBGTasks_5784_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.132s user 0.000s sys 0.054s
I0821 10:51:44.637900 3812866 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.312s
I0821 10:51:47.835047 911 pg_client_service.cc:2621] Fetching set of database oids
W0821 10:51:48.693876 910 periodic.cc:174] PeriodicTimer callback delayed by 7.889s
W0821 10:51:54.714753 910 periodic.cc:174] PeriodicTimer callback delayed by 3.754s
W0821 10:51:54.909649 917 thread.cc:888] Master-high-pri_481_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 1.425s user 0.000s sys 0.048s
W0821 10:51:57.086994 917 thread.cc:844] Master-high-pri_481_worker (Master-high-pri) Time spent starting thread: real 5.580s user 0.000s sys 0.113s
W0821 10:51:56.579144 912 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34793 => 10.0.0.2:7100 (request call id 25706) took 7587ms (client timeout 3000ms).
W0821 10:51:59.311125 913 periodic.cc:174] PeriodicTimer callback delayed by 2.672s
W0821 10:51:59.632674 915 yb_rpc.cc:553] Connection (0x000050403b1b7520) client 10.0.0.2:35911 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 19.192s, timeout: 15.000s, now: 97162.202s, last_read_time_: 97143.010s
W0821 10:52:04.522686 912 periodic.cc:174] PeriodicTimer callback delayed by 1.373s
W0821 10:52:08.125471 917 inbound_call.cc:111] 0x000050403deb9c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34793 => 10.0.0.2:7100 (request call id 25706): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34793 => 10.0.0.2:7100 (request call id 25706) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:52:13.666275 910 periodic.cc:174] PeriodicTimer callback delayed by 3.712s
W0821 10:52:08.125443 3812874 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34793 => 10.0.0.2:7100 (request call id 25728) took 9088ms (client timeout 3000ms).
W0821 10:52:18.317240 914 thread.cc:844] Master_2329_worker (Master) Time spent starting thread: real 1.582s user 0.000s sys 0.014s
W0821 10:52:18.316745 913 thread.cc:844] CatalogManagerBGTasks_5785_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.582s user 0.000s sys 0.014s
W0821 10:52:20.992570 912 periodic.cc:174] PeriodicTimer callback delayed by 2.678s
W0821 10:52:24.307453 917 inbound_call.cc:111] 0x000050403cb04800 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:45301 => 10.0.0.2:7100 (request call id 8597236): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:45301 => 10.0.0.2:7100 (request call id 8597236) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:51:46.326040 3812866 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:52:28.266782 913 periodic.cc:174] PeriodicTimer callback delayed by 2.257s
I0821 10:52:29.099298 3812866 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:52:31.556102 3812879 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:34903 => 10.0.0.2:7100 (request call id 8596620) took 19331ms (client timeout 24527ms).
W0821 10:52:34.275460 910 periodic.cc:174] PeriodicTimer callback delayed by 2.014s
I0821 10:52:35.376236 3812866 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:52:35.518399 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403f923880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313016, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97137.630s (123.030s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97147.906s (112.754s ago), now: 97260.660s, connection: Connection (0x000050403ac7ee20) client 10.0.0.2:44741 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:52:35.537204 3812900 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313016) to 10.0.0.4:7100 timed out after 10.181s
W0821 10:52:35.558722 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403bf18000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313018, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97201.872s (62.682s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97263.105s (1.449s ago), now: 97264.554s, connection: Connection (0x0000504038e021e0) client 10.0.0.2:41845 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 10:52:35.558394 3812900 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:52:35.561257 3812866 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 12.256s
I0821 10:52:35.561287 3812866 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:52:35.571131 3812900 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313018) to 10.0.0.4:7100 timed out after 10.316s
W0821 10:52:35.575799 917 inbound_call.cc:111] 0x000050403e313c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34793 => 10.0.0.2:7100 (request call id 25728): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34793 => 10.0.0.2:7100 (request call id 25728) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:52:35.606379 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:52:35.799239 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x00005040379eea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313017, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97197.196s (67.583s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97224.492s (40.287s ago), now: 97264.779s, connection: Connection (0x000050403b099360) client 10.0.0.2:40415 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:52:35.820200 3812900 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313017) to 10.0.0.3:7100 timed out after 10.316s
I0821 10:52:35.832279 3812900 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:52:35.835335 3812866 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.858s
I0821 10:52:35.844833 3812866 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:52:40.455881 910 periodic.cc:174] PeriodicTimer callback delayed by 2.550s
W0821 10:52:41.426026 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403d8b4000: yb.master.MasterService.ListNamespaces -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313019, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97218.667s (51.746s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97265.755s (4.658s ago), now: 97270.414s, connection: Connection (0x0000504038e69c20) client 10.0.0.2:35357 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 10:52:41.427932 3812900 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356232.015s
I0821 10:52:41.430140 944 client-internal.cc:2923] Reinitialize master addresses from file: ./master.conf
W0821 10:52:41.430979 911 pg_client_service.cc:2658] Could not get the set of database oids: Timed out (yb/client/client_master_rpc.cc:138): ListNamespaces timed out after deadline expired, passed 52.080s of 1.457s
I0821 10:52:41.431005 911 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0821 10:52:41.435817 944 client-internal.cc:2952] New master addresses: [10.0.0.2:7100,10.0.0.3:7100,10.0.0.4:7100, 10.0.0.2:7100, 10.0.0.3:7100, 10.0.0.4:7100]
W0821 10:52:41.449350 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403be92a80: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.2:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313020, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97260.062s (10.392s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97266.199s (4.255s ago), now: 97270.455s, connection: Connection (0x000050403b1021e0) client 10.0.0.2:44301 => 10.0.0.2:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:52:41.449647 3812875 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.2:7100]) id (55d29a1415af4e44ac228b2c8a82f866): GetMasterRegistration RPC (request call id 313020) to 10.0.0.2:7100 timed out after 1.500s
W0821 10:52:42.965660 3812875 master-path-handlers.cc:237] Not found (yb/master/master-path-handlers.cc:277): Unable to locate leader master to redirect this request: /
W0821 10:52:43.555011 912 periodic.cc:174] PeriodicTimer callback delayed by 0.863s
W0821 10:52:45.924896 911 periodic.cc:174] PeriodicTimer callback delayed by 0.089s
W0821 10:52:47.443967 3812986 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313022) to 10.0.0.4:7100 timed out after 10.312s
W0821 10:52:48.055837 913 periodic.cc:174] PeriodicTimer callback delayed by 1.335s
I0821 10:52:49.087628 3812986 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:52:51.010671 911 thread.cc:844] CatalogManagerBGTasks_5786_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.869s user 0.000s sys 0.011s
W0821 10:52:51.010332 913 thread.cc:844] raft_19687_worker (raft) Time spent starting thread: real 0.870s user 0.000s sys 0.011s
I0821 10:52:50.977581 3813047 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:52:52.379791 3812900 long_operation_tracker.cc:155] UpdateReplica took a long time: 1.092s
I0821 10:52:52.936976 3813047 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:52:54.875339 3813047 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 17.510s
W0821 10:52:56.301317 3812986 thread.cc:844] raft_19688_worker (raft) Time spent starting thread: real 1.300s user 0.001s sys 0.027s
W0821 10:52:57.448158 910 periodic.cc:174] PeriodicTimer callback delayed by 4.680s
W0821 10:52:56.385689 3812900 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:46565 => 10.0.0.2:7100 (request call id 25826) took 10921ms (client timeout 3000ms).
W0821 10:52:58.826742 3812986 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf66000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313022, state: TIMED_OUT, transfer_state: PENDING took 11.349s
W0821 10:53:03.178653 913 periodic.cc:174] PeriodicTimer callback delayed by 1.382s
W0821 10:53:07.228212 913 periodic.cc:174] PeriodicTimer callback delayed by 1.794s
W0821 10:53:11.369478 913 periodic.cc:174] PeriodicTimer callback delayed by 1.894s
W0821 10:53:15.783346 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403dcaf180: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313025, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97270.443s (23.830s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97285.437s (8.836s ago), now: 97294.273s, connection: Connection (0x000050403b08f1a0) client 10.0.0.2:41539 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:53:15.848435 913 periodic.cc:174] PeriodicTimer callback delayed by 2.162s
W0821 10:53:17.977738 912 periodic.cc:174] PeriodicTimer callback delayed by 2.142s
W0821 10:53:23.030876 915 thread.cc:888] Master_2330_worker (Master) Time spent waiting for new thread to publish its TID: real 1.000s user 0.000s sys 0.042s
W0821 10:53:25.400758 911 periodic.cc:174] PeriodicTimer callback delayed by 2.407s
W0821 10:53:25.841279 913 thread.cc:850] CatalogManagerBGTasks_5787_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.593s user 0.000s sys 0.014s
W0821 10:53:25.841270 915 thread.cc:844] Master_2330_worker (Master) Time spent starting thread: real 5.612s user 0.000s sys 0.122s
W0821 10:53:28.572814 913 thread.cc:844] CatalogManagerBGTasks_5787_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.216s user 0.000s sys 0.076s
W0821 10:53:23.972357 3813064 outbound_call.cc:526] RPC callback for RPC call 0x000050403dcaf180: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313025, state: TIMED_OUT, transfer_state: PENDING took 1.713s
W0821 10:53:33.739667 910 periodic.cc:174] PeriodicTimer callback delayed by 3.617s
W0821 10:53:41.180836 911 periodic.cc:174] PeriodicTimer callback delayed by 2.404s
W0821 10:53:49.134385 912 periodic.cc:174] PeriodicTimer callback delayed by 2.384s
W0821 10:53:55.643043 912 periodic.cc:174] PeriodicTimer callback delayed by 1.636s
W0821 10:53:57.355767 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403dd90380: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313028, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97270.443s (49.250s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97285.448s (34.245s ago), now: 97319.693s, connection: Connection (0x000050403bbfe720) client 10.0.0.2:38915 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 10:52:58.911617 3813047 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:54:01.793090 916 thread.cc:844] Master_2331_worker (Master) Time spent starting thread: real 0.666s user 0.000s sys 0.014s
W0821 10:54:02.597985 911 thread.cc:844] CatalogManagerBGTasks_5788_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.981s user 0.000s sys 0.036s
W0821 10:54:02.769680 912 periodic.cc:174] PeriodicTimer callback delayed by 2.270s
W0821 10:54:03.419296 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:54:04.167446 3813047 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:54:04.195580 3813049 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 17.167s
W0821 10:54:03.346541 3813078 outbound_call.cc:526] RPC callback for RPC call 0x000050403dd90380: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313028, state: TIMED_OUT, transfer_state: PENDING took 1.572s
W0821 10:54:05.547729 917 yb_rpc.cc:553] Connection (0x000050403b1fd8a0) client 10.0.0.2:44937 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.599s, timeout: 15.000s, now: 97299.261s, last_read_time_: 97275.662s
I0821 10:54:05.769987 3813049 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:54:09.248070 913 periodic.cc:174] PeriodicTimer callback delayed by 1.957s
I0821 10:54:08.147276 3813047 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:54:11.471397 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:54:12.790961 913 periodic.cc:174] PeriodicTimer callback delayed by 1.321s
W0821 10:54:16.601435 913 periodic.cc:174] PeriodicTimer callback delayed by 1.563s
W0821 10:54:20.950402 911 periodic.cc:174] PeriodicTimer callback delayed by 1.858s
W0821 10:54:20.361864 914 yb_rpc.cc:553] Connection (0x0000504038e69c20) client 10.0.0.2:35357 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 45.873s, timeout: 15.000s, now: 97322.545s, last_read_time_: 97276.672s
W0821 10:54:24.906195 912 periodic.cc:174] PeriodicTimer callback delayed by 1.967s
W0821 10:54:23.372598 915 yb_rpc.cc:553] Connection (0x000050403b191360) client 10.0.0.2:38947 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 18.229s, timeout: 15.000s, now: 97294.005s, last_read_time_: 97275.776s
W0821 10:54:27.064010 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:54:28.083724 3813095 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:54:28.113214 3813095 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:54:28.162179 3813095 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 11.069s
W0821 10:54:31.081816 914 yb_rpc.cc:553] Connection (0x000050403b8ca8e0) client 10.0.0.2:34169 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 52.112s, timeout: 15.000s, now: 97322.545s, last_read_time_: 97270.433s
W0821 10:54:32.214942 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:54:34.670902 911 periodic.cc:174] PeriodicTimer callback delayed by 3.817s
W0821 10:54:38.393913 910 thread.cc:888] CatalogManagerBGTasks_5789_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.249s user 0.000s sys 0.079s
W0821 10:54:38.885484 913 periodic.cc:174] PeriodicTimer callback delayed by 2.034s
W0821 10:54:36.445133 915 yb_rpc.cc:553] Connection (0x000050403b0cd360) client 10.0.0.2:41417 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.578s, timeout: 15.000s, now: 97294.005s, last_read_time_: 97270.427s
W0821 10:54:40.766577 910 thread.cc:844] CatalogManagerBGTasks_5789_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.231s user 0.000s sys 0.103s
W0821 10:54:42.208571 911 periodic.cc:174] PeriodicTimer callback delayed by 2.937s
W0821 10:54:48.592738 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:54:28.203040 3813095 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:54:49.660760 910 periodic.cc:174] PeriodicTimer callback delayed by 2.036s
W0821 10:54:49.831300 914 yb_rpc.cc:553] Connection (0x0000504038e42c60) client 10.0.0.2:42531 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 45.873s, timeout: 15.000s, now: 97322.545s, last_read_time_: 97276.672s
I0821 10:54:51.812476 3813095 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:54:52.390923 3813103 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39493 => 10.0.0.2:7100 (request call id 8602076) took 21782ms (client timeout 14981ms).
I0821 10:54:52.747991 3813095 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:54:53.068759 916 yb_rpc.cc:553] Connection (0x000050403b1021e0) client 10.0.0.2:44301 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 48.895s, timeout: 15.000s, now: 97319.352s, last_read_time_: 97270.457s
W0821 10:54:55.639814 913 periodic.cc:174] PeriodicTimer callback delayed by 1.802s
W0821 10:54:56.489385 917 inbound_call.cc:111] 0x000050403cb04800 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39493 => 10.0.0.2:7100 (request call id 8602076): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39493 => 10.0.0.2:7100 (request call id 8602076) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:55:00.378286 913 thread.cc:888] raft_19690_worker (raft) Time spent waiting for new thread to publish its TID: real 0.762s user 0.000s sys 0.023s
I0821 10:55:00.433868 3813149 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:55:02.795290 913 thread.cc:844] raft_19690_worker (raft) Time spent starting thread: real 4.005s user 0.000s sys 0.059s
I0821 10:55:02.952033 3813149 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:55:05.043859 3813149 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.910s
W0821 10:55:10.456478 911 periodic.cc:174] PeriodicTimer callback delayed by 9.433s
W0821 10:55:13.901997 910 periodic.cc:174] PeriodicTimer callback delayed by 2.768s
W0821 10:55:16.097596 912 thread.cc:850] CatalogManagerBGTasks_5790_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.728s user 0.000s sys 0.011s
W0821 10:55:19.441963 912 thread.cc:844] CatalogManagerBGTasks_5790_worker (CatalogManagerBGTasks) Time spent starting thread: real 5.893s user 0.000s sys 0.077s
W0821 10:55:21.479050 913 periodic.cc:174] PeriodicTimer callback delayed by 2.946s
W0821 10:55:23.152858 916 yb_rpc.cc:553] Connection (0x000050403b08e020) client 10.0.0.2:34509 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 42.151s, timeout: 15.000s, now: 97319.352s, last_read_time_: 97277.201s
W0821 10:55:30.155777 912 periodic.cc:174] PeriodicTimer callback delayed by 3.047s
I0821 10:55:07.223639 3813149 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:55:30.475795 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313031, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97355.066s (72.356s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97417.598s (9.824s ago), now: 97427.422s, connection: Connection (0x000050403b14d360) client 10.0.0.2:33949 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:55:30.809396 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403f923880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313034, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97402.090s (28.699s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97422.241s (8.548s ago), now: 97430.789s, connection: Connection (0x000050403b88a720) client 10.0.0.2:37499 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 10:55:33.991171 3813149 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:55:36.407102 917 thread.cc:888] Master-high-pri_484_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.683s user 0.000s sys 0.013s
W0821 10:55:36.201510 915 thread.cc:844] Master-high-pri_485_worker (Master-high-pri) Time spent starting thread: real 1.367s user 0.000s sys 0.020s
W0821 10:55:36.201478 3813174 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313031) to 10.0.0.3:7100 timed out after 17.510s
W0821 10:55:37.280206 913 periodic.cc:174] PeriodicTimer callback delayed by 2.198s
W0821 10:55:36.759613 3813173 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313034) to 10.0.0.4:7100 timed out after 11.069s
W0821 10:55:38.237565 917 thread.cc:844] Master-high-pri_484_worker (Master-high-pri) Time spent starting thread: real 3.336s user 0.000s sys 0.063s
I0821 10:55:40.649557 3813149 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:55:39.079380 3813174 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313031, state: TIMED_OUT, transfer_state: PENDING took 2.919s
W0821 10:55:39.717782 3813173 outbound_call.cc:526] RPC callback for RPC call 0x000050403f923880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313034, state: TIMED_OUT, transfer_state: PENDING took 2.949s
W0821 10:55:46.120121 913 periodic.cc:174] PeriodicTimer callback delayed by 3.170s
W0821 10:55:50.545279 913 thread.cc:850] raft_19691_worker (raft) Time spent creating pthread: real 0.619s user 0.000s sys 0.008s
W0821 10:55:53.543344 913 thread.cc:844] raft_19691_worker (raft) Time spent starting thread: real 3.681s user 0.000s sys 0.046s
I0821 10:55:53.552201 3813181 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:55:56.210057 911 thread.cc:844] CatalogManagerBGTasks_5791_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.061s user 0.000s sys 0.023s
I0821 10:55:57.808220 3813181 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:55:57.896973 3813181 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 11.180s
I0821 10:55:57.930699 3813181 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:55:58.498647 3813181 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:55:58.515606 3813181 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:55:58.544488 916 inbound_call.cc:111] 0x000050403bc02800 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:42295 => 10.0.0.2:7100 (request call id 8602082): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:42295 => 10.0.0.2:7100 (request call id 8602082) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:56:02.513304 912 periodic.cc:174] PeriodicTimer callback delayed by 10.833s
I0821 10:56:04.373968 3813227 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:56:05.088351 3813227 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:56:06.370716 3813227 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 11.280s
W0821 10:56:07.846140 911 periodic.cc:174] PeriodicTimer callback delayed by 2.227s
W0821 10:56:11.218907 910 periodic.cc:174] PeriodicTimer callback delayed by 1.126s
I0821 10:56:06.405762 3813227 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:56:12.918959 3813227 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:56:12.919040 3813227 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:56:12.916590 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x00005040379eea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313033, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97401.718s (80.204s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97478.611s (3.311s ago), now: 97481.922s, connection: Connection (0x000050403b08efe0) client 10.0.0.2:45569 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:56:12.921573 3813173 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313033) to 10.0.0.3:7100 timed out after 11.069s
I0821 10:56:12.921602 3813173 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:56:12.921878 3813227 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.466s
I0821 10:56:12.921893 3813227 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:56:12.951758 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313035, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97445.517s (36.440s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97470.626s (11.331s ago), now: 97481.957s, connection: Connection (0x000050403b84a720) client 10.0.0.2:35011 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:56:12.955895 3813173 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313035) to 10.0.0.3:7100 timed out after 3.910s
W0821 10:56:12.969173 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403dd4aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313036, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97452.136s (29.837s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97470.816s (11.158s ago), now: 97481.974s, connection: Connection (0x000050403b8a9de0) client 10.0.0.2:42229 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:56:12.970752 3813173 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313036) to 10.0.0.4:7100 timed out after 3.910s
I0821 10:56:12.970778 3813173 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 10:56:12.970818 3813227 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 8.697s
I0821 10:56:12.970829 3813227 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:56:12.996313 3813274 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356020.443s
I0821 10:56:13.001946 3813274 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223356020.443s
W0821 10:56:14.790618 910 periodic.cc:174] PeriodicTimer callback delayed by 1.355s
W0821 10:56:16.016753 3813274 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313032) to 10.0.0.4:7100 timed out after 17.510s
W0821 10:56:17.331787 912 periodic.cc:174] PeriodicTimer callback delayed by 0.265s
I0821 10:56:16.235157 3813274 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:56:20.228884 912 periodic.cc:174] PeriodicTimer callback delayed by 1.439s
W0821 10:56:21.362916 3813274 thread.cc:888] raft_19693_worker (raft) Time spent waiting for new thread to publish its TID: real 1.149s user 0.003s sys 0.047s
I0821 10:56:21.392481 3813334 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.760s
W0821 10:56:22.966665 3813274 thread.cc:844] raft_19693_worker (raft) Time spent starting thread: real 3.217s user 0.005s sys 0.084s
I0821 10:56:23.136402 3813334 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:56:23.934140 912 thread.cc:888] raft_19694_worker (raft) Time spent waiting for new thread to publish its TID: real 0.578s user 0.000s sys 0.018s
W0821 10:56:25.431073 3813173 long_operation_tracker.cc:155] UpdateReplica took a long time: 2.564s
W0821 10:56:26.363451 912 thread.cc:844] raft_19694_worker (raft) Time spent starting thread: real 3.430s user 0.000s sys 0.073s
I0821 10:56:23.934876 3813335 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:56:28.252410 3813335 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:56:27.756798 3813274 outbound_call.cc:526] RPC callback for RPC call 0x000050403d92aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313032, state: TIMED_OUT, transfer_state: PENDING took 11.549s
W0821 10:56:28.069578 3813173 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43901 => 10.0.0.2:7100 (request call id 25988) took 9462ms (client timeout 3000ms).
I0821 10:56:30.529883 3813335 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 14.417s
W0821 10:56:33.501137 910 periodic.cc:174] PeriodicTimer callback delayed by 8.794s
W0821 10:56:33.373667 913 thread.cc:888] CatalogManagerBGTasks_5792_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.179s user 0.000s sys 0.049s
W0821 10:56:35.795395 913 thread.cc:844] CatalogManagerBGTasks_5792_worker (CatalogManagerBGTasks) Time spent starting thread: real 5.082s user 0.000s sys 0.133s
W0821 10:56:37.384922 914 thread.cc:850] Master_2336_worker (Master) Time spent creating pthread: real 0.520s user 0.000s sys 0.013s
W0821 10:56:39.225632 911 periodic.cc:174] PeriodicTimer callback delayed by 1.785s
W0821 10:56:39.676067 914 thread.cc:844] Master_2336_worker (Master) Time spent starting thread: real 2.844s user 0.000s sys 0.066s
W0821 10:56:43.594420 911 periodic.cc:174] PeriodicTimer callback delayed by 2.131s
W0821 10:56:47.274063 911 periodic.cc:174] PeriodicTimer callback delayed by 1.429s
W0821 10:56:52.334936 911 periodic.cc:174] PeriodicTimer callback delayed by 3.072s
W0821 10:56:58.034039 913 periodic.cc:174] PeriodicTimer callback delayed by 1.732s
W0821 10:56:56.278096 914 yb_rpc.cc:553] Connection (0x0000504038eeb360) client 10.0.0.2:44899 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.917s, timeout: 15.000s, now: 97511.437s, last_read_time_: 97487.519s
W0821 10:56:59.739365 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x00005040379ef500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313038, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97467.522s (48.741s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97493.148s (23.115s ago), now: 97516.263s, connection: Connection (0x000050403bb82e20) client 10.0.0.2:45645 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:57:03.331465 915 thread.cc:888] Master-high-pri_487_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 1.102s user 0.000s sys 0.040s
W0821 10:57:03.276122 3813344 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313038) to 10.0.0.4:7100 timed out after 11.180s
W0821 10:57:05.449198 912 periodic.cc:174] PeriodicTimer callback delayed by 3.406s
W0821 10:57:06.926900 915 thread.cc:844] Master-high-pri_487_worker (Master-high-pri) Time spent starting thread: real 2.834s user 0.000s sys 0.068s
I0821 10:57:07.068866 3813344 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:57:11.207140 3813344 thread.cc:844] raft_19695_worker (raft) Time spent starting thread: real 1.327s user 0.001s sys 0.018s
W0821 10:57:11.160151 912 thread.cc:888] CatalogManagerBGTasks_5793_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.633s user 0.000s sys 0.033s
W0821 10:57:12.626502 911 periodic.cc:174] PeriodicTimer callback delayed by 2.792s
W0821 10:57:13.058780 912 thread.cc:844] CatalogManagerBGTasks_5793_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.496s user 0.000s sys 0.075s
W0821 10:57:13.601794 3813344 outbound_call.cc:526] RPC callback for RPC call 0x00005040379ef500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313038, state: TIMED_OUT, transfer_state: PENDING took 10.335s
W0821 10:57:19.192198 913 periodic.cc:174] PeriodicTimer callback delayed by 1.814s
W0821 10:57:21.999250 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x0000504039a20700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313040, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97481.924s (53.001s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97493.263s (41.662s ago), now: 97534.926s, connection: Connection (0x000050403bb838a0) client 10.0.0.2:39519 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 10:57:24.266439 3813344 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313040) to 10.0.0.4:7100 timed out after 11.280s
W0821 10:57:25.401783 912 periodic.cc:174] PeriodicTimer callback delayed by 1.396s
I0821 10:57:26.757364 3813344 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 10:57:31.214831 3813344 thread.cc:844] raft_19696_worker (raft) Time spent starting thread: real 1.237s user 0.000s sys 0.028s
I0821 10:56:35.364859 3813335 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 10:57:31.922084 910 periodic.cc:174] PeriodicTimer callback delayed by 1.907s
I0821 10:57:35.423271 3813348 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.651s
I0821 10:57:34.884030 3813335 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:57:36.508636 3813359 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 17.063s
W0821 10:57:35.800107 3813337 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39825 => 10.0.0.2:7100 (request call id 8611119) took 65591ms (client timeout 15000ms).
I0821 10:57:36.508692 3813359 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 10:57:36.507815 3813348 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 10:57:33.934996 3813344 outbound_call.cc:526] RPC callback for RPC call 0x0000504039a20700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313040, state: TIMED_OUT, transfer_state: PENDING took 9.612s
I0821 10:57:38.265332 3813335 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:57:39.051579 912 periodic.cc:174] PeriodicTimer callback delayed by 2.041s
W0821 10:57:39.662164 914 inbound_call.cc:111] 0x00005040375ce400 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39825 => 10.0.0.2:7100 (request call id 8611119): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39825 => 10.0.0.2:7100 (request call id 8611119) could send its response: Network error (yb/util/net/socket.cc:536): recvmsg got EOF from remote (system error 108)
W0821 10:57:42.499442 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:57:44.452281 911 periodic.cc:174] PeriodicTimer callback delayed by 3.213s
W0821 10:57:45.801083 910 thread.cc:888] CatalogManagerBGTasks_5794_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.873s user 0.000s sys 0.031s
W0821 10:57:47.819108 910 thread.cc:844] CatalogManagerBGTasks_5794_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.963s user 0.000s sys 0.081s
W0821 10:57:48.816890 912 periodic.cc:174] PeriodicTimer callback delayed by 2.116s
W0821 10:57:51.503268 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:57:52.878051 912 periodic.cc:174] PeriodicTimer callback delayed by 1.833s
W0821 10:57:57.008632 912 periodic.cc:174] PeriodicTimer callback delayed by 3.404s
I0821 10:58:01.203951 3813375 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:58:02.379500 912 thread.cc:844] raft_19697_worker (raft) Time spent starting thread: real 1.214s user 0.000s sys 0.020s
I0821 10:58:04.007361 3813375 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:58:07.004781 3813375 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 16.768s
W0821 10:58:09.581646 913 periodic.cc:174] PeriodicTimer callback delayed by 8.232s
W0821 10:58:11.608444 915 yb_rpc.cc:553] Connection (0x000050403bb83c20) client 10.0.0.2:42103 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 28.105s, timeout: 15.000s, now: 97515.834s, last_read_time_: 97487.728s
W0821 10:58:13.421789 912 periodic.cc:174] PeriodicTimer callback delayed by 1.770s
W0821 10:58:17.078295 910 periodic.cc:174] PeriodicTimer callback delayed by 1.440s
W0821 10:58:21.543345 911 thread.cc:844] CatalogManagerBGTasks_5795_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.558s user 0.000s sys 0.024s
W0821 10:58:21.454064 912 periodic.cc:174] PeriodicTimer callback delayed by 2.143s
W0821 10:58:25.744611 913 periodic.cc:174] PeriodicTimer callback delayed by 2.024s
W0821 10:58:29.206864 910 periodic.cc:174] PeriodicTimer callback delayed by 3.107s
W0821 10:58:29.193683 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:58:35.348778 910 periodic.cc:174] PeriodicTimer callback delayed by 1.879s
W0821 10:58:36.174935 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 10:58:42.019690 911 periodic.cc:174] PeriodicTimer callback delayed by 1.964s
W0821 10:58:41.057029 917 inbound_call.cc:111] 0x000050403703a400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43901 => 10.0.0.2:7100 (request call id 25988): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43901 => 10.0.0.2:7100 (request call id 25988) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 10:58:09.330259 3813375 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:58:44.785655 3813375 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 10:58:50.996346 3813375 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:58:52.067610 912 periodic.cc:174] PeriodicTimer callback delayed by 4.945s
W0821 10:58:54.514991 910 thread.cc:844] CatalogManagerBGTasks_5796_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.910s user 0.000s sys 0.014s
W0821 10:58:58.852623 911 periodic.cc:174] PeriodicTimer callback delayed by 1.655s
I0821 10:59:01.823215 3813419 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 10:59:02.471387 911 thread.cc:844] raft_19698_worker (raft) Time spent starting thread: real 1.287s user 0.000s sys 0.024s
I0821 10:59:03.554729 3813419 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 10:59:04.380677 3813419 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 12.946s
W0821 10:59:04.415411 916 yb_rpc.cc:553] Connection (0x000050403b08efe0) client 10.0.0.2:45569 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 45.976s, timeout: 15.000s, now: 97533.705s, last_read_time_: 97487.728s
W0821 10:59:10.546375 910 periodic.cc:174] PeriodicTimer callback delayed by 6.358s
W0821 10:59:15.282243 913 periodic.cc:174] PeriodicTimer callback delayed by 2.675s
I0821 10:59:04.415597 3813419 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 10:59:17.671934 3813419 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 10:59:18.896347 911 periodic.cc:174] PeriodicTimer callback delayed by 1.529s
W0821 10:59:24.542753 912 periodic.cc:174] PeriodicTimer callback delayed by 1.596s
I0821 10:59:23.238722 3813419 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 10:59:31.648082 911 periodic.cc:174] PeriodicTimer callback delayed by 2.637s
W0821 10:59:32.311777 910 thread.cc:888] CatalogManagerBGTasks_5797_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.856s user 0.000s sys 0.062s
W0821 10:59:37.109746 911 thread.cc:850] raft_19699_worker (raft) Time spent creating pthread: real 1.545s user 0.000s sys 0.021s
W0821 10:59:35.925940 910 thread.cc:844] CatalogManagerBGTasks_5797_worker (CatalogManagerBGTasks) Time spent starting thread: real 8.225s user 0.000s sys 0.157s
W0821 10:59:37.364548 911 thread.cc:844] raft_19699_worker (raft) Time spent starting thread: real 1.972s user 0.000s sys 0.028s
W0821 10:59:37.344061 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403f878700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313042, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97569.671s (100.610s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97666.997s (3.283s ago), now: 97670.281s, connection: Connection (0x0000504038f17a60) client 10.0.0.2:36157 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 10:59:37.365967 3813451 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 10:59:37.723728 3813451 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 10:59:39.293881 916 thread.cc:844] Master-high-pri_488_worker (Master-high-pri) Time spent starting thread: real 1.081s user 0.000s sys 0.024s
I0821 10:59:39.650458 3813451 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.569s
W0821 10:59:37.405920 913 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:42649 => 10.0.0.2:7100 (request call id 25990) took 10023ms (client timeout 3000ms).
W0821 10:59:42.042479 911 periodic.cc:174] PeriodicTimer callback delayed by 6.140s
W0821 10:59:43.124114 917 thread.cc:888] Master-high-pri_489_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 1.583s user 0.000s sys 0.067s
W0821 10:59:43.234658 3813454 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313042) to 10.0.0.4:7100 timed out after 14.417s
W0821 10:59:45.892477 911 periodic.cc:174] PeriodicTimer callback delayed by 1.572s
W0821 10:59:45.605322 917 thread.cc:844] Master-high-pri_489_worker (Master-high-pri) Time spent starting thread: real 4.692s user 0.000s sys 0.110s
W0821 10:59:46.730165 3813454 outbound_call.cc:526] RPC callback for RPC call 0x000050403f878700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313042, state: TIMED_OUT, transfer_state: PENDING took 3.452s
W0821 10:59:50.246833 912 periodic.cc:174] PeriodicTimer callback delayed by 2.118s
W0821 10:59:53.711781 911 periodic.cc:174] PeriodicTimer callback delayed by 2.618s
W0821 11:00:01.723657 913 periodic.cc:174] PeriodicTimer callback delayed by 2.563s
W0821 11:00:02.989470 916 inbound_call.cc:111] 0x000050403bc02800 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:42649 => 10.0.0.2:7100 (request call id 25990): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:42649 => 10.0.0.2:7100 (request call id 25990) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:00:08.424847 910 thread.cc:844] CatalogManagerBGTasks_5798_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.239s user 0.000s sys 0.044s
W0821 11:00:09.074702 913 periodic.cc:174] PeriodicTimer callback delayed by 2.283s
W0821 11:00:09.519162 917 thread.cc:844] Master-high-pri_490_worker (Master-high-pri) Time spent starting thread: real 1.044s user 0.000s sys 0.017s
W0821 11:00:09.252712 3813460 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x0000504038f676e0) client 10.0.0.2:33023 => 10.0.0.3:7100, passed: 14.970s, timeout: 15.000s: kConnectFailed (network error 1)
W0821 11:00:13.806341 3813460 outbound_call.cc:526] RPC callback for RPC call 0x000050403be27180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313043, state: FINISHED_ERROR, transfer_state: PENDING took 4.576s
W0821 11:00:17.227106 913 periodic.cc:174] PeriodicTimer callback delayed by 3.158s
I0821 10:59:41.562311 3813451 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:00:24.517590 913 periodic.cc:174] PeriodicTimer callback delayed by 2.225s
I0821 11:00:26.306427 3813451 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:00:34.834240 911 periodic.cc:174] PeriodicTimer callback delayed by 4.791s
I0821 11:00:36.516417 3813451 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:00:41.315374 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313041, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97565.522s (184.798s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97708.452s (41.867s ago), now: 97750.320s, connection: Connection (0x000050403bb94aa0) client 10.0.0.2:46265 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:00:41.314544 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403defc000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313046, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97675.720s (74.594s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97727.605s (22.709s ago), now: 97750.314s, connection: Connection (0x000050403b087de0) client 10.0.0.2:33833 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:00:41.317879 3813467 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313046) to 10.0.0.4:7100 timed out after 12.946s
W0821 11:00:41.317950 3813468 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313041) to 10.0.0.3:7100 timed out after 14.417s
I0821 11:00:41.318878 3813468 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:00:41.324468 3813451 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.763s
I0821 11:00:41.324506 3813451 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:00:41.414304 3813467 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223355752.026s
I0821 11:00:41.414302 3813468 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223355752.026s
I0821 11:00:41.414362 3813468 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:00:41.414501 3813451 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.544s
I0821 11:00:41.414539 3813451 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:00:41.638620 913 periodic.cc:174] PeriodicTimer callback delayed by 1.795s
W0821 11:00:46.641388 910 periodic.cc:174] PeriodicTimer callback delayed by 0.436s
W0821 11:00:49.111748 912 periodic.cc:174] PeriodicTimer callback delayed by 2.410s
W0821 11:00:53.935066 912 thread.cc:888] raft_19700_worker (raft) Time spent waiting for new thread to publish its TID: real 1.257s user 0.000s sys 0.044s
I0821 11:00:53.665556 3813659 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:00:51.576058 3813467 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:43025 => 10.0.0.2:7100 (request call id 26226) took 4765ms (client timeout 3000ms).
W0821 11:00:55.653914 912 thread.cc:844] raft_19700_worker (raft) Time spent starting thread: real 4.250s user 0.000s sys 0.101s
I0821 11:00:56.548700 3813659 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:00:56.584450 3813659 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 18.932s
W0821 11:01:02.118422 913 periodic.cc:174] PeriodicTimer callback delayed by 8.854s
W0821 11:01:06.723299 913 periodic.cc:174] PeriodicTimer callback delayed by 2.331s
W0821 11:01:07.588857 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bffa380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313048, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97749.165s (16.393s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97759.892s (5.666s ago), now: 97765.559s, connection: Connection (0x000050403b048c60) client 10.0.0.2:43875 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:01:11.234436 913 periodic.cc:174] PeriodicTimer callback delayed by 2.286s
W0821 11:01:09.492715 3813467 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313048) to 10.0.0.4:7100 timed out after 9.569s
I0821 11:01:12.758150 3813467 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:01:15.105316 913 periodic.cc:174] PeriodicTimer callback delayed by 1.594s
W0821 11:01:15.154727 910 thread.cc:850] CatalogManagerBGTasks_5800_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.539s user 0.000s sys 0.011s
W0821 11:01:17.072571 3813467 thread.cc:844] raft_19701_worker (raft) Time spent starting thread: real 1.136s user 0.000s sys 0.028s
W0821 11:01:17.047760 910 thread.cc:844] CatalogManagerBGTasks_5800_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.290s user 0.000s sys 0.047s
W0821 11:01:18.760823 910 periodic.cc:174] PeriodicTimer callback delayed by 1.406s
W0821 11:01:17.733587 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:01:21.413380 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403f923880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313044, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97641.517s (139.753s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97767.133s (14.136s ago), now: 97781.270s, connection: Connection (0x000050403b0878a0) client 10.0.0.2:37277 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:01:19.282210 3813467 outbound_call.cc:526] RPC callback for RPC call 0x000050403bffa380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313048, state: TIMED_OUT, transfer_state: PENDING took 9.675s
W0821 11:01:25.795951 911 periodic.cc:174] PeriodicTimer callback delayed by 2.071s
W0821 11:01:27.363651 914 thread.cc:850] Master-high-pri_493_worker (Master-high-pri) Time spent creating pthread: real 0.743s user 0.000s sys 0.013s
W0821 11:01:29.249557 914 thread.cc:844] Master-high-pri_493_worker (Master-high-pri) Time spent starting thread: real 3.296s user 0.000s sys 0.061s
W0821 11:01:28.745978 3813675 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313044) to 10.0.0.4:7100 timed out after 16.768s
W0821 11:01:31.185384 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:01:30.888397 3813675 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:01:32.358263 911 periodic.cc:174] PeriodicTimer callback delayed by 1.699s
W0821 11:01:36.083667 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:01:37.722267 3813675 thread.cc:888] raft_19702_worker (raft) Time spent waiting for new thread to publish its TID: real 1.386s user 0.005s sys 0.037s
W0821 11:01:39.574589 912 periodic.cc:174] PeriodicTimer callback delayed by 2.260s
W0821 11:01:39.686679 3813675 thread.cc:844] raft_19702_worker (raft) Time spent starting thread: real 5.206s user 0.006s sys 0.109s
W0821 11:01:44.124236 917 thread.cc:844] Master_2339_worker (Master) Time spent starting thread: real 2.418s user 0.000s sys 0.040s
W0821 11:01:43.679461 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:01:42.674836 915 tcp_stream.cc:226] { local: 10.0.0.2:7100 remote: 10.0.0.3:45505 }: Send failed: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:01:42.948868 3813675 outbound_call.cc:526] RPC callback for RPC call 0x000050403f923880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313044, state: TIMED_OUT, transfer_state: PENDING took 14.195s
W0821 11:01:46.485689 910 periodic.cc:174] PeriodicTimer callback delayed by 2.901s
W0821 11:01:45.700582 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:01:53.201300 912 periodic.cc:174] PeriodicTimer callback delayed by 1.690s
W0821 11:01:53.254294 913 thread.cc:888] CatalogManagerBGTasks_5801_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.633s user 0.000s sys 0.026s
W0821 11:01:54.931075 913 thread.cc:844] CatalogManagerBGTasks_5801_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.172s user 0.000s sys 0.086s
W0821 11:01:59.308804 910 periodic.cc:174] PeriodicTimer callback delayed by 1.882s
W0821 11:01:58.669303 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:02.179564 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:01.762063 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:00:58.485198 3813659 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:02:02.215173 3813670 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 8.209s
I0821 11:02:02.210402 3813659 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:02:02.200242 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:02.200337 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:02.201618 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:02.201063 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:02:02.227952 3813679 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 5.105s
I0821 11:02:02.227891 3813670 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:02:02.241175 3813679 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:02:02.257437 3813659 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:02:02.242344 915 yb_rpc.cc:553] Connection (0x000050403b101a60) client 10.0.0.2:34119 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 39.626s, timeout: 15.000s, now: 97800.248s, last_read_time_: 97760.621s
I0821 11:02:02.254491 3813682 yb_rpc.cc:395] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:42213 => 10.0.0.2:7100 (request call id 8622222) took 25585ms.
W0821 11:02:05.341259 912 periodic.cc:174] PeriodicTimer callback delayed by 1.658s
W0821 11:02:09.751595 913 periodic.cc:174] PeriodicTimer callback delayed by 2.186s
W0821 11:02:09.847208 914 yb_rpc.cc:553] Connection (0x000050403b132fe0) client 10.0.0.2:37247 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.136s, timeout: 15.000s, now: 97780.945s, last_read_time_: 97755.808s
W0821 11:02:18.735200 913 thread.cc:844] raft_19703_worker (raft) Time spent starting thread: real 1.680s user 0.000s sys 0.030s
I0821 11:02:18.178444 3813700 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:02:22.899510 917 thread.cc:844] Master-high-pri_494_worker (Master-high-pri) Time spent starting thread: real 2.790s user 0.000s sys 0.043s
W0821 11:02:19.381673 911 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39665 => 10.0.0.2:7100 (request call id 26230) took 10452ms (client timeout 3000ms).
I0821 11:02:23.187618 3813700 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 11:02:26.817469 913 periodic.cc:174] PeriodicTimer callback delayed by 12.384s
I0821 11:02:25.619709 3813700 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.229s
W0821 11:02:27.894701 912 thread.cc:844] CatalogManagerBGTasks_5802_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.770s user 0.000s sys 0.029s
W0821 11:02:27.895268 916 thread.cc:844] Master-high-pri_495_worker (Master-high-pri) Time spent starting thread: real 1.770s user 0.000s sys 0.035s
W0821 11:02:26.897984 917 inbound_call.cc:111] 0x000050403bda1400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39665 => 10.0.0.2:7100 (request call id 26230): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39665 => 10.0.0.2:7100 (request call id 26230) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:27.778136 3813706 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:37159 => 10.0.0.2:7100 (request call id 26234) took 10490ms (client timeout 3000ms).
W0821 11:02:33.236389 913 periodic.cc:174] PeriodicTimer callback delayed by 1.851s
W0821 11:02:37.581441 911 periodic.cc:174] PeriodicTimer callback delayed by 2.973s
W0821 11:02:43.647607 910 periodic.cc:174] PeriodicTimer callback delayed by 2.097s
W0821 11:02:43.798156 917 inbound_call.cc:111] 0x0000504039bf3400 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:42213 => 10.0.0.2:7100 (request call id 8622222): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:42213 => 10.0.0.2:7100 (request call id 8622222) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:02:50.632452 910 periodic.cc:174] PeriodicTimer callback delayed by 2.145s
W0821 11:02:47.538486 3813706 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:45139 => 10.0.0.2:7100 (request call id 26232) took 4753ms (client timeout 3000ms).
W0821 11:02:55.209833 915 thread.cc:888] Master_2340_worker (Master) Time spent waiting for new thread to publish its TID: real 0.648s user 0.000s sys 0.025s
W0821 11:02:55.209672 914 thread.cc:888] Master_2341_worker (Master) Time spent waiting for new thread to publish its TID: real 0.649s user 0.000s sys 0.025s
W0821 11:02:58.106760 915 thread.cc:844] Master_2340_worker (Master) Time spent starting thread: real 2.976s user 0.000s sys 0.055s
W0821 11:02:58.107008 914 thread.cc:844] Master_2341_worker (Master) Time spent starting thread: real 2.976s user 0.000s sys 0.054s
W0821 11:02:59.807919 911 periodic.cc:174] PeriodicTimer callback delayed by 4.112s
W0821 11:02:59.752193 913 thread.cc:850] CatalogManagerBGTasks_5803_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.796s user 0.000s sys 0.016s
W0821 11:02:57.900460 916 inbound_call.cc:111] 0x000050403bc19000 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:37159 => 10.0.0.2:7100 (request call id 26234): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:37159 => 10.0.0.2:7100 (request call id 26234) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:03:01.430298 913 thread.cc:844] CatalogManagerBGTasks_5803_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.751s user 0.000s sys 0.042s
W0821 11:03:03.213711 3813699 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.2:7100]) id (55d29a1415af4e44ac228b2c8a82f866): GetMasterRegistration RPC (request call id 313051) to 10.0.0.2:7100 timed out after 1.500s
W0821 11:03:06.696552 910 periodic.cc:174] PeriodicTimer callback delayed by 2.099s
W0821 11:03:14.373862 912 periodic.cc:174] PeriodicTimer callback delayed by 3.089s
W0821 11:03:20.615633 917 thread.cc:850] Master-high-pri_496_worker (Master-high-pri) Time spent creating pthread: real 0.679s user 0.000s sys 0.011s
W0821 11:03:21.485607 912 periodic.cc:174] PeriodicTimer callback delayed by 2.253s
W0821 11:03:21.926445 917 thread.cc:844] Master-high-pri_496_worker (Master-high-pri) Time spent starting thread: real 2.199s user 0.000s sys 0.038s
I0821 11:02:28.338930 3813700 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:03:21.502274 3813731 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x000050403b11fa60) client 10.0.0.2:44367 => 10.0.0.4:7100, passed: 6.176s, timeout: 15.000s: kConnectFailed (network error 1)
I0821 11:03:24.650346 3813700 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:03:25.348013 916 thread.cc:844] Master-high-pri_497_worker (Master-high-pri) Time spent starting thread: real 2.642s user 0.000s sys 0.034s
W0821 11:03:23.009896 3813731 outbound_call.cc:526] RPC callback for RPC call 0x000050403bffa380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313050, state: FINISHED_ERROR, transfer_state: PENDING took 1.494s
W0821 11:03:25.091840 3813739 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x0000504038e24560) client 10.0.0.2:42457 => 10.0.0.3:7100, passed: 7.697s, timeout: 15.000s: kConnectFailed (network error 1)
W0821 11:03:25.120307 3813715 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:36867 => 10.0.0.2:7100 (request call id 8622151) took 35261ms (client timeout 15000ms).
W0821 11:03:27.634564 911 periodic.cc:174] PeriodicTimer callback delayed by 2.146s
I0821 11:03:28.505079 3813739 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:03:25.992082 3813714 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:45505 => 10.0.0.2:7100 (request call id 8622170) took 36036ms (client timeout 15000ms).
W0821 11:03:32.309971 3813739 thread.cc:850] raft_19704_worker (raft) Time spent creating pthread: real 0.564s user 0.000s sys 0.010s
I0821 11:03:32.585736 3813700 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:03:33.273207 3813739 thread.cc:844] raft_19704_worker (raft) Time spent starting thread: real 1.642s user 0.000s sys 0.029s
W0821 11:03:33.273217 913 thread.cc:844] CatalogManagerBGTasks_5804_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.587s user 0.000s sys 0.020s
W0821 11:03:33.334636 911 periodic.cc:174] PeriodicTimer callback delayed by 1.901s
I0821 11:03:33.273229 3813741 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 16.557s
W0821 11:03:34.771497 914 inbound_call.cc:111] 0x000050403fce9c00 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:36867 => 10.0.0.2:7100 (request call id 8622151): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:36867 => 10.0.0.2:7100 (request call id 8622151) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:03:35.756454 3813741 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:03:39.072383 910 periodic.cc:174] PeriodicTimer callback delayed by 1.477s
W0821 11:03:38.849082 3813739 outbound_call.cc:526] RPC callback for RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313049, state: FINISHED_ERROR, transfer_state: PENDING took 13.660s
W0821 11:03:43.208612 910 periodic.cc:174] PeriodicTimer callback delayed by 1.914s
W0821 11:03:47.384661 913 periodic.cc:174] PeriodicTimer callback delayed by 1.933s
W0821 11:03:50.520627 913 periodic.cc:174] PeriodicTimer callback delayed by 0.865s
W0821 11:03:54.283068 910 periodic.cc:174] PeriodicTimer callback delayed by 1.998s
W0821 11:03:58.801424 910 thread.cc:888] raft_19705_worker (raft) Time spent waiting for new thread to publish its TID: real 0.503s user 0.000s sys 0.024s
I0821 11:03:59.059885 3813755 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:04:01.777405 910 thread.cc:844] raft_19705_worker (raft) Time spent starting thread: real 3.235s user 0.000s sys 0.069s
I0821 11:04:01.209429 3813755 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 11:03:59.867832 915 inbound_call.cc:111] 0x000050403dad0c00 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:45505 => 10.0.0.2:7100 (request call id 8622170): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:45505 => 10.0.0.2:7100 (request call id 8622170) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:04:04.241096 3813755 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.061s
W0821 11:04:07.865834 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403bef3180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313053, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97916.398s (30.026s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 97936.227s (10.198s ago), now: 97946.425s, connection: Connection (0x0000504038e38560) client 10.0.0.2:36061 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:04:07.662289 912 thread.cc:850] CatalogManagerBGTasks_5805_worker (CatalogManagerBGTasks) Time spent creating pthread: real 1.483s user 0.000s sys 0.023s
W0821 11:04:10.081926 911 periodic.cc:174] PeriodicTimer callback delayed by 10.591s
W0821 11:04:10.235694 912 thread.cc:844] CatalogManagerBGTasks_5805_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.443s user 0.000s sys 0.066s
W0821 11:04:14.408943 913 periodic.cc:174] PeriodicTimer callback delayed by 2.103s
W0821 11:04:14.356005 914 thread.cc:844] Master-high-pri_498_worker (Master-high-pri) Time spent starting thread: real 1.605s user 0.000s sys 0.027s
W0821 11:04:13.840797 3813769 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313053) to 10.0.0.3:7100 timed out after 6.229s
W0821 11:04:17.529773 913 periodic.cc:174] PeriodicTimer callback delayed by 2.234s
W0821 11:04:16.723100 3813769 outbound_call.cc:526] RPC callback for RPC call 0x000050403bef3180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313053, state: TIMED_OUT, transfer_state: PENDING took 2.767s
W0821 11:04:25.064146 910 periodic.cc:174] PeriodicTimer callback delayed by 2.577s
W0821 11:04:31.528324 912 periodic.cc:174] PeriodicTimer callback delayed by 2.465s
I0821 11:04:06.171029 3813755 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:04:39.280469 910 periodic.cc:174] PeriodicTimer callback delayed by 2.782s
I0821 11:04:39.280002 3813755 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:04:43.171653 912 thread.cc:850] CatalogManagerBGTasks_5806_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.596s user 0.000s sys 0.012s
W0821 11:04:45.783578 912 thread.cc:844] CatalogManagerBGTasks_5806_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.386s user 0.000s sys 0.050s
W0821 11:04:47.515661 912 periodic.cc:174] PeriodicTimer callback delayed by 3.202s
W0821 11:04:55.519313 910 periodic.cc:174] PeriodicTimer callback delayed by 4.143s
I0821 11:05:07.318871 3813755 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:05:12.022342 910 periodic.cc:174] PeriodicTimer callback delayed by 7.259s
W0821 11:05:13.907207 915 thread.cc:844] Master_2343_worker (Master) Time spent starting thread: real 0.671s user 0.000s sys 0.012s
W0821 11:05:14.979735 3813757 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.2:7100]) id (55d29a1415af4e44ac228b2c8a82f866): GetMasterRegistration RPC (request call id 313057) to 10.0.0.2:7100 timed out after 1.500s
I0821 11:05:15.961390 3813815 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:05:16.005235 910 thread.cc:844] raft_19706_worker (raft) Time spent starting thread: real 0.932s user 0.000s sys 0.029s
I0821 11:05:17.789511 3813815 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 11:05:19.996153 913 thread.cc:850] CatalogManagerBGTasks_5807_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.586s user 0.000s sys 0.009s
I0821 11:05:19.991333 3813815 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 16.279s
W0821 11:05:22.628809 910 periodic.cc:174] PeriodicTimer callback delayed by 6.589s
W0821 11:05:23.304754 913 thread.cc:844] CatalogManagerBGTasks_5807_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.574s user 0.000s sys 0.051s
W0821 11:05:27.109311 913 periodic.cc:174] PeriodicTimer callback delayed by 2.153s
W0821 11:05:33.371265 912 periodic.cc:174] PeriodicTimer callback delayed by 4.035s
W0821 11:05:36.526768 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403e342e00: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.2:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313055, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97932.115s (102.285s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98019.848s (14.551s ago), now: 98034.400s, connection: Connection (0x000050403b8f5520) client 10.0.0.2:45151 => 10.0.0.2:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:05:38.110693 911 periodic.cc:174] PeriodicTimer callback delayed by 2.471s
W0821 11:05:41.076360 911 periodic.cc:174] PeriodicTimer callback delayed by 2.194s
W0821 11:05:43.853592 914 thread.cc:850] Master_2344_worker (Master) Time spent creating pthread: real 0.926s user 0.000s sys 0.014s
W0821 11:05:43.958127 914 thread.cc:844] Master_2344_worker (Master) Time spent starting thread: real 2.743s user 0.000s sys 0.040s
W0821 11:05:43.965996 3813744 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.2:7100]) id (55d29a1415af4e44ac228b2c8a82f866): GetMasterRegistration RPC (request call id 313055) to 10.0.0.2:7100 timed out after 1.500s
W0821 11:05:44.770948 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313054, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97922.510s (130.481s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98024.393s (28.599s ago), now: 98052.992s, connection: Connection (0x000050403b81a020) client 10.0.0.2:37813 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:05:44.802753 3813824 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313054) to 10.0.0.4:7100 timed out after 6.229s
I0821 11:05:44.833734 3813824 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:05:22.573239 3813815 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:05:44.925877 3813824 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313054, state: TIMED_OUT, transfer_state: PENDING took 0.123s
I0821 11:05:45.008598 3813829 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.654s
I0821 11:05:45.004590 3813815 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:05:45.037567 3813829 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:05:45.100317 3813815 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:05:45.383205 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403bff2a80: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313052, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97895.301s (158.954s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98021.682s (32.573s ago), now: 98054.255s, connection: Connection (0x0000504038e4fc20) client 10.0.0.2:41111 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:05:47.151736 912 periodic.cc:174] PeriodicTimer callback delayed by 1.793s
W0821 11:05:50.553144 912 periodic.cc:174] PeriodicTimer callback delayed by 1.742s
W0821 11:05:55.888438 912 thread.cc:844] raft_19708_worker (raft) Time spent starting thread: real 2.858s user 0.000s sys 0.068s
W0821 11:05:56.423359 913 thread.cc:844] CatalogManagerBGTasks_5808_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.377s user 0.000s sys 0.035s
I0821 11:05:55.582849 3813848 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:06:00.819607 3813848 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:06:02.197701 3813848 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 15.268s
I0821 11:06:02.199739 3813848 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:06:02.203606 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403bf35500: yb.master.MasterService.GetMasterRegistration -> { remote: 10.0.0.3:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313061, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98053.741s (17.467s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98055.422s (15.786s ago), now: 98071.209s, connection: Connection (0x0000504038e4fa60) client 10.0.0.2:39329 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 11:06:02.204481 3813848 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:06:02.206332 3813848 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:06:02.211216 3813744 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313061) to 10.0.0.3:7100 timed out after 1.500s
W0821 11:06:02.271821 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313056, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 97990.404s (80.873s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98027.471s (43.806s ago), now: 98071.278s, connection: Connection (0x000050403b818e20) client 10.0.0.2:41237 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:06:02.273258 3813903 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313056) to 10.0.0.3:7100 timed out after 9.061s
W0821 11:06:02.274097 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403e4d2000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313059, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98017.713s (53.566s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98029.146s (42.133s ago), now: 98071.280s, connection: Connection (0x000050403b924c60) client 10.0.0.2:45439 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:06:02.275084 3813903 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313059) to 10.0.0.4:7100 timed out after 9.061s
I0821 11:06:02.275096 3813903 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:06:02.274370 3813901 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)
I0821 11:06:02.276026 3813901 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:06:02.276146 3813900 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)
I0821 11:06:02.276152 3813900 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:06:02.276228 3813902 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)
I0821 11:06:02.276235 3813902 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:06:02.276576 3813848 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 11.386s
I0821 11:06:02.276593 3813848 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:06:02.295847 3813900 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)
I0821 11:06:02.295862 3813900 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29069. Preceding OpId from leader: 29.29093. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:06:02.296058 3813902 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223355431.142s
I0821 11:06:02.298472 3813902 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223355431.141s
W0821 11:06:03.520313 910 periodic.cc:174] PeriodicTimer callback delayed by 8.514s
W0821 11:06:03.714187 3813699 master-path-handlers.cc:237] Not found (yb/master/master-path-handlers.cc:277): Unable to locate leader master to redirect this request: /
W0821 11:06:03.772035 3813744 master-path-handlers.cc:237] Not found (yb/master/master-path-handlers.cc:277): Unable to locate leader master to redirect this request: /
W0821 11:06:03.815189 3813757 master-path-handlers.cc:237] Not found (yb/master/master-path-handlers.cc:277): Unable to locate leader master to redirect this request: /
W0821 11:06:08.293110 911 periodic.cc:174] PeriodicTimer callback delayed by 0.248s
W0821 11:06:10.129231 913 periodic.cc:174] PeriodicTimer callback delayed by 1.304s
W0821 11:06:11.411420 1039 long_operation_tracker.cc:125] UpdateReplica running for 1.033s in thread 3813900:
Thread did not respond: maybe it was in the middle of a stack trace collection
I0821 11:06:12.443517 3813997 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:06:12.494830 913 thread.cc:844] raft_19709_worker (raft) Time spent starting thread: real 0.982s user 0.000s sys 0.032s
W0821 11:06:12.075233 3813900 long_operation_tracker.cc:155] UpdateReplica took a long time: 1.775s
I0821 11:06:12.596498 3813997 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:06:12.600212 3813997 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.809s
W0821 11:06:12.612372 3813900 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:42807 => 10.0.0.2:7100 (request call id 26504) took 4463ms (client timeout 3000ms).
W0821 11:06:15.782315 910 periodic.cc:174] PeriodicTimer callback delayed by 2.632s
W0821 11:06:16.601859 917 thread.cc:844] Master_2348_worker (Master) Time spent starting thread: real 1.854s user 0.000s sys 0.037s
W0821 11:06:19.500337 913 periodic.cc:174] PeriodicTimer callback delayed by 1.470s
W0821 11:06:24.342640 913 periodic.cc:174] PeriodicTimer callback delayed by 2.561s
W0821 11:06:26.192066 916 thread.cc:850] Master-high-pri_504_worker (Master-high-pri) Time spent creating pthread: real 1.147s user 0.000s sys 0.020s
W0821 11:06:28.971287 913 thread.cc:844] CatalogManagerBGTasks_5809_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.998s user 0.000s sys 0.012s
W0821 11:06:28.948938 916 thread.cc:844] Master-high-pri_504_worker (Master-high-pri) Time spent starting thread: real 3.769s user 0.000s sys 0.065s
W0821 11:06:27.834244 3814014 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x000050403b830fe0) client 10.0.0.2:44147 => 10.0.0.4:7100, passed: 15.304s, timeout: 15.000s: kConnectFailed (network error 1)
W0821 11:06:30.434240 911 periodic.cc:174] PeriodicTimer callback delayed by 1.923s
I0821 11:06:31.014030 3814014 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:06:34.301638 3814014 thread.cc:850] raft_19710_worker (raft) Time spent creating pthread: real 0.519s user 0.000s sys 0.009s
W0821 11:06:36.062695 3814014 thread.cc:844] raft_19710_worker (raft) Time spent starting thread: real 3.081s user 0.002s sys 0.065s
W0821 11:06:37.452582 910 periodic.cc:174] PeriodicTimer callback delayed by 1.925s
W0821 11:06:37.837649 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x0000504037b9ea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313066, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98071.213s (26.763s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98086.527s (11.449s ago), now: 98097.977s, connection: Connection (0x0000504038e60560) client 10.0.0.2:44469 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:06:38.498478 3814014 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf9aa80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313063, state: FINISHED_ERROR, transfer_state: PENDING took 10.673s
W0821 11:06:44.169131 910 periodic.cc:174] PeriodicTimer callback delayed by 2.244s
W0821 11:06:45.672374 3814020 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313066) to 10.0.0.4:7100 timed out after 15.268s
W0821 11:06:47.409994 914 thread.cc:888] Master-high-pri_505_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 2.012s user 0.000s sys 0.097s
I0821 11:06:48.962100 3814020 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:06:49.448254 914 thread.cc:844] Master-high-pri_505_worker (Master-high-pri) Time spent starting thread: real 6.559s user 0.000s sys 0.197s
W0821 11:06:50.431282 910 periodic.cc:174] PeriodicTimer callback delayed by 1.804s
W0821 11:06:48.727214 915 yb_rpc.cc:553] Connection (0x0000504038f2ae20) client 10.0.0.2:45427 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 21.123s, timeout: 15.000s, now: 98097.978s, last_read_time_: 98076.854s
W0821 11:06:51.789232 916 yb_rpc.cc:553] Connection (0x0000504038fcac60) client 10.0.0.2:35507 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 31.678s, timeout: 15.000s, now: 98113.300s, last_read_time_: 98081.621s
W0821 11:06:54.735394 3814020 thread.cc:844] raft_19711_worker (raft) Time spent starting thread: real 2.006s user 0.001s sys 0.047s
W0821 11:06:55.672961 910 periodic.cc:174] PeriodicTimer callback delayed by 1.356s
W0821 11:06:57.305716 3814020 outbound_call.cc:526] RPC callback for RPC call 0x0000504037b9ea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313066, state: TIMED_OUT, transfer_state: PENDING took 11.650s
I0821 11:06:12.605067 3813997 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:07:02.334261 912 periodic.cc:174] PeriodicTimer callback delayed by 1.563s
W0821 11:07:01.771409 914 yb_rpc.cc:553] Connection (0x000050403b9836e0) client 10.0.0.2:42055 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 19.170s, timeout: 15.000s, now: 98097.977s, last_read_time_: 98078.806s
I0821 11:07:03.473075 3813997 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:07:04.631560 3813900 long_operation_tracker.cc:155] UpdateReplica took a long time: 37.673s
W0821 11:07:04.271371 3814001 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:36555 => 10.0.0.2:7100 (request call id 8633691) took 51611ms (client timeout 60000ms).
W0821 11:07:03.829880 3813888 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:34569 => 10.0.0.2:7100 (request call id 8633682) took 51156ms (client timeout 15000ms).
W0821 11:07:04.271430 3814009 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:41443 => 10.0.0.2:7100 (request call id 8633787) took 51524ms (client timeout 15000ms).
I0821 11:07:04.729177 3814018 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.966s
I0821 11:07:06.701061 3814018 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:07:06.843284 3814024 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.983s
W0821 11:07:06.441097 3813900 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44093 => 10.0.0.2:7100 (request call id 26507) took 47074ms (client timeout 3000ms).
I0821 11:07:08.583047 3814024 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:07:08.966499 913 periodic.cc:174] PeriodicTimer callback delayed by 1.866s
I0821 11:07:09.459102 3813997 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:07:09.927812 916 yb_rpc.cc:553] Connection (0x000050403b8308e0) client 10.0.0.2:35815 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 31.678s, timeout: 15.000s, now: 98113.300s, last_read_time_: 98081.621s
W0821 11:07:12.808008 912 periodic.cc:174] PeriodicTimer callback delayed by 1.590s
W0821 11:07:16.523234 912 periodic.cc:174] PeriodicTimer callback delayed by 1.486s
W0821 11:07:20.185359 913 periodic.cc:174] PeriodicTimer callback delayed by 1.656s
W0821 11:07:23.969951 913 thread.cc:844] raft_19712_worker (raft) Time spent starting thread: real 1.735s user 0.000s sys 0.018s
I0821 11:07:23.874071 3814045 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:07:26.314699 3814045 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:07:26.643649 3814045 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 8.713s
W0821 11:07:28.749317 911 periodic.cc:174] PeriodicTimer callback delayed by 4.780s
W0821 11:07:29.273415 916 yb_rpc.cc:553] Connection (0x000050403b815520) client 10.0.0.2:40103 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 28.858s, timeout: 15.000s, now: 98113.300s, last_read_time_: 98084.441s
I0821 11:07:30.979460 3814045 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:07:32.472944 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:07:32.472996 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:07:32.530483 3814045 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:07:35.169447 910 periodic.cc:174] PeriodicTimer callback delayed by 1.930s
W0821 11:07:36.463022 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:07:36.463132 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:07:37.473022 3814045 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:07:39.477694 910 periodic.cc:174] PeriodicTimer callback delayed by 2.057s
W0821 11:07:43.263006 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 11:07:43.263218 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:07:43.254612 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:07:46.582506 911 periodic.cc:174] PeriodicTimer callback delayed by 2.662s
I0821 11:07:48.222220 3814134 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:07:49.971760 911 thread.cc:888] raft_19713_worker (raft) Time spent waiting for new thread to publish its TID: real 0.856s user 0.000s sys 0.024s
I0821 11:07:50.776207 3814134 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 11:07:53.121132 911 thread.cc:844] raft_19713_worker (raft) Time spent starting thread: real 5.486s user 0.000s sys 0.059s
I0821 11:07:54.634214 3814134 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 15.953s
W0821 11:07:53.650612 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:07:55.556308 913 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39471 => 10.0.0.2:7100 (request call id 26512) took 8538ms (client timeout 3000ms).
W0821 11:07:56.375298 914 inbound_call.cc:111] 0x00005040375cd400 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:34569 => 10.0.0.2:7100 (request call id 8633682): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:34569 => 10.0.0.2:7100 (request call id 8633682) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:07:58.554188 911 periodic.cc:174] PeriodicTimer callback delayed by 8.750s
W0821 11:07:59.432490 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:08:02.280598 911 periodic.cc:174] PeriodicTimer callback delayed by 1.488s
W0821 11:08:04.945263 913 thread.cc:844] CatalogManagerBGTasks_5812_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.139s user 0.000s sys 0.020s
W0821 11:08:05.588230 910 periodic.cc:174] PeriodicTimer callback delayed by 1.028s
W0821 11:08:10.599080 910 periodic.cc:174] PeriodicTimer callback delayed by 2.785s
W0821 11:08:11.507584 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:08:14.968700 913 periodic.cc:174] PeriodicTimer callback delayed by 3.944s
W0821 11:08:15.138562 1039 long_operation_tracker.cc:125] UpdateReplica running for 1.446s in thread 3814143:
Thread did not respond: maybe it is blocking signals
W0821 11:08:17.928599 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:08:19.513026 914 inbound_call.cc:111] 0x0000504037d71400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39471 => 10.0.0.2:7100 (request call id 26512): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:39471 => 10.0.0.2:7100 (request call id 26512) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:08:21.753607 915 thread.cc:850] Master-high-pri_507_worker (Master-high-pri) Time spent creating pthread: real 0.531s user 0.000s sys 0.009s
W0821 11:08:19.565130 912 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33103 => 10.0.0.2:7100 (request call id 26515) took 9353ms (client timeout 3000ms).
W0821 11:08:24.714573 915 thread.cc:844] Master-high-pri_507_worker (Master-high-pri) Time spent starting thread: real 3.516s user 0.000s sys 0.053s
W0821 11:08:23.988013 911 periodic.cc:174] PeriodicTimer callback delayed by 3.714s
W0821 11:08:31.902597 912 periodic.cc:174] PeriodicTimer callback delayed by 2.746s
W0821 11:08:31.871344 917 inbound_call.cc:111] 0x000050403bd73400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44093 => 10.0.0.2:7100 (request call id 26507): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44093 => 10.0.0.2:7100 (request call id 26507) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:07:55.019104 3814134 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:08:36.917029 911 periodic.cc:174] PeriodicTimer callback delayed by 0.963s
I0821 11:08:38.555207 3814134 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:08:39.181005 3814143 long_operation_tracker.cc:155] UpdateReplica took a long time: 26.672s
W0821 11:08:40.040436 910 thread.cc:844] CatalogManagerBGTasks_5813_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.423s user 0.000s sys 0.028s
W0821 11:08:43.635259 911 periodic.cc:174] PeriodicTimer callback delayed by 2.501s
W0821 11:08:42.996711 3814143 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38663 => 10.0.0.2:7100 (request call id 26517) took 38871ms (client timeout 3000ms).
I0821 11:08:47.368927 3814134 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:08:51.964622 911 periodic.cc:174] PeriodicTimer callback delayed by 3.068s
W0821 11:08:59.783195 911 periodic.cc:174] PeriodicTimer callback delayed by 2.346s
W0821 11:09:05.179459 911 thread.cc:850] raft_19714_worker (raft) Time spent creating pthread: real 0.729s user 0.000s sys 0.014s
W0821 11:09:05.799984 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x0000504037b9ea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313072, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98139.310s (100.441s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98171.269s (68.482s ago), now: 98239.751s, connection: Connection (0x0000504038f1cfe0) client 10.0.0.2:38763 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:09:04.659638 915 inbound_call.cc:111] 0x00005040377ff000 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33103 => 10.0.0.2:7100 (request call id 26515): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33103 => 10.0.0.2:7100 (request call id 26515) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:09:07.198763 911 thread.cc:844] raft_19714_worker (raft) Time spent starting thread: real 4.193s user 0.000s sys 0.067s
I0821 11:09:07.032331 3814173 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:09:08.876245 3814173 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:09:10.917297 3814173 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 16.208s
W0821 11:09:12.254446 914 thread.cc:844] Master-high-pri_508_worker (Master-high-pri) Time spent starting thread: real 1.777s user 0.000s sys 0.026s
W0821 11:09:11.640363 3814177 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313072) to 10.0.0.4:7100 timed out after 9.809s
W0821 11:09:13.642630 910 periodic.cc:174] PeriodicTimer callback delayed by 9.296s
W0821 11:09:14.170923 911 thread.cc:844] CatalogManagerBGTasks_5814_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.277s user 0.000s sys 0.020s
W0821 11:09:17.442107 911 periodic.cc:174] PeriodicTimer callback delayed by 1.522s
W0821 11:09:14.171015 3814177 outbound_call.cc:526] RPC callback for RPC call 0x0000504037b9ea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313072, state: TIMED_OUT, transfer_state: PENDING took 2.542s
W0821 11:09:21.632661 911 periodic.cc:174] PeriodicTimer callback delayed by 1.951s
W0821 11:09:26.541203 912 periodic.cc:174] PeriodicTimer callback delayed by 2.589s
W0821 11:09:29.843176 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403f88ee00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313071, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98134.729s (133.088s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98171.269s (96.548s ago), now: 98267.817s, connection: Connection (0x000050403b15d1a0) client 10.0.0.2:35893 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:09:29.821878 912 periodic.cc:174] PeriodicTimer callback delayed by 1.318s
W0821 11:09:32.906237 3814177 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313071) to 10.0.0.3:7100 timed out after 9.809s
I0821 11:09:35.478318 3814177 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:09:36.871881 910 periodic.cc:174] PeriodicTimer callback delayed by 2.828s
W0821 11:09:37.386904 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313073, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98164.244s (113.035s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98230.111s (47.167s ago), now: 98277.279s, connection: Connection (0x000050403b148560) client 10.0.0.2:42827 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:09:43.565992 914 thread.cc:888] Master-high-pri_509_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 1.488s user 0.000s sys 0.042s
W0821 11:09:44.093878 913 periodic.cc:174] PeriodicTimer callback delayed by 2.222s
W0821 11:09:43.566745 3814182 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313073) to 10.0.0.3:7100 timed out after 8.713s
W0821 11:09:45.751540 914 thread.cc:844] Master-high-pri_509_worker (Master-high-pri) Time spent starting thread: real 5.022s user 0.000s sys 0.107s
W0821 11:09:45.585135 3814177 outbound_call.cc:526] RPC callback for RPC call 0x000050403f88ee00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313071, state: TIMED_OUT, transfer_state: PENDING took 12.676s
W0821 11:09:46.376596 3814182 outbound_call.cc:526] RPC callback for RPC call 0x000050403eed7180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313073, state: TIMED_OUT, transfer_state: PENDING took 2.666s
W0821 11:09:51.970448 913 periodic.cc:174] PeriodicTimer callback delayed by 2.545s
W0821 11:09:52.718873 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bffbc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313074, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98167.627s (122.582s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98238.970s (51.239s ago), now: 98290.210s, connection: Connection (0x000050403b1d48e0) client 10.0.0.2:46055 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:09:52.661151 912 thread.cc:888] CatalogManagerBGTasks_5815_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 2.925s user 0.000s sys 0.094s
W0821 11:09:55.864972 912 thread.cc:844] CatalogManagerBGTasks_5815_worker (CatalogManagerBGTasks) Time spent starting thread: real 6.613s user 0.000s sys 0.172s
W0821 11:09:56.978392 3814177 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313074) to 10.0.0.4:7100 timed out after 8.713s
W0821 11:09:59.270749 913 periodic.cc:174] PeriodicTimer callback delayed by 2.148s
I0821 11:10:00.524000 3814177 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:10:01.264289 915 inbound_call.cc:111] 0x000050403e486400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38663 => 10.0.0.2:7100 (request call id 26517): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38663 => 10.0.0.2:7100 (request call id 26517) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:10:06.516311 3814177 thread.cc:850] raft_19716_worker (raft) Time spent creating pthread: real 0.943s user 0.000s sys 0.020s
W0821 11:10:07.055874 913 periodic.cc:174] PeriodicTimer callback delayed by 2.241s
W0821 11:10:09.657372 3814177 thread.cc:844] raft_19716_worker (raft) Time spent starting thread: real 4.622s user 0.002s sys 0.094s
W0821 11:10:13.904675 912 periodic.cc:174] PeriodicTimer callback delayed by 2.305s
W0821 11:10:20.328512 910 periodic.cc:174] PeriodicTimer callback delayed by 2.293s
W0821 11:10:19.167975 3814177 outbound_call.cc:526] RPC callback for RPC call 0x000050403bffbc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313074, state: TIMED_OUT, transfer_state: PENDING took 22.085s
W0821 11:10:26.168552 911 thread.cc:850] CatalogManagerBGTasks_5816_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.846s user 0.000s sys 0.016s
W0821 11:10:27.409209 910 periodic.cc:174] PeriodicTimer callback delayed by 2.536s
W0821 11:10:30.298395 911 thread.cc:844] CatalogManagerBGTasks_5816_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.834s user 0.000s sys 0.076s
I0821 11:09:12.286834 3814173 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:10:36.141665 913 periodic.cc:174] PeriodicTimer callback delayed by 2.897s
I0821 11:10:39.064092 3814173 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:10:39.185762 3814181 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 17.025s
W0821 11:10:43.550714 912 periodic.cc:174] PeriodicTimer callback delayed by 2.482s
I0821 11:10:45.783596 3814186 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 14.244s
I0821 11:10:45.759624 3814181 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:10:50.412298 913 periodic.cc:174] PeriodicTimer callback delayed by 2.157s
I0821 11:10:50.076879 3814186 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:10:51.219228 3814173 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:10:53.518990 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bef3180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313075, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98230.257s (119.301s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98325.519s (24.039s ago), now: 98349.558s, connection: Connection (0x000050403b93ba60) client 10.0.0.2:40967 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:10:55.361068 912 periodic.cc:174] PeriodicTimer callback delayed by 2.687s
W0821 11:11:00.516644 912 periodic.cc:174] PeriodicTimer callback delayed by 2.879s
W0821 11:11:01.917668 3814194 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313075) to 10.0.0.3:7100 timed out after 15.953s
W0821 11:11:03.240108 915 thread.cc:844] Master-high-pri_510_worker (Master-high-pri) Time spent starting thread: real 2.744s user 0.000s sys 0.091s
W0821 11:11:03.569633 913 thread.cc:850] CatalogManagerBGTasks_5817_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.549s user 0.000s sys 0.015s
W0821 11:11:05.194746 912 periodic.cc:174] PeriodicTimer callback delayed by 2.463s
W0821 11:11:07.481494 913 thread.cc:844] CatalogManagerBGTasks_5817_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.907s user 0.000s sys 0.132s
W0821 11:11:09.562276 912 thread.cc:850] raft_19717_worker (raft) Time spent creating pthread: real 0.617s user 0.000s sys 0.015s
W0821 11:11:06.117581 3814194 outbound_call.cc:526] RPC callback for RPC call 0x000050403bef3180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313075, state: TIMED_OUT, transfer_state: PENDING took 4.014s
I0821 11:11:12.351394 3814199 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:11:12.666430 912 thread.cc:844] raft_19717_worker (raft) Time spent starting thread: real 4.310s user 0.000s sys 0.128s
I0821 11:11:14.711081 3814199 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:11:18.148217 3814199 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 19.160s
W0821 11:11:20.379956 911 periodic.cc:174] PeriodicTimer callback delayed by 10.264s
W0821 11:11:29.204141 911 periodic.cc:174] PeriodicTimer callback delayed by 3.392s
W0821 11:11:34.478140 910 periodic.cc:174] PeriodicTimer callback delayed by 2.928s
I0821 11:11:24.386411 3814199 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:11:37.771960 3814199 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:11:37.773474 3814199 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:11:37.797093 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403f88ee00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313078, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98362.904s (43.898s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98404.480s (2.322s ago), now: 98406.802s, connection: Connection (0x000050403b915a60) client 10.0.0.2:32987 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:11:37.798314 3814263 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313078) to 10.0.0.4:7100 timed out after 16.208s
I0821 11:11:37.834246 3814269 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223355095.605s
I0821 11:11:37.839996 3814269 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223355095.598s
I0821 11:11:37.840063 3814269 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:11:37.840260 3814199 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 7.695s
I0821 11:11:37.840286 3814199 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:11:39.812186 912 periodic.cc:174] PeriodicTimer callback delayed by 3.194s
W0821 11:11:42.256798 912 periodic.cc:174] PeriodicTimer callback delayed by 0.159s
W0821 11:11:44.887671 912 periodic.cc:174] PeriodicTimer callback delayed by 1.279s
W0821 11:11:48.661808 912 thread.cc:844] raft_19718_worker (raft) Time spent starting thread: real 0.707s user 0.000s sys 0.026s
I0821 11:11:48.574501 3814356 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:11:47.894595 3814269 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38429 => 10.0.0.2:7100 (request call id 26724) took 4478ms (client timeout 3000ms).
I0821 11:11:51.950551 3814356 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 11:11:54.925846 910 periodic.cc:174] PeriodicTimer callback delayed by 5.599s
I0821 11:11:55.096478 3814356 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 11.529s
W0821 11:12:02.382982 913 periodic.cc:174] PeriodicTimer callback delayed by 2.420s
W0821 11:12:06.220212 911 periodic.cc:174] PeriodicTimer callback delayed by 1.614s
W0821 11:12:09.990645 912 periodic.cc:174] PeriodicTimer callback delayed by 1.528s
W0821 11:12:13.792058 912 periodic.cc:174] PeriodicTimer callback delayed by 0.558s
W0821 11:12:16.778493 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403d710380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313076, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98240.339s (199.212s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98422.754s (16.796s ago), now: 98439.551s, connection: Connection (0x0000504039eab520) client 10.0.0.2:42431 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:12:20.212149 913 periodic.cc:174] PeriodicTimer callback delayed by 1.871s
W0821 11:12:25.361815 916 thread.cc:888] Master-high-pri_515_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 2.797s user 0.000s sys 0.150s
W0821 11:12:26.016508 3814369 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313076) to 10.0.0.4:7100 timed out after 15.953s
W0821 11:12:27.916970 912 periodic.cc:174] PeriodicTimer callback delayed by 2.881s
W0821 11:12:27.447746 916 thread.cc:844] Master-high-pri_515_worker (Master-high-pri) Time spent starting thread: real 5.496s user 0.000s sys 0.196s
I0821 11:12:29.165911 3814369 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
W0821 11:12:33.324044 913 periodic.cc:174] PeriodicTimer callback delayed by 1.659s
W0821 11:12:32.670684 917 yb_rpc.cc:553] Connection (0x000050403b88e1e0) client 10.0.0.2:40229 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 27.242s, timeout: 15.000s, now: 98439.339s, last_read_time_: 98412.096s
W0821 11:12:38.983741 3814369 thread.cc:850] raft_19719_worker (raft) Time spent creating pthread: real 1.234s user 0.000s sys 0.025s
W0821 11:12:41.115175 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403e4d2000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313080, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98406.778s (47.545s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98425.978s (28.345s ago), now: 98454.324s, connection: Connection (0x000050403b914fe0) client 10.0.0.2:45549 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:12:41.322592 913 periodic.cc:174] PeriodicTimer callback delayed by 2.808s
W0821 11:12:41.717734 3814369 thread.cc:844] raft_19719_worker (raft) Time spent starting thread: real 5.823s user 0.002s sys 0.113s
W0821 11:12:43.646842 910 thread.cc:844] CatalogManagerBGTasks_5820_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.666s user 0.000s sys 0.026s
W0821 11:12:46.288153 912 periodic.cc:174] PeriodicTimer callback delayed by 1.232s
W0821 11:12:46.623634 914 thread.cc:850] Master-high-pri_516_worker (Master-high-pri) Time spent creating pthread: real 0.989s user 0.000s sys 0.017s
W0821 11:12:45.608084 3814369 outbound_call.cc:526] RPC callback for RPC call 0x000050403d710380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313076, state: TIMED_OUT, transfer_state: PENDING took 19.573s
W0821 11:12:49.904807 914 thread.cc:844] Master-high-pri_516_worker (Master-high-pri) Time spent starting thread: real 4.825s user 0.000s sys 0.084s
W0821 11:12:49.035789 3814380 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313080) to 10.0.0.4:7100 timed out after 19.160s
W0821 11:12:52.366215 913 periodic.cc:174] PeriodicTimer callback delayed by 2.135s
I0821 11:12:52.393915 3814380 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:11:57.912981 3814356 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:12:56.400413 3814356 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:12:56.408388 3814374 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 15.210s
W0821 11:12:58.387171 912 periodic.cc:174] PeriodicTimer callback delayed by 1.909s
W0821 11:13:00.167397 3814380 thread.cc:844] raft_19720_worker (raft) Time spent starting thread: real 3.117s user 0.000s sys 0.033s
I0821 11:13:00.460155 3814374 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:13:00.515560 3814382 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 13.923s
W0821 11:13:01.725780 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:13:04.056711 3814356 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 11:13:03.527853 3814382 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:13:06.459782 913 periodic.cc:174] PeriodicTimer callback delayed by 3.355s
W0821 11:13:03.779985 3814380 outbound_call.cc:526] RPC callback for RPC call 0x000050403e4d2000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313080, state: TIMED_OUT, transfer_state: PENDING took 14.613s
W0821 11:13:08.228298 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:12.603497 910 periodic.cc:174] PeriodicTimer callback delayed by 3.887s
W0821 11:13:11.919435 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:16.967980 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:17.497292 911 periodic.cc:174] PeriodicTimer callback delayed by 2.675s
W0821 11:13:18.466739 910 thread.cc:844] CatalogManagerBGTasks_5821_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.124s user 0.000s sys 0.036s
W0821 11:13:19.535897 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:22.273856 911 thread.cc:850] raft_19721_worker (raft) Time spent creating pthread: real 0.616s user 0.000s sys 0.004s
W0821 11:13:24.848012 911 thread.cc:844] raft_19721_worker (raft) Time spent starting thread: real 3.864s user 0.000s sys 0.028s
I0821 11:13:24.555064 3814402 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:13:30.614555 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:32.763195 911 periodic.cc:174] PeriodicTimer callback delayed by 10.961s
W0821 11:13:32.763286 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:32.763192 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:13:32.777199 3814402 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:13:35.837419 3814402 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 7.623s
W0821 11:13:40.333621 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:13:41.682543 913 periodic.cc:174] PeriodicTimer callback delayed by 3.062s
W0821 11:13:45.663342 912 periodic.cc:174] PeriodicTimer callback delayed by 1.834s
W0821 11:13:48.754248 911 periodic.cc:174] PeriodicTimer callback delayed by 2.620s
W0821 11:13:48.726378 914 thread.cc:844] Master_2351_worker (Master) Time spent starting thread: real 2.097s user 0.000s sys 0.050s
W0821 11:13:49.346696 917 thread.cc:888] Master-high-pri_517_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.512s user 0.000s sys 0.024s
W0821 11:13:43.558844 916 yb_rpc.cc:553] Connection (0x0000504038e60560) client 10.0.0.2:33149 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 27.397s, timeout: 15.000s, now: 98439.339s, last_read_time_: 98411.941s
W0821 11:13:51.557439 917 thread.cc:844] Master-high-pri_517_worker (Master-high-pri) Time spent starting thread: real 3.739s user 0.000s sys 0.077s
W0821 11:13:53.295017 913 thread.cc:850] CatalogManagerBGTasks_5822_worker (CatalogManagerBGTasks) Time spent creating pthread: real 1.892s user 0.000s sys 0.043s
W0821 11:13:55.442723 913 thread.cc:844] CatalogManagerBGTasks_5822_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.897s user 0.000s sys 0.084s
W0821 11:13:57.356761 913 periodic.cc:174] PeriodicTimer callback delayed by 3.455s
W0821 11:13:56.911911 915 thread.cc:888] Master-high-pri_518_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 1.099s user 0.000s sys 0.036s
W0821 11:13:50.022365 912 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33257 => 10.0.0.2:7100 (request call id 26733) took 6853ms (client timeout 3000ms).
W0821 11:13:58.753820 915 thread.cc:844] Master-high-pri_518_worker (Master-high-pri) Time spent starting thread: real 3.806s user 0.000s sys 0.071s
W0821 11:13:58.753826 914 thread.cc:888] Master_2352_worker (Master) Time spent waiting for new thread to publish its TID: real 0.938s user 0.000s sys 0.054s
W0821 11:14:00.655049 914 thread.cc:844] Master_2352_worker (Master) Time spent starting thread: real 3.984s user 0.000s sys 0.084s
W0821 11:13:57.812179 3814426 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38007 => 10.0.0.2:7100 (request call id 26731) took 10672ms (client timeout 3000ms).
W0821 11:14:03.712965 910 periodic.cc:174] PeriodicTimer callback delayed by 2.263s
W0821 11:14:04.058936 917 inbound_call.cc:111] 0x000050403cef8c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33257 => 10.0.0.2:7100 (request call id 26733): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:33257 => 10.0.0.2:7100 (request call id 26733) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:14:09.506480 914 thread.cc:844] Master_2353_worker (Master) Time spent starting thread: real 0.990s user 0.000s sys 0.017s
W0821 11:14:09.506520 915 thread.cc:844] Master_2354_worker (Master) Time spent starting thread: real 0.852s user 0.000s sys 0.014s
W0821 11:14:10.873322 910 periodic.cc:174] PeriodicTimer callback delayed by 2.475s
I0821 11:13:38.522413 3814402 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:14:13.039700 3814402 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:14:15.581830 915 thread.cc:844] Master_2355_worker (Master) Time spent starting thread: real 0.901s user 0.000s sys 0.012s
W0821 11:14:13.919796 3814416 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:36755 => 10.0.0.2:7100 (request call id 8646452) took 33633ms (client timeout 14998ms).
W0821 11:14:13.919811 3814428 yb_rpc.cc:388] Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:36755 => 10.0.0.2:7100 (request call id 8646839) took 22995ms (client timeout 15000ms).
W0821 11:14:16.733822 913 periodic.cc:174] PeriodicTimer callback delayed by 1.589s
I0821 11:14:16.016387 3814439 yb_rpc.cc:395] Call yb.tserver.TabletServerService.Read 10.0.0.3:46273 => 10.0.0.2:7100 (request call id 8646175) took 11142ms.
I0821 11:14:19.291970 3814402 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:14:17.967841 915 inbound_call.cc:111] 0x000050403e5f6400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38429 => 10.0.0.2:7100 (request call id 26724): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:38429 => 10.0.0.2:7100 (request call id 26724) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:14:23.270447 913 periodic.cc:174] PeriodicTimer callback delayed by 1.877s
W0821 11:14:28.189703 913 thread.cc:850] CatalogManagerBGTasks_5823_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.519s user 0.000s sys 0.015s
W0821 11:14:30.335589 913 thread.cc:844] CatalogManagerBGTasks_5823_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.074s user 0.000s sys 0.066s
W0821 11:14:32.356822 912 periodic.cc:174] PeriodicTimer callback delayed by 3.563s
W0821 11:14:30.472518 914 inbound_call.cc:111] 0x00005040375ce800 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39831 => 10.0.0.2:7100 (request call id 8646259): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:39831 => 10.0.0.2:7100 (request call id 8646259) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:14:35.912017 912 thread.cc:844] raft_19722_worker (raft) Time spent starting thread: real 1.147s user 0.000s sys 0.023s
I0821 11:14:36.035658 3814473 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:14:37.691967 3814473 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:14:39.434859 3814473 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.412s
W0821 11:14:42.252887 910 periodic.cc:174] PeriodicTimer callback delayed by 4.978s
I0821 11:14:41.873138 3814473 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:14:44.041425 3814473 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:14:44.406150 3814473 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:14:44.445089 915 inbound_call.cc:111] 0x000050403e3d4400 -> Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:46273 => 10.0.0.2:7100 (request call id 8646308): Connection torn down before Call yb.master.MasterService.GetMasterRegistration 10.0.0.3:46273 => 10.0.0.2:7100 (request call id 8646308) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:14:44.581426 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403f878700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313081, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98487.816s (105.728s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98577.782s (15.761s ago), now: 98593.544s, connection: Connection (0x000050403b9151a0) client 10.0.0.2:43063 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:14:44.593823 3814512 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313081) to 10.0.0.3:7100 timed out after 11.529s
W0821 11:14:45.535322 913 periodic.cc:174] PeriodicTimer callback delayed by 1.176s
W0821 11:14:49.100030 912 periodic.cc:174] PeriodicTimer callback delayed by 1.322s
I0821 11:14:49.656831 3814512 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.703s
I0821 11:14:49.657306 3814512 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:14:49.658418 3814560 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.751s
I0821 11:14:49.658442 3814560 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:14:49.661773 3814512 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.695s
I0821 11:14:49.682287 3814512 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)
I0821 11:14:49.682337 3814512 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:14:49.683275 3814562 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)
I0821 11:14:49.683286 3814562 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:14:49.683312 3814563 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)
I0821 11:14:49.683322 3814563 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:14:49.684585 3814564 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.675s
I0821 11:14:49.685098 3814564 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223354903.755s
I0821 11:14:49.685120 3814564 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:14:49.685120 3814562 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)
I0821 11:14:49.685173 3814562 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29104. Preceding OpId from leader: 29.29126. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:14:49.685253 3814562 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica is already servicing an update from a current leader or another vote.
I0821 11:14:49.685267 3814562 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:14:49.685557 3814560 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.255s
I0821 11:14:49.685582 3814560 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:14:49.686010 3814565 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 4.115s
I0821 11:14:49.686024 3814565 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:14:51.480116 912 periodic.cc:174] PeriodicTimer callback delayed by 0.097s
W0821 11:14:56.399370 911 periodic.cc:174] PeriodicTimer callback delayed by 2.683s
W0821 11:14:59.519191 911 thread.cc:850] raft_19725_worker (raft) Time spent creating pthread: real 0.536s user 0.000s sys 0.013s
I0821 11:15:01.355178 3814634 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:15:02.648306 911 thread.cc:844] raft_19725_worker (raft) Time spent starting thread: real 3.237s user 0.000s sys 0.071s
W0821 11:15:04.179165 913 thread.cc:844] CatalogManagerBGTasks_5824_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.688s user 0.000s sys 0.026s
W0821 11:15:02.304751 3814562 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44903 => 10.0.0.2:7100 (request call id 27113) took 5632ms (client timeout 3000ms).
I0821 11:15:03.816773 3814634 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:15:07.414850 3814634 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.103s
W0821 11:15:10.997711 912 periodic.cc:174] PeriodicTimer callback delayed by 9.762s
W0821 11:15:13.902344 917 yb_rpc.cc:553] Connection (0x0000504038f736e0) client 10.0.0.2:40927 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.896s, timeout: 15.000s, now: 98614.539s, last_read_time_: 98598.643s
W0821 11:15:19.132123 910 periodic.cc:174] PeriodicTimer callback delayed by 2.796s
W0821 11:15:23.540804 911 periodic.cc:174] PeriodicTimer callback delayed by 2.135s
W0821 11:15:21.672982 916 yb_rpc.cc:553] Connection (0x000050403b1a63a0) client 10.0.0.2:43993 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.985s, timeout: 15.000s, now: 98628.212s, last_read_time_: 98604.226s
W0821 11:15:30.968757 911 periodic.cc:174] PeriodicTimer callback delayed by 2.919s
W0821 11:15:37.764029 910 periodic.cc:174] PeriodicTimer callback delayed by 1.815s
W0821 11:15:39.474879 912 thread.cc:850] CatalogManagerBGTasks_5825_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.547s user 0.000s sys 0.008s
W0821 11:15:42.926046 912 thread.cc:844] CatalogManagerBGTasks_5825_worker (CatalogManagerBGTasks) Time spent starting thread: real 5.160s user 0.000s sys 0.084s
W0821 11:15:44.850749 911 periodic.cc:174] PeriodicTimer callback delayed by 2.601s
W0821 11:15:52.360459 910 periodic.cc:174] PeriodicTimer callback delayed by 2.081s
W0821 11:15:59.574816 912 periodic.cc:174] PeriodicTimer callback delayed by 2.324s
I0821 11:15:13.269433 3814634 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:16:01.994557 916 yb_rpc.cc:553] Connection (0x000050403b1548e0) client 10.0.0.2:45001 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.985s, timeout: 15.000s, now: 98628.212s, last_read_time_: 98604.226s
W0821 11:16:05.168093 910 periodic.cc:174] PeriodicTimer callback delayed by 1.617s
I0821 11:16:07.429899 3814634 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:16:10.066716 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:16:12.795104 910 periodic.cc:174] PeriodicTimer callback delayed by 2.748s
I0821 11:16:18.774011 3814634 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:16:19.442374 912 thread.cc:844] CatalogManagerBGTasks_5826_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.290s user 0.000s sys 0.034s
W0821 11:16:20.714759 911 periodic.cc:174] PeriodicTimer callback delayed by 2.920s
W0821 11:16:27.324385 913 periodic.cc:174] PeriodicTimer callback delayed by 2.069s
W0821 11:16:37.037822 913 periodic.cc:174] PeriodicTimer callback delayed by 3.879s
I0821 11:16:41.019502 3814679 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:16:41.020291 913 thread.cc:888] raft_19726_worker (raft) Time spent waiting for new thread to publish its TID: real 0.636s user 0.000s sys 0.019s
I0821 11:16:43.493624 3814679 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 11:16:43.949908 913 thread.cc:844] raft_19726_worker (raft) Time spent starting thread: real 3.952s user 0.000s sys 0.062s
I0821 11:16:45.446261 3814679 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 12.280s
W0821 11:16:46.792891 917 thread.cc:850] Master-high-pri_523_worker (Master-high-pri) Time spent creating pthread: real 0.564s user 0.000s sys 0.008s
W0821 11:16:46.861979 912 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:40883 => 10.0.0.2:7100 (request call id 27169) took 6415ms (client timeout 3000ms).
W0821 11:16:49.433450 917 thread.cc:888] Master-high-pri_523_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.597s user 0.000s sys 0.009s
W0821 11:16:50.063628 910 periodic.cc:174] PeriodicTimer callback delayed by 9.522s
W0821 11:16:50.672789 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:16:51.481632 917 thread.cc:844] Master-high-pri_523_worker (Master-high-pri) Time spent starting thread: real 5.614s user 0.000s sys 0.072s
W0821 11:16:54.248853 913 thread.cc:888] CatalogManagerBGTasks_5827_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.266s user 0.000s sys 0.052s
W0821 11:16:54.707285 912 periodic.cc:174] PeriodicTimer callback delayed by 2.276s
W0821 11:16:56.153657 913 thread.cc:844] CatalogManagerBGTasks_5827_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.292s user 0.000s sys 0.089s
W0821 11:16:54.387409 917 inbound_call.cc:111] 0x000050403e316c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:40883 => 10.0.0.2:7100 (request call id 27169): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:40883 => 10.0.0.2:7100 (request call id 27169) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 11:16:58.840224 912 periodic.cc:174] PeriodicTimer callback delayed by 2.021s
W0821 11:17:02.035888 911 periodic.cc:174] PeriodicTimer callback delayed by 2.358s
W0821 11:17:09.789999 913 periodic.cc:174] PeriodicTimer callback delayed by 2.681s
W0821 11:17:18.088372 912 periodic.cc:174] PeriodicTimer callback delayed by 2.974s
I0821 11:16:47.452167 3814679 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:17:25.801499 911 periodic.cc:174] PeriodicTimer callback delayed by 2.415s
I0821 11:17:25.885120 3814679 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:17:26.473766 3814679 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:17:26.483475 915 yb_rpc.cc:553] Connection (0x000050403b0e4720) client 10.0.0.2:46865 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 118.171s, timeout: 15.000s, now: 98729.587s, last_read_time_: 98611.415s
W0821 11:17:30.731699 913 periodic.cc:174] PeriodicTimer callback delayed by 0.929s
I0821 11:17:36.483289 3814805 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:17:36.616803 913 thread.cc:844] raft_19727_worker (raft) Time spent starting thread: real 2.589s user 0.000s sys 0.049s
I0821 11:17:39.316612 3814805 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:17:42.089570 3814805 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 10.366s
W0821 11:17:44.008586 912 periodic.cc:174] PeriodicTimer callback delayed by 8.617s
W0821 11:17:42.837059 915 yb_rpc.cc:553] Connection (0x000050403b8c5c20) client 10.0.0.2:44859 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 118.171s, timeout: 15.000s, now: 98729.587s, last_read_time_: 98611.415s
W0821 11:17:47.894168 911 periodic.cc:174] PeriodicTimer callback delayed by 1.786s
W0821 11:17:51.684149 911 periodic.cc:174] PeriodicTimer callback delayed by 1.535s
W0821 11:17:55.461694 911 periodic.cc:174] PeriodicTimer callback delayed by 2.261s
W0821 11:17:58.867422 910 thread.cc:850] CatalogManagerBGTasks_5829_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.646s user 0.000s sys 0.017s
W0821 11:18:00.901377 910 thread.cc:844] CatalogManagerBGTasks_5829_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.685s user 0.000s sys 0.056s
W0821 11:18:02.361384 911 periodic.cc:174] PeriodicTimer callback delayed by 2.579s
W0821 11:18:04.593926 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:17:42.810935 3814805 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:18:04.785058 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403d92b180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313087, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98680.206s (113.339s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98749.926s (43.619s ago), now: 98793.546s, connection: Connection (0x000050403b9f8720) client 10.0.0.2:34697 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:18:04.801072 915 inbound_call.cc:111] 0x000050403e459000 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44903 => 10.0.0.2:7100 (request call id 27113): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:44903 => 10.0.0.2:7100 (request call id 27113) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 11:18:04.951836 3814805 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:18:07.860590 913 periodic.cc:174] PeriodicTimer callback delayed by 1.363s
W0821 11:18:08.462291 917 thread.cc:888] Master-high-pri_524_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.517s user 0.000s sys 0.025s
W0821 11:18:08.394552 3814829 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313087) to 10.0.0.3:7100 timed out after 10.103s
I0821 11:18:09.058508 3814805 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:18:10.258484 917 thread.cc:844] Master-high-pri_524_worker (Master-high-pri) Time spent starting thread: real 2.862s user 0.000s sys 0.056s
W0821 11:18:13.484184 910 periodic.cc:174] PeriodicTimer callback delayed by 1.694s
W0821 11:18:10.784679 3814829 outbound_call.cc:526] RPC callback for RPC call 0x000050403d92b180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313087, state: TIMED_OUT, transfer_state: PENDING took 2.149s
W0821 11:18:20.219615 912 periodic.cc:174] PeriodicTimer callback delayed by 2.543s
I0821 11:18:27.856331 3814832 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 11:18:29.114739 912 thread.cc:844] raft_19728_worker (raft) Time spent starting thread: real 2.559s user 0.000s sys 0.041s
I0821 11:18:30.187191 3814832 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:18:32.075095 3814832 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 12.616s
W0821 11:18:33.754005 910 thread.cc:844] CatalogManagerBGTasks_5830_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.028s user 0.000s sys 0.018s
W0821 11:18:35.846100 913 periodic.cc:174] PeriodicTimer callback delayed by 11.076s
W0821 11:18:39.520097 910 periodic.cc:174] PeriodicTimer callback delayed by 1.472s
W0821 11:18:43.616701 911 periodic.cc:174] PeriodicTimer callback delayed by 1.848s
W0821 11:18:47.813827 911 periodic.cc:174] PeriodicTimer callback delayed by 1.937s
W0821 11:18:54.442600 911 periodic.cc:174] PeriodicTimer callback delayed by 2.283s
W0821 11:19:02.242751 910 periodic.cc:174] PeriodicTimer callback delayed by 2.338s
I0821 11:18:34.346047 3814832 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 11:19:07.211418 911 thread.cc:844] CatalogManagerBGTasks_5831_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.693s user 0.000s sys 0.015s
I0821 11:19:07.447229 3814832 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 11:19:09.359755 911 periodic.cc:174] PeriodicTimer callback delayed by 2.949s
W0821 11:19:17.172484 913 periodic.cc:174] PeriodicTimer callback delayed by 2.914s
I0821 11:19:19.148129 3814832 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 11:19:20.360975 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403f878700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313090, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98755.507s (99.525s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98826.981s (28.050s ago), now: 98855.032s, connection: Connection (0x0000504038f91360) client 10.0.0.2:35255 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:19:23.646214 912 periodic.cc:174] PeriodicTimer callback delayed by 1.839s
W0821 11:19:25.702777 915 thread.cc:850] Master-high-pri_525_worker (Master-high-pri) Time spent creating pthread: real 0.700s user 0.000s sys 0.011s
W0821 11:19:28.178251 915 thread.cc:844] Master-high-pri_525_worker (Master-high-pri) Time spent starting thread: real 3.114s user 0.000s sys 0.047s
W0821 11:19:27.698215 3814863 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313090) to 10.0.0.4:7100 timed out after 12.280s
W0821 11:19:29.575996 912 periodic.cc:174] PeriodicTimer callback delayed by 1.563s
W0821 11:19:32.873477 912 thread.cc:850] raft_19729_worker (raft) Time spent creating pthread: real 0.647s user 0.000s sys 0.009s
W0821 11:19:32.950837 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bef3180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313091, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 98796.346s (82.480s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 98827.229s (51.597s ago), now: 98878.826s, connection: Connection (0x0000504038e50020) client 10.0.0.2:33453 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 11:19:30.393227 3814863 outbound_call.cc:526] RPC callback for RPC call 0x000050403f878700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313090, state: TIMED_OUT, transfer_state: PENDING took 2.653s
W0821 11:19:32.956555 3814903 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313091) to 10.0.0.3:7100 timed out after 10.366s
I0821 11:19:32.956974 3814868 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:19:32.957008 3814868 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:19:32.957022 3814868 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 3.966s
W0821 11:19:32.957085 912 thread.cc:844] raft_19729_worker (raft) Time spent starting thread: real 1.161s user 0.000s sys 0.015s
I0821 11:19:32.957034 3814868 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:19:32.957182 3814868 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:19:32.959283 3814868 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 11:19:33.043987 3814903 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.969s
I0821 11:19:33.046331 3814863 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223354620.393s
I0821 11:19:33.049175 3814863 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.965s
I0821 11:19:33.049201 3814863 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:19:33.049810 3814868 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 19.827s
I0821 11:19:33.049834 3814868 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:19:33.049873 3814863 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223354620.391s
I0821 11:19:33.050518 3814863 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:19:33.050856 3814868 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 9.019s
I0821 11:19:33.050872 3814868 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:19:33.054374 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)
I0821 11:19:33.054678 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:33.073614 3814929 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)
I0821 11:19:33.075090 3814929 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:33.075838 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)
I0821 11:19:33.075855 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:33.075903 3814863 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)
I0821 11:19:33.075915 3814863 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:33.077337 3814863 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)
I0821 11:19:33.077353 3814863 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29144. Preceding OpId from leader: 29.29175. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:33.080788 3814903 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223354620.359s
I0821 11:19:33.080816 3814903 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:19:33.080919 3814929 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.934s
I0821 11:19:33.080935 3814929 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:19:33.080991 3814868 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 7.701s
I0821 11:19:33.081005 3814868 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:19:33.081032 3814932 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 15.481s
I0821 11:19:33.081040 3814932 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
I0821 11:19:33.081094 3814863 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.934s
I0821 11:19:33.081117 3814863 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:19:33.081184 3814868 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.405s
I0821 11:19:33.081197 3814868 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:19:38.341642 911 periodic.cc:174] PeriodicTimer callback delayed by 3.649s
W0821 11:19:41.063601 913 thread.cc:844] CatalogManagerBGTasks_5832_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.437s user 0.000s sys 0.053s
W0821 11:19:42.076756 911 thread.cc:844] raft_19731_worker (raft) Time spent starting thread: real 1.292s user 0.001s sys 0.039s
I0821 11:19:42.077950 3814998 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:19:42.088442 3814998 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:19:42.088472 3814998 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 6.468s
W0821 11:19:40.780961 3814929 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.3:34565 => 10.0.0.2:7100 (request call id 27895) took 4544ms (client timeout 3000ms).
I0821 11:19:42.088502 3814998 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:19:42.094792 3814998 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:19:42.094851 3814998 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 11:19:42.094977 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29189. Preceding OpId from leader: 29.29197. (index mismatch)
I0821 11:19:42.094988 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29189. Preceding OpId from leader: 29.29197. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:42.095104 3814863 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Refusing update from remote peer a5110a40aa5b46eaa183f0022d8d9063: Log matching property violated. Preceding OpId in replica: 29.29189. Preceding OpId from leader: 29.29219. (index mismatch)
I0821 11:19:42.095119 3814863 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29189. Preceding OpId from leader: 29.29219. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:19:42.127029 3814903 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.912s
I0821 11:19:42.153877 3814903 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 29 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 9223354611.297s
I0821 11:19:42.154844 3814903 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:19:42.155012 3814998 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 18.435s
I0821 11:19:42.155040 3814998 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:19:46.658111 913 periodic.cc:174] PeriodicTimer callback delayed by 3.759s
I0821 11:20:51.106065 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 64fedbe39ec148ab8f9e982b6d19f6d2 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 0.320s
I0821 11:20:57.625286 3817579 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 11:20:57.625347 3817579 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Fail or stepdown of leader a5110a40aa5b46eaa183f0022d8d9063 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 11:20:57.625365 3817579 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Snoozing leader timeout detection for 14.789s
I0821 11:20:57.625386 3817579 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 11:20:57.625487 3817579 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 11:20:57.625556 3817579 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 11:20:57.630780 3814903 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 29 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 30 because replica believes a valid leader to be alive (leader uuid: a5110a40aa5b46eaa183f0022d8d9063). Time left: 2.377s
I0821 11:21:00.011502 3814903 raft_consensus.cc:2475] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Pre-election. Granting vote for candidate 64fedbe39ec148ab8f9e982b6d19f6d2 in term 30
I0821 11:21:00.027379 3814903 raft_consensus.cc:3695] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 29 FOLLOWER]: Advancing to term 30
I0821 11:21:00.027467 3814903 consensus_meta.cc:361] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866: Updating active role from FOLLOWER to FOLLOWER. Consensus state: current_term: 30 leader_uuid: "" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }, has_pending_config = 0
I0821 11:21:00.031612 3814903 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.430s
I0821 11:21:00.035336 3814903 raft_consensus.cc:3183] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 64fedbe39ec148ab8f9e982b6d19f6d2 in term 30.
I0821 11:21:00.045946 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 29.29245. Preceding OpId from leader: 30.29246. (index mismatch)
I0821 11:21:00.045984 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 29.29245. Preceding OpId from leader: 30.29246. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 11:21:00.046835 3814903 consensus_meta.cc:361] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866: Updating active role from FOLLOWER to FOLLOWER. Consensus state: current_term: 30 leader_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }, has_pending_config = 0
I0821 11:21:00.046900 3814903 raft_consensus.cc:3580] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Calling mark dirty synchronously for reason code NEW_LEADER_ELECTED
I0821 11:21:00.046922 3814903 sys_catalog.cc:447] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: SysCatalogTable state changed. Locked=1. Reason: New leader 64fedbe39ec148ab8f9e982b6d19f6d2 elected. Latest consensus state: current_term: 30 leader_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 11:21:00.046939 3814903 sys_catalog.cc:451] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: This master's current role is: FOLLOWER
I0821 11:21:00.046995 944 client-internal.cc:2923] Reinitialize master addresses from file: ./master.conf
I0821 11:21:00.047793 944 client-internal.cc:2952] New master addresses: [10.0.0.2:7100,10.0.0.3:7100,10.0.0.4:7100, 10.0.0.2:7100, 10.0.0.3:7100, 10.0.0.4:7100]
I0821 11:21:00.051183 3814903 raft_consensus.cc:3580] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Calling mark dirty synchronously for reason code FOLLOWER_NO_OP_COMPLETE
I0821 11:21:00.051227 3814903 sys_catalog.cc:447] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: SysCatalogTable state changed. Locked=1. Reason: Replicate of NO_OP complete on follower. Latest consensus state: current_term: 30 leader_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" config { opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } } }
I0821 11:21:00.051270 3814903 sys_catalog.cc:451] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [sys.catalog]: This master's current role is: FOLLOWER
I0821 11:21:01.446077 3814903 raft_consensus.cc:1755] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Illegal state (yb/consensus/raft_consensus.cc:1754): Rejecting Update request from peer a5110a40aa5b46eaa183f0022d8d9063 for earlier term 29. Current term is 30. Ops: []
I0821 11:21:01.446111 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: INVALID_TERM status { code: ILLEGAL_STATE message: "Rejecting Update request from peer a5110a40aa5b46eaa183f0022d8d9063 for earlier term 29. Current term is 30. Ops: []" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1754 errors: "00" } }
W0821 11:21:12.414707 3814903 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313099) to 10.0.0.3:7100 timed out after 14.789s
I0821 11:21:12.414773 3814903 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 30 pre-election: Election decided. Result: candidate lost.
I0821 11:21:12.414870 3818403 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.271s
I0821 11:21:12.414891 3818403 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 30. Reason: None given. Originator:
W0821 11:21:16.945847 916 yb_rpc.cc:553] Connection (0x000050403b82c1e0) client 10.0.0.2:45899 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.000s, timeout: 15.000s, now: 98985.894s, last_read_time_: 98970.893s
I0821 11:21:18.452767 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.731s
I0821 11:21:44.847401 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.556s
I0821 11:21:57.497983 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.891s
I0821 11:22:07.226795 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.857s
I0821 11:22:24.527302 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.659s
I0821 11:22:42.513847 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.754s
I0821 11:22:50.500113 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.415s
W0821 11:23:15.878729 917 yb_rpc.cc:553] Connection (0x000050403b996020) client 10.0.0.2:39001 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 99104.883s, last_read_time_: 99089.784s
W0821 11:23:15.888619 914 yb_rpc.cc:553] Connection (0x000050403b18caa0) client 10.0.0.2:34529 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 99104.894s, last_read_time_: 99089.795s
I0821 11:23:20.075440 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.780s
I0821 11:23:27.219599 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.713s
I0821 11:23:29.530872 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.898s
I0821 11:24:04.503808 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.870s
I0821 11:24:04.503933 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.870s
W0821 11:24:24.589027 914 yb_rpc.cc:553] Connection (0x0000504038e903a0) client 10.0.0.2:43181 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 99173.593s, last_read_time_: 99158.494s
I0821 11:24:30.385810 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.520s
I0821 11:24:41.487398 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.852s
I0821 11:24:46.492913 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.791s
W0821 11:24:58.795151 3826140 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313108) to 10.0.0.3:7100 timed out after 1.500s
I0821 11:25:11.306231 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.904s
I0821 11:25:40.337613 915 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:43693 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
I0821 11:25:43.161191 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.845s
I0821 11:25:47.278242 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.504s
W0821 11:26:09.000869 916 yb_rpc.cc:553] Connection (0x0000504038ebe020) client 10.0.0.2:36383 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.000s, timeout: 15.000s, now: 99277.994s, last_read_time_: 99262.993s
I0821 11:26:19.925987 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.592s
I0821 11:26:19.927337 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.591s
I0821 11:26:41.306111 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.712s
I0821 11:26:58.680429 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.634s
I0821 11:27:55.722601 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.792s
I0821 11:27:55.722815 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.792s
I0821 13:52:41.431128 910 pg_client_service.cc:2621] Fetching set of database oids
I0821 13:52:41.440904 4131086 client_master_rpc.cc:86] 0x000050403ec93520 -> ListNamespaces: Failed, got resp error: Illegal state (yb/consensus/consensus.cc:154): Not the leader (master error 7)
W0821 13:52:41.440972 4131086 client_master_rpc.cc:107] ListNamespaces: Leader Master has changed (10.0.0.3:7100 is no longer the leader), re-trying...
I0821 13:52:41.441067 944 client-internal.cc:2923] Reinitialize master addresses from file: ./master.conf
I0821 13:52:41.441174 944 client-internal.cc:2952] New master addresses: [10.0.0.2:7100,10.0.0.3:7100,10.0.0.4:7100, 10.0.0.2:7100, 10.0.0.3:7100, 10.0.0.4:7100]
I0821 13:52:41.442869 910 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0821 13:52:41.442906 910 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0821 14:33:23.871668 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.948s
I0821 14:50:22.386945 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.918s
I0821 14:53:07.795749 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.804s
I0821 14:53:20.199499 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.899s
I0821 14:53:58.327361 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.689s
I0821 14:53:58.337090 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.679s
I0821 14:54:22.282516 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.870s
I0821 14:54:47.823505 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.488s
I0821 14:54:53.899259 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.626s
I0821 14:55:31.296169 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.846s
I0821 14:55:49.492228 3814903 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.987s
W0821 14:56:10.814961 911 periodic.cc:174] PeriodicTimer callback delayed by 0.285s
I0821 14:56:12.336702 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30086. Preceding OpId from leader: 30.30087. (index mismatch)
I0821 14:56:12.339607 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30086. Preceding OpId from leader: 30.30087. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:56:12.779745 910 periodic.cc:174] PeriodicTimer callback delayed by 1.587s
W0821 14:56:15.202917 910 periodic.cc:174] PeriodicTimer callback delayed by 0.147s
W0821 14:56:18.712046 913 periodic.cc:174] PeriodicTimer callback delayed by 2.636s
I0821 14:56:20.230108 140014 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 14:56:20.228868 3814903 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36073 => 10.0.0.2:7100 (request call id 5039342) took 4723ms (client timeout 3000ms).
I0821 14:56:20.448544 140014 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:56:20.637171 140014 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.004s
I0821 14:56:20.646981 140014 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:56:22.518822 140014 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 14:56:22.650030 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30144. (index mismatch)
I0821 14:56:22.677016 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30144. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:56:22.800295 140014 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:56:23.924480 3814903 long_operation_tracker.cc:155] UpdateReplica took a long time: 3.248s
W0821 14:56:24.439316 3814903 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38771 => 10.0.0.2:7100 (request call id 5039473) took 3780ms (client timeout 3000ms).
I0821 14:56:24.482635 140081 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30098. (index mismatch)
I0821 14:56:24.492118 140081 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30098. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:56:24.623481 3814903 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.878s
W0821 14:56:26.393911 911 periodic.cc:174] PeriodicTimer callback delayed by 2.636s
I0821 14:56:26.497068 140081 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
W0821 14:56:25.101743 3814903 outbound_call.cc:526] RPC callback for RPC call 0x000050403daebc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313148, state: FINISHED_SUCCESS, transfer_state: FINISHED took 0.394s
I0821 14:56:28.369274 140081 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:56:30.244374 140081 long_operation_tracker.cc:155] UpdateReplica took a long time: 5.262s
I0821 14:56:30.443326 140083 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 14:56:30.489403 140083 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:56:30.550015 140083 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.613s
W0821 14:56:30.566495 140081 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38461 => 10.0.0.2:7100 (request call id 5039486) took 5941ms (client timeout 3000ms).
I0821 14:56:30.587847 140083 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:56:30.623140 140083 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 14:56:30.636583 140083 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 14:56:30.959159 140129 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888626.868s
I0821 14:56:30.952111 140081 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888628.777s
I0821 14:56:31.082013 140081 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 14:56:32.490859 140129 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.590s
W0821 14:56:32.491523 140081 outbound_call.cc:526] RPC callback for RPC call 0x000050403d9ee000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313149, state: FINISHED_SUCCESS, transfer_state: FINISHED took 1.541s
I0821 14:56:32.499617 140129 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 14:56:30.988382 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:56:32.504472 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:56:32.929399 140164 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.028s
W0821 14:56:34.227972 3814903 long_operation_tracker.cc:155] UpdateReplica took a long time: 3.251s
W0821 14:56:34.219408 913 periodic.cc:174] PeriodicTimer callback delayed by 4.299s
W0821 14:56:36.762759 913 periodic.cc:174] PeriodicTimer callback delayed by 2.521s
I0821 14:56:36.454938 140167 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.182s
I0821 14:56:36.143819 140164 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 14:56:36.342132 3814903 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:46415 => 10.0.0.2:7100 (request call id 5039502) took 5305ms (client timeout 3000ms).
I0821 14:56:38.172158 140167 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 14:56:38.640594 140166 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:56:38.640626 140166 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:56:38.713959 140166 long_operation_tracker.cc:155] UpdateReplica took a long time: 3.144s
W0821 14:56:40.818722 912 periodic.cc:174] PeriodicTimer callback delayed by 2.132s
W0821 14:56:43.567586 911 periodic.cc:174] PeriodicTimer callback delayed by 2.245s
W0821 14:56:43.139348 140166 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42029 => 10.0.0.2:7100 (request call id 5039494) took 10525ms (client timeout 3000ms).
I0821 14:56:45.554291 3814903 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
W0821 14:56:45.864465 913 thread.cc:844] CatalogManagerBGTasks_6266_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.509s user 0.000s sys 0.016s
I0821 14:56:45.863549 140169 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 14:56:45.959721 916 yb_rpc.cc:553] Connection (0x000050403b9a48e0) client 10.0.0.2:45105 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.162s, timeout: 15.000s, now: 111914.933s, last_read_time_: 111899.770s
I0821 14:56:45.907646 3814903 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:56:45.999759 3814903 long_operation_tracker.cc:155] UpdateReplica took a long time: 2.824s
I0821 14:56:46.009002 140169 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:56:46.102073 140169 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.125s
W0821 14:56:46.175298 3814903 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:40845 => 10.0.0.2:7100 (request call id 5039510) took 7546ms (client timeout 3000ms).
I0821 14:56:46.149755 140169 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:56:46.275707 140169 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 14:56:46.676679 140169 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:56:51.025183 911 periodic.cc:174] PeriodicTimer callback delayed by 3.132s
W0821 14:56:54.174353 911 thread.cc:844] raft_19739_worker (raft) Time spent starting thread: real 0.592s user 0.002s sys 0.022s
I0821 14:56:54.173245 140194 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 14:56:55.483677 140194 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:56:57.614617 140194 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.306s
W0821 14:57:00.026193 911 periodic.cc:174] PeriodicTimer callback delayed by 3.770s
I0821 14:56:58.933976 140194 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:57:00.178323 140194 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 14:57:03.550381 910 periodic.cc:174] PeriodicTimer callback delayed by 1.267s
I0821 14:57:05.058583 140194 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:57:07.319123 917 yb_rpc.cc:553] Connection (0x0000504038ebf1a0) client 10.0.0.2:42795 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 36.370s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111899.954s
W0821 14:57:07.319141 914 yb_rpc.cc:553] Connection (0x000050403b8ab520) client 10.0.0.2:39177 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.779s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111913.544s
W0821 14:57:07.316969 916 yb_rpc.cc:553] Connection (0x0000504038f8c8e0) client 10.0.0.2:40035 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 36.363s, timeout: 15.000s, now: 111936.318s, last_read_time_: 111899.955s
W0821 14:57:07.487628 915 yb_rpc.cc:553] Connection (0x0000504039e1c1e0) client 10.0.0.2:42467 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 28.859s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111907.630s
W0821 14:57:07.500370 140250 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 28.859s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111907.630s
W0821 14:57:07.488008 916 yb_rpc.cc:553] Connection (0x000050403b996e20) client 10.0.0.2:45147 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.818s, timeout: 15.000s, now: 111936.318s, last_read_time_: 111901.500s
W0821 14:57:07.487957 914 yb_rpc.cc:553] Connection (0x0000504038ebeaa0) client 10.0.0.2:33401 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 36.371s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111899.953s
W0821 14:57:07.487826 917 yb_rpc.cc:553] Connection (0x000050403b196aa0) client 10.0.0.2:41949 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.044s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111911.279s
W0821 14:57:07.519704 915 yb_rpc.cc:553] Connection (0x0000504038f8c560) client 10.0.0.2:35231 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 33.267s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111903.223s
W0821 14:57:07.519894 914 yb_rpc.cc:553] Connection (0x0000504039f36e20) client 10.0.0.2:42373 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 36.371s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111899.953s
W0821 14:57:07.519907 917 yb_rpc.cc:553] Connection (0x000050403b997a60) client 10.0.0.2:37633 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.044s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111911.279s
W0821 14:57:07.520746 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521343 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521366 915 yb_rpc.cc:553] Connection (0x0000504038fa48e0) client 10.0.0.2:42399 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 28.859s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111907.630s
W0821 14:57:07.521435 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521471 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521592 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521636 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521656 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.521129 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:07.521162 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:07.522975 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:07.521515 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:07.523039 917 yb_rpc.cc:553] Connection (0x0000504039e1ce20) client 10.0.0.2:37305 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.832s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111901.492s
W0821 14:57:07.523109 917 yb_rpc.cc:553] Connection (0x0000504038f8dde0) client 10.0.0.2:38423 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 32.349s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111903.974s
W0821 14:57:07.523159 917 yb_rpc.cc:553] Connection (0x000050403b197360) client 10.0.0.2:33333 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.044s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111911.279s
W0821 14:57:07.523018 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.525131 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.525157 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.525243 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.525264 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.525286 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:07.525300 916 yb_rpc.cc:553] Connection (0x0000504039e1caa0) client 10.0.0.2:35641 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 21.385s, timeout: 15.000s, now: 111936.318s, last_read_time_: 111914.933s
W0821 14:57:07.521719 915 tcp_stream.cc:226] { local: 10.0.0.2:7100 remote: 10.0.0.4:36073 }: Send failed: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.201563 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.223359 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:08.223383 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:08.223397 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.223479 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:08.223505 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:08.223516 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.221140 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.223793 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.223881 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:08.223547 915 yb_rpc.cc:553] Connection (0x0000504038ebf8a0) client 10.0.0.2:35329 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 28.859s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111907.630s
W0821 14:57:08.223917 914 yb_rpc.cc:553] Connection (0x000050403b8e18a0) client 10.0.0.2:44927 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.779s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111913.544s
W0821 14:57:08.232669 140283 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.779s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111913.544s
I0821 14:57:08.232726 140283 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 14:57:08.232825 140285 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.301s
I0821 14:57:08.232879 140285 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 14:57:08.234949 140250 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.044s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111911.279s
W0821 14:57:08.254202 916 yb_rpc.cc:553] Connection (0x000050403b1968e0) client 10.0.0.2:32915 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 21.385s, timeout: 15.000s, now: 111936.318s, last_read_time_: 111914.933s
W0821 14:57:08.471462 915 yb_rpc.cc:553] Connection (0x000050403b8c3520) client 10.0.0.2:42805 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 36.537s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111899.953s
W0821 14:57:08.471408 914 yb_rpc.cc:553] Connection (0x0000504038e508e0) client 10.0.0.2:33253 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.779s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111913.544s
W0821 14:57:08.624718 140250 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.779s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111913.544s
W0821 14:57:08.762796 914 yb_rpc.cc:553] Connection (0x0000504038e903a0) client 10.0.0.2:42423 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.779s, timeout: 15.000s, now: 111936.324s, last_read_time_: 111913.544s
W0821 14:57:08.762796 916 yb_rpc.cc:553] Connection (0x0000504038e4cc60) client 10.0.0.2:33565 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 21.385s, timeout: 15.000s, now: 111936.318s, last_read_time_: 111914.933s
W0821 14:57:08.762794 915 yb_rpc.cc:553] Connection (0x0000504038f8d8a0) client 10.0.0.2:45043 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 36.537s, timeout: 15.000s, now: 111936.490s, last_read_time_: 111899.953s
I0821 14:57:08.779832 140250 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 14:57:08.819224 140296 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.561s
I0821 14:57:08.823791 140296 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 14:57:08.819495 140250 outbound_call.cc:526] RPC callback for RPC call 0x000050403daebc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313153, state: FINISHED_ERROR, transfer_state: FINISHED took 0.221s
W0821 14:57:09.681681 914 inbound_call.cc:111] 0x000050403cb91400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42029 => 10.0.0.2:7100 (request call id 5039494): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42029 => 10.0.0.2:7100 (request call id 5039494) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 14:57:10.223011 140250 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:57:10.225570 140250 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:10.231316 140283 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:57:10.231345 140283 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:10.231652 140329 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:57:10.231668 140329 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:10.232147 140323 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:57:10.232168 140323 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:57:13.142179 911 periodic.cc:174] PeriodicTimer callback delayed by 3.544s
W0821 14:57:13.789291 911 periodic.cc:174] PeriodicTimer callback delayed by 1.328s
I0821 14:57:17.696899 140376 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 14:57:19.507663 911 thread.cc:844] raft_19742_worker (raft) Time spent starting thread: real 1.813s user 0.001s sys 0.031s
W0821 14:57:20.552449 910 thread.cc:844] CatalogManagerBGTasks_6267_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.888s user 0.000s sys 0.039s
I0821 14:57:20.095057 140376 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:57:22.585079 140376 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.301s
W0821 14:57:26.497840 912 periodic.cc:174] PeriodicTimer callback delayed by 8.552s
W0821 14:57:29.829929 912 periodic.cc:174] PeriodicTimer callback delayed by 1.141s
W0821 14:57:34.136942 912 periodic.cc:174] PeriodicTimer callback delayed by 3.801s
I0821 14:57:24.978278 140376 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:57:37.474298 140376 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 14:57:37.525695 140376 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:57:37.543466 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:37.550944 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:37.551574 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 14:57:37.562090 915 inbound_call.cc:111] 0x000050403cb5c400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36073 => 10.0.0.2:7100 (request call id 5039524): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36073 => 10.0.0.2:7100 (request call id 5039524) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 14:57:37.580651 140438 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)
I0821 14:57:37.580683 140438 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30146. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:37.603564 140438 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)
I0821 14:57:37.604362 140438 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:37.609635 140442 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)
I0821 14:57:37.610127 140442 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:37.610234 140441 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)
I0821 14:57:37.610241 140441 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:37.610301 140444 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)
I0821 14:57:37.610325 140444 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:37.610369 140445 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)
I0821 14:57:37.610380 140445 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30095. Preceding OpId from leader: 30.30148. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:37.616432 140445 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888560.205s
I0821 14:57:37.624961 140442 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.821s
I0821 14:57:37.625007 140442 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 14:57:37.637605 140376 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.972s
I0821 14:57:37.637629 140376 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 14:57:40.178236 913 periodic.cc:174] PeriodicTimer callback delayed by 1.634s
W0821 14:57:42.830955 911 periodic.cc:174] PeriodicTimer callback delayed by 0.400s
W0821 14:57:44.284621 913 periodic.cc:174] PeriodicTimer callback delayed by 1.272s
W0821 14:57:49.760949 913 thread.cc:844] raft_19743_worker (raft) Time spent starting thread: real 0.917s user 0.000s sys 0.014s
I0821 14:57:49.478379 140539 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 14:57:53.371717 910 thread.cc:888] CatalogManagerBGTasks_6268_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.043s user 0.005s sys 0.054s
W0821 14:57:55.052626 910 thread.cc:844] CatalogManagerBGTasks_6268_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.810s user 0.005s sys 0.107s
W0821 14:57:55.766932 910 periodic.cc:174] PeriodicTimer callback delayed by 7.403s
W0821 14:57:55.881371 140445 long_operation_tracker.cc:155] UpdateReplica took a long time: 4.591s
W0821 14:57:56.952502 901 log.cc:1013] Time spent Append to log took a long time: real 0.282s user 0.000s sys 0.005s
I0821 14:57:55.881384 140539 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:57:58.578708 140539 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.823s
W0821 14:57:58.550074 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:58.617702 915 yb_rpc.cc:553] Connection (0x000050403b9036e0) client 10.0.0.2:34485 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.759s, timeout: 15.000s, now: 111987.577s, last_read_time_: 111971.817s
I0821 14:57:59.003708 140539 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:57:59.429515 140539 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 14:57:59.521966 140445 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36831 => 10.0.0.2:7100 (request call id 5039721) took 17251ms (client timeout 3000ms).
I0821 14:57:59.570017 140539 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 14:57:59.623422 140556 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30193. (index mismatch)
I0821 14:57:59.635203 140556 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30193. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:57:59.702155 140555 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30191. (index mismatch)
W0821 14:57:59.716084 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 14:57:59.740216 140555 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30191. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:57:59.928558 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:57:59.930562 914 inbound_call.cc:111] 0x00005040397fdc00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38621 => 10.0.0.2:7100 (request call id 5039828): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38621 => 10.0.0.2:7100 (request call id 5039828) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 14:58:00.355384 140555 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30192. (index mismatch)
I0821 14:58:00.384544 140555 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30192. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 14:58:00.485159 140556 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30195. (index mismatch)
I0821 14:58:00.492652 140556 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30195. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:58:00.831456 913 periodic.cc:174] PeriodicTimer callback delayed by 1.045s
I0821 14:58:00.575985 140445 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30181. (index mismatch)
I0821 14:58:01.606794 140445 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30181. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:58:01.634400 140445 long_operation_tracker.cc:155] UpdateReplica took a long time: 1.103s
I0821 14:58:01.673727 140602 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30195. (index mismatch)
I0821 14:58:01.673825 140602 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30195. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 14:58:03.715831 912 periodic.cc:174] PeriodicTimer callback delayed by 1.251s
I0821 14:58:06.699699 140665 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 14:58:06.707401 912 thread.cc:844] raft_19744_worker (raft) Time spent starting thread: real 0.934s user 0.000s sys 0.025s
I0821 14:58:06.816643 140665 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:58:08.514004 140665 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.490s
W0821 14:58:11.785128 911 periodic.cc:174] PeriodicTimer callback delayed by 4.673s
W0821 14:58:14.737387 910 periodic.cc:174] PeriodicTimer callback delayed by 1.334s
W0821 14:58:19.597501 912 periodic.cc:174] PeriodicTimer callback delayed by 0.729s
W0821 14:58:25.819765 913 periodic.cc:174] PeriodicTimer callback delayed by 2.033s
W0821 14:58:27.584149 914 thread.cc:844] Master-high-pri_543_worker (Master-high-pri) Time spent starting thread: real 0.929s user 0.000s sys 0.012s
W0821 14:58:27.583894 910 thread.cc:844] CatalogManagerBGTasks_6269_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.929s user 0.000s sys 0.013s
W0821 14:58:26.041783 911 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36831 => 10.0.0.2:7100 (request call id 5039896) took 9744ms (client timeout 3000ms).
W0821 14:58:29.055495 914 inbound_call.cc:111] 0x00005040397fdc00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36831 => 10.0.0.2:7100 (request call id 5039896): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36831 => 10.0.0.2:7100 (request call id 5039896) could send its response: Network error (yb/util/net/socket.cc:536): recvmsg got EOF from remote (system error 108)
W0821 14:58:32.178892 911 periodic.cc:174] PeriodicTimer callback delayed by 1.602s
I0821 14:58:09.905678 140665 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 14:58:38.130838 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 14:58:39.214831 912 periodic.cc:174] PeriodicTimer callback delayed by 2.119s
I0821 14:58:39.174815 140665 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 14:58:45.638681 913 periodic.cc:174] PeriodicTimer callback delayed by 2.437s
I0821 14:58:45.044827 140665 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:58:51.414798 911 periodic.cc:174] PeriodicTimer callback delayed by 2.088s
I0821 14:58:54.059398 140682 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 14:58:55.359754 911 thread.cc:844] raft_19745_worker (raft) Time spent starting thread: real 1.818s user 0.001s sys 0.032s
I0821 14:58:56.331168 140682 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 14:58:56.707760 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 14:58:57.857999 140682 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.152s
W0821 14:58:59.946875 910 thread.cc:844] CatalogManagerBGTasks_6270_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.056s user 0.000s sys 0.020s
W0821 14:59:02.419272 910 periodic.cc:174] PeriodicTimer callback delayed by 7.188s
W0821 14:59:07.635790 910 periodic.cc:174] PeriodicTimer callback delayed by 2.993s
W0821 14:59:14.107151 910 periodic.cc:174] PeriodicTimer callback delayed by 1.809s
I0821 14:58:59.498819 140682 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 14:59:20.431339 911 periodic.cc:174] PeriodicTimer callback delayed by 1.503s
I0821 14:59:21.409250 140682 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 14:59:29.482751 913 periodic.cc:174] PeriodicTimer callback delayed by 5.861s
I0821 14:59:29.286909 140682 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:59:35.119776 910 thread.cc:888] CatalogManagerBGTasks_6271_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.920s user 0.006s sys 0.061s
W0821 14:59:36.809720 910 thread.cc:844] CatalogManagerBGTasks_6271_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.424s user 0.006s sys 0.098s
W0821 14:59:38.039254 912 periodic.cc:174] PeriodicTimer callback delayed by 3.644s
W0821 14:59:41.481258 912 thread.cc:844] raft_19746_worker (raft) Time spent starting thread: real 1.093s user 0.002s sys 0.030s
I0821 14:59:40.887501 140728 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 14:59:43.450093 140728 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 14:59:44.155803 140728 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.359s
I0821 14:59:44.161737 140728 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 14:59:45.000610 140728 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 14:59:47.847127 910 periodic.cc:174] PeriodicTimer callback delayed by 5.618s
I0821 14:59:51.871870 140728 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 14:59:55.450549 911 periodic.cc:174] PeriodicTimer callback delayed by 2.534s
W0821 15:00:01.385633 912 periodic.cc:174] PeriodicTimer callback delayed by 2.225s
W0821 15:00:06.555075 912 thread.cc:888] raft_19747_worker (raft) Time spent waiting for new thread to publish its TID: real 1.599s user 0.005s sys 0.068s
I0821 15:00:05.371999 140739 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:00:08.481626 912 thread.cc:844] raft_19747_worker (raft) Time spent starting thread: real 5.165s user 0.005s sys 0.133s
I0821 15:00:08.456705 140739 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:00:10.774080 910 thread.cc:844] CatalogManagerBGTasks_6272_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.133s user 0.001s sys 0.043s
I0821 15:00:10.853801 140739 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.260s
W0821 15:00:15.572013 910 periodic.cc:174] PeriodicTimer callback delayed by 9.801s
W0821 15:00:19.816632 911 periodic.cc:174] PeriodicTimer callback delayed by 2.006s
W0821 15:00:28.340785 913 periodic.cc:174] PeriodicTimer callback delayed by 3.307s
W0821 15:00:35.157347 913 periodic.cc:174] PeriodicTimer callback delayed by 1.708s
W0821 15:00:43.290970 910 periodic.cc:174] PeriodicTimer callback delayed by 3.571s
W0821 15:00:43.608654 913 thread.cc:850] CatalogManagerBGTasks_6273_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.756s user 0.000s sys 0.013s
W0821 15:00:45.336826 913 thread.cc:844] CatalogManagerBGTasks_6273_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.750s user 0.002s sys 0.044s
W0821 15:00:52.507371 913 periodic.cc:174] PeriodicTimer callback delayed by 5.209s
I0821 15:00:13.946017 140739 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:00:56.778123 140739 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:00:59.268559 910 periodic.cc:174] PeriodicTimer callback delayed by 3.091s
I0821 15:01:07.718321 140739 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:01:07.817181 913 periodic.cc:174] PeriodicTimer callback delayed by 3.702s
W0821 15:01:14.982733 910 periodic.cc:174] PeriodicTimer callback delayed by 2.312s
W0821 15:01:20.378798 912 periodic.cc:174] PeriodicTimer callback delayed by 1.222s
W0821 15:01:22.310607 910 thread.cc:844] CatalogManagerBGTasks_6274_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.998s user 0.005s sys 0.044s
I0821 15:01:24.430371 140773 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:01:25.517217 912 thread.cc:844] raft_19748_worker (raft) Time spent starting thread: real 1.123s user 0.000s sys 0.020s
I0821 15:01:26.312983 140773 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:01:27.835325 140773 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.559s
W0821 15:01:34.758991 910 periodic.cc:174] PeriodicTimer callback delayed by 10.082s
W0821 15:01:40.999251 911 periodic.cc:174] PeriodicTimer callback delayed by 2.296s
W0821 15:01:47.357105 913 periodic.cc:174] PeriodicTimer callback delayed by 1.618s
W0821 15:01:52.966046 910 periodic.cc:174] PeriodicTimer callback delayed by 1.798s
W0821 15:01:55.198546 911 thread.cc:888] CatalogManagerBGTasks_6275_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.633s user 0.000s sys 0.023s
W0821 15:01:57.941601 911 thread.cc:844] CatalogManagerBGTasks_6275_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.316s user 0.001s sys 0.062s
W0821 15:02:01.800354 910 periodic.cc:174] PeriodicTimer callback delayed by 4.508s
I0821 15:01:29.642634 140773 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:02:05.565214 140773 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:02:08.492053 910 periodic.cc:174] PeriodicTimer callback delayed by 1.790s
I0821 15:02:12.359560 140773 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:02:17.826417 910 periodic.cc:174] PeriodicTimer callback delayed by 4.941s
W0821 15:02:24.783145 910 periodic.cc:174] PeriodicTimer callback delayed by 2.925s
W0821 15:02:28.248768 910 thread.cc:850] raft_19749_worker (raft) Time spent creating pthread: real 0.727s user 0.000s sys 0.007s
W0821 15:02:29.052345 911 thread.cc:844] CatalogManagerBGTasks_6276_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.717s user 0.000s sys 0.014s
W0821 15:02:29.428217 910 thread.cc:844] raft_19749_worker (raft) Time spent starting thread: real 2.522s user 0.001s sys 0.026s
I0821 15:02:29.027135 140800 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:02:31.174391 140800 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:02:32.828781 140800 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.263s
I0821 15:02:35.066972 140800 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:02:36.211810 911 periodic.cc:174] PeriodicTimer callback delayed by 7.595s
I0821 15:02:38.341663 140800 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:02:39.875334 911 periodic.cc:174] PeriodicTimer callback delayed by 1.439s
W0821 15:02:42.713786 913 periodic.cc:174] PeriodicTimer callback delayed by 2.409s
I0821 15:02:46.052482 140800 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:02:49.265568 911 periodic.cc:174] PeriodicTimer callback delayed by 2.684s
W0821 15:02:57.097395 913 periodic.cc:174] PeriodicTimer callback delayed by 3.935s
W0821 15:03:02.857802 913 thread.cc:888] raft_19750_worker (raft) Time spent waiting for new thread to publish its TID: real 1.760s user 0.002s sys 0.059s
I0821 15:03:02.430884 140820 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:03:05.675475 913 thread.cc:844] raft_19750_worker (raft) Time spent starting thread: real 5.428s user 0.002s sys 0.132s
W0821 15:03:06.476631 912 thread.cc:844] CatalogManagerBGTasks_6277_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.867s user 0.002s sys 0.019s
I0821 15:03:06.498624 140820 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:03:09.099354 140820 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.322s
I0821 15:03:12.421059 140820 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:03:13.264180 911 periodic.cc:174] PeriodicTimer callback delayed by 10.990s
I0821 15:03:14.712716 140820 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:03:19.334607 911 periodic.cc:174] PeriodicTimer callback delayed by 3.596s
I0821 15:03:22.456789 140820 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:03:25.865787 911 periodic.cc:174] PeriodicTimer callback delayed by 2.047s
W0821 15:03:32.640218 911 periodic.cc:174] PeriodicTimer callback delayed by 2.083s
W0821 15:03:38.735222 911 thread.cc:844] raft_19751_worker (raft) Time spent starting thread: real 1.293s user 0.001s sys 0.025s
I0821 15:03:38.204982 140833 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:03:40.045965 913 thread.cc:844] CatalogManagerBGTasks_6278_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.649s user 0.000s sys 0.025s
I0821 15:03:41.727128 140833 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:03:44.477588 912 periodic.cc:174] PeriodicTimer callback delayed by 8.471s
I0821 15:03:43.847476 140833 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.480s
W0821 15:03:51.703125 912 periodic.cc:174] PeriodicTimer callback delayed by 2.722s
W0821 15:03:55.020221 913 periodic.cc:174] PeriodicTimer callback delayed by 3.063s
W0821 15:04:01.019738 913 periodic.cc:174] PeriodicTimer callback delayed by 1.779s
W0821 15:04:09.865031 911 periodic.cc:174] PeriodicTimer callback delayed by 3.988s
W0821 15:04:13.211261 913 thread.cc:850] CatalogManagerBGTasks_6279_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.548s user 0.000s sys 0.007s
W0821 15:04:15.178886 913 thread.cc:844] CatalogManagerBGTasks_6279_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.685s user 0.000s sys 0.037s
W0821 15:04:17.284073 910 periodic.cc:174] PeriodicTimer callback delayed by 3.036s
I0821 15:03:46.467957 140833 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:04:25.095232 910 periodic.cc:174] PeriodicTimer callback delayed by 3.633s
I0821 15:04:27.905546 140833 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:04:33.308054 910 periodic.cc:174] PeriodicTimer callback delayed by 3.871s
I0821 15:04:38.641179 140833 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:04:40.335582 912 periodic.cc:174] PeriodicTimer callback delayed by 2.706s
W0821 15:04:48.508962 912 periodic.cc:174] PeriodicTimer callback delayed by 2.914s
W0821 15:04:51.946138 913 thread.cc:844] CatalogManagerBGTasks_6280_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.291s user 0.000s sys 0.040s
W0821 15:04:51.948242 912 thread.cc:850] raft_19752_worker (raft) Time spent creating pthread: real 0.835s user 0.000s sys 0.012s
I0821 15:04:52.239514 140868 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:04:52.798580 912 thread.cc:844] raft_19752_worker (raft) Time spent starting thread: real 1.708s user 0.001s sys 0.028s
I0821 15:04:53.671257 140868 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:04:55.322956 140868 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.954s
W0821 15:05:01.032299 911 periodic.cc:174] PeriodicTimer callback delayed by 7.861s
W0821 15:05:08.423308 913 periodic.cc:174] PeriodicTimer callback delayed by 2.292s
W0821 15:05:15.657451 910 periodic.cc:174] PeriodicTimer callback delayed by 3.402s
W0821 15:05:23.894143 911 periodic.cc:174] PeriodicTimer callback delayed by 4.130s
W0821 15:05:26.787247 910 thread.cc:850] CatalogManagerBGTasks_6281_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.530s user 0.000s sys 0.010s
W0821 15:05:30.022149 910 thread.cc:844] CatalogManagerBGTasks_6281_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.992s user 0.001s sys 0.068s
W0821 15:05:31.122601 913 periodic.cc:174] PeriodicTimer callback delayed by 2.048s
W0821 15:05:37.984004 913 periodic.cc:174] PeriodicTimer callback delayed by 1.787s
I0821 15:04:56.932199 140868 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:05:46.349213 140868 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:05:47.826033 910 periodic.cc:174] PeriodicTimer callback delayed by 3.876s
I0821 15:05:55.176939 140868 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:05:55.972191 913 periodic.cc:174] PeriodicTimer callback delayed by 3.500s
W0821 15:06:02.341259 913 periodic.cc:174] PeriodicTimer callback delayed by 1.893s
W0821 15:06:04.449750 912 thread.cc:850] CatalogManagerBGTasks_6282_worker (CatalogManagerBGTasks) Time spent creating pthread: real 1.070s user 0.000s sys 0.021s
I0821 15:06:06.306372 140896 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:06:06.690248 913 thread.cc:844] raft_19753_worker (raft) Time spent starting thread: real 1.465s user 0.002s sys 0.041s
W0821 15:06:07.568384 912 thread.cc:844] CatalogManagerBGTasks_6282_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.469s user 0.001s sys 0.067s
I0821 15:06:08.101544 140896 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:06:08.569643 140896 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.040s
I0821 15:06:08.569664 140896 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:06:08.569780 140896 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:06:08.570746 140896 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:06:08.586776 140905 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/util/net/socket.cc:536): recvmsg got EOF from remote (system error 108)
W0821 15:06:08.587909 140907 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/util/net/socket.cc:536): recvmsg got EOF from remote (system error 108)
I0821 15:06:08.587944 140907 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.592855 140896 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.112s
I0821 15:06:08.593446 140896 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:06:08.634027 916 yb_rpc.cc:553] Connection (0x000050403b1a58a0) client 10.0.0.2:40739 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 489.931s, timeout: 15.000s, now: 112477.616s, last_read_time_: 111987.685s
W0821 15:06:08.654230 140907 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 489.931s, timeout: 15.000s, now: 112477.616s, last_read_time_: 111987.685s
I0821 15:06:08.657184 140907 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30196. (index mismatch)
I0821 15:06:08.657200 140907 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30196. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 15:06:08.667441 914 inbound_call.cc:111] 0x000050403bd57c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:46211 => 10.0.0.2:7100 (request call id 5039934): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:46211 => 10.0.0.2:7100 (request call id 5039934) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:06:08.723673 140907 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)
I0821 15:06:08.741892 140907 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:06:08.743479 140954 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)
I0821 15:06:08.743497 140954 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:06:08.743548 140905 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)
I0821 15:06:08.743554 140905 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:06:08.743780 140957 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)
I0821 15:06:08.743789 140957 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:06:08.745126 140957 raft_consensus.cc:2475] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Pre-election. Granting vote for candidate a5110a40aa5b46eaa183f0022d8d9063 in term 31
I0821 15:06:08.747171 140957 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.071s
I0821 15:06:08.747202 140957 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.071s
I0821 15:06:08.748112 140957 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.071s
I0821 15:06:08.790081 140957 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)
I0821 15:06:08.790112 140957 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30156. Preceding OpId from leader: 30.30197. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:06:08.801306 140907 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.016s
I0821 15:06:08.801342 140905 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.017s
I0821 15:06:08.801502 140954 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.802802 140954 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.025s
I0821 15:06:08.802920 140905 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.803390 140957 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221888049.020s
I0821 15:06:08.812762 140905 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.812786 140905 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.812793 140954 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.812836 140954 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.812937 140905 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.812959 140905 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.813231 140907 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.813262 140907 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.813406 140969 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.813417 140969 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.813544 140905 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.813555 140905 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.814463 140896 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.142s
I0821 15:06:08.814476 140896 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.814482 140896 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.322s
I0821 15:06:08.814486 140896 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.814489 140896 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.252s
I0821 15:06:08.814502 140896 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.814512 140968 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.795s
I0821 15:06:08.814519 140968 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.814541 140971 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.803s
I0821 15:06:08.814563 140971 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.814911 140970 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.814935 140970 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.814970 140971 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.696s
I0821 15:06:08.814975 140971 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.815059 140972 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.815068 140972 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.815093 140971 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.136s
I0821 15:06:08.815099 140971 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.816795 140970 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:06:08.816810 140970 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.816844 140971 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.328s
I0821 15:06:08.816852 140971 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.819836 140967 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.805s
I0821 15:06:08.819880 140967 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:06:08.819907 140971 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 6.390s
I0821 15:06:08.819929 140971 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:06:08.821704 140966 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.530s
I0821 15:06:08.821730 140966 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:06:12.452801 912 periodic.cc:174] PeriodicTimer callback delayed by 5.904s
W0821 15:06:14.833971 911 periodic.cc:174] PeriodicTimer callback delayed by 0.088s
W0821 15:06:17.373224 913 periodic.cc:174] PeriodicTimer callback delayed by 1.284s
W0821 15:06:20.525125 913 thread.cc:888] raft_19757_worker (raft) Time spent waiting for new thread to publish its TID: real 0.509s user 0.002s sys 0.019s
W0821 15:06:20.790817 1039 long_operation_tracker.cc:125] UpdateReplica running for 1.029s in thread 140957:
Thread did not respond: maybe it is blocking signals
I0821 15:06:20.547211 141057 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:06:22.081130 913 thread.cc:844] raft_19757_worker (raft) Time spent starting thread: real 2.730s user 0.002s sys 0.067s
I0821 15:06:25.237914 141057 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:06:26.451336 140957 long_operation_tracker.cc:155] UpdateReplica took a long time: 7.806s
I0821 15:06:27.620774 141057 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.013s
W0821 15:06:29.385926 913 periodic.cc:174] PeriodicTimer callback delayed by 8.310s
W0821 15:06:23.912175 140954 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:43499 => 10.0.0.2:7100 (request call id 5041016) took 4987ms (client timeout 3000ms).
W0821 15:06:30.502764 901 long_operation_tracker.cc:155] Log append took a long time: 3.307s
W0821 15:06:31.849037 901 log.cc:1013] Time spent Append to log took a long time: real 4.695s user 0.000s sys 0.086s
I0821 15:06:33.173919 901 log.cc:1248] Time spent Fsync log took a long time: real 0.072s user 0.000s sys 0.001s
W0821 15:06:35.114424 911 periodic.cc:174] PeriodicTimer callback delayed by 1.860s
W0821 15:06:42.082960 910 thread.cc:844] CatalogManagerBGTasks_6283_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.846s user 0.000s sys 0.064s
W0821 15:06:43.029475 911 periodic.cc:174] PeriodicTimer callback delayed by 3.849s
I0821 15:06:31.371908 141057 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:06:48.642295 914 yb_rpc.cc:553] Connection (0x000050403b0c5520) client 10.0.0.2:46789 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.541s, timeout: 15.000s, now: 112505.736s, last_read_time_: 112483.194s
W0821 15:06:50.552793 911 periodic.cc:174] PeriodicTimer callback delayed by 2.645s
I0821 15:06:49.996388 141057 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:06:51.199306 140957 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:34047 => 10.0.0.2:7100 (request call id 5041005) took 36434ms (client timeout 3000ms).
W0821 15:06:54.899575 912 periodic.cc:174] PeriodicTimer callback delayed by 2.106s
I0821 15:06:55.760042 141057 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:07:01.624064 912 periodic.cc:174] PeriodicTimer callback delayed by 2.590s
I0821 15:07:08.858089 141075 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:07:09.436002 912 thread.cc:844] raft_19758_worker (raft) Time spent starting thread: real 3.801s user 0.002s sys 0.070s
I0821 15:07:11.821930 141075 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:07:12.098727 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:07:13.857272 141075 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.213s
W0821 15:07:15.878240 910 thread.cc:844] CatalogManagerBGTasks_6284_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.528s user 0.000s sys 0.017s
W0821 15:07:12.695058 917 yb_rpc.cc:553] Connection (0x000050403b950fe0) client 10.0.0.2:45165 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 33.738s, timeout: 15.000s, now: 112522.057s, last_read_time_: 112488.319s
W0821 15:07:16.390847 913 periodic.cc:174] PeriodicTimer callback delayed by 10.575s
W0821 15:07:16.770120 915 yb_rpc.cc:553] Connection (0x000050403b101c20) client 10.0.0.2:41485 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 18.876s, timeout: 15.000s, now: 112502.025s, last_read_time_: 112483.148s
W0821 15:07:22.361230 910 periodic.cc:174] PeriodicTimer callback delayed by 3.734s
W0821 15:07:26.464211 910 periodic.cc:174] PeriodicTimer callback delayed by 1.828s
W0821 15:07:31.401363 913 periodic.cc:174] PeriodicTimer callback delayed by 2.590s
W0821 15:07:38.898384 912 periodic.cc:174] PeriodicTimer callback delayed by 2.077s
I0821 15:07:16.139639 141075 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:07:41.281500 141075 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:07:42.195518 917 yb_rpc.cc:553] Connection (0x000050403b9a9360) client 10.0.0.2:37753 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 33.738s, timeout: 15.000s, now: 112522.057s, last_read_time_: 112488.319s
W0821 15:07:40.136559 914 yb_rpc.cc:553] Connection (0x0000504038ea61e0) client 10.0.0.2:35111 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 17.416s, timeout: 15.000s, now: 112505.736s, last_read_time_: 112488.319s
I0821 15:07:42.315486 141075 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:07:42.320219 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403f878000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313181, state: SENT, transfer_state: FINISHED, start_time: 112527.404s (42.123s ago), sent_time: 112560.313s (9.214s ago), trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112555.147s (14.380s ago), now: 112569.528s, connection: Connection (0x000050403b8671a0) client 10.0.0.2:46735 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:07:42.299255 915 yb_rpc.cc:553] Connection (0x000050403b12d8a0) client 10.0.0.2:34587 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 18.876s, timeout: 15.000s, now: 112502.025s, last_read_time_: 112483.148s
W0821 15:07:45.087332 912 periodic.cc:174] PeriodicTimer callback delayed by 1.730s
W0821 15:07:44.332571 141107 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313181) to 10.0.0.4:7100 timed out after 3.013s
W0821 15:07:47.466017 916 thread.cc:844] Master-high-pri_552_worker (Master-high-pri) Time spent starting thread: real 4.253s user 0.000s sys 0.079s
I0821 15:07:48.327431 141110 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:07:48.325642 912 thread.cc:844] raft_19759_worker (raft) Time spent starting thread: real 1.055s user 0.001s sys 0.019s
W0821 15:07:48.342023 916 yb_rpc.cc:553] Connection (0x000050403b8671a0) client 10.0.0.2:46735 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 68.248s, timeout: 15.000s, now: 112561.235s, last_read_time_: 112492.987s
W0821 15:07:47.182354 141107 outbound_call.cc:526] RPC callback for RPC call 0x000050403f878000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313181, state: TIMED_OUT, transfer_state: FINISHED took 2.831s
I0821 15:07:48.347172 141110 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:07:48.355500 141110 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.585s
W0821 15:07:53.301877 911 periodic.cc:174] PeriodicTimer callback delayed by 4.075s
W0821 15:07:52.166661 914 yb_rpc.cc:553] Connection (0x000050403b132560) client 10.0.0.2:46577 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 22.541s, timeout: 15.000s, now: 112505.736s, last_read_time_: 112483.194s
W0821 15:07:53.903259 917 yb_rpc.cc:553] Connection (0x000050403b9ae3a0) client 10.0.0.2:39659 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 33.738s, timeout: 15.000s, now: 112522.057s, last_read_time_: 112488.319s
W0821 15:07:52.166684 915 yb_rpc.cc:553] Connection (0x000050403b867a60) client 10.0.0.2:43131 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 18.876s, timeout: 15.000s, now: 112502.025s, last_read_time_: 112483.148s
W0821 15:07:57.485497 910 periodic.cc:174] PeriodicTimer callback delayed by 1.946s
W0821 15:08:00.966639 913 periodic.cc:174] PeriodicTimer callback delayed by 2.833s
I0821 15:07:48.355523 141110 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:08:05.944808 141110 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:08:06.756919 916 yb_rpc.cc:553] Connection (0x000050403b866e20) client 10.0.0.2:35913 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 68.248s, timeout: 15.000s, now: 112561.235s, last_read_time_: 112492.987s
W0821 15:08:08.366968 911 periodic.cc:174] PeriodicTimer callback delayed by 2.183s
I0821 15:08:08.829164 141110 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:08:10.315429 914 yb_rpc.cc:553] Connection (0x000050403b04d520) client 10.0.0.2:42809 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 17.416s, timeout: 15.000s, now: 112505.736s, last_read_time_: 112488.319s
W0821 15:08:10.315624 917 yb_rpc.cc:553] Connection (0x000050403b9a9520) client 10.0.0.2:36895 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 33.738s, timeout: 15.000s, now: 112522.057s, last_read_time_: 112488.319s
W0821 15:08:10.315614 915 yb_rpc.cc:553] Connection (0x000050403b19c1e0) client 10.0.0.2:39583 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 18.876s, timeout: 15.000s, now: 112502.025s, last_read_time_: 112483.148s
W0821 15:08:12.409327 912 periodic.cc:174] PeriodicTimer callback delayed by 1.220s
I0821 15:08:12.916313 141146 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:08:12.916366 141146 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:08:12.916392 141146 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 14.554s
I0821 15:08:12.916410 141146 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:08:13.711385 916 yb_rpc.cc:553] Connection (0x000050403b14e8e0) client 10.0.0.2:37421 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 68.248s, timeout: 15.000s, now: 112561.235s, last_read_time_: 112492.987s
I0821 15:08:15.955435 141146 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:08:17.002467 913 periodic.cc:174] PeriodicTimer callback delayed by 1.018s
I0821 15:08:18.833091 141146 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:08:20.537631 910 periodic.cc:174] PeriodicTimer callback delayed by 1.270s
W0821 15:08:24.612291 910 periodic.cc:174] PeriodicTimer callback delayed by 1.822s
W0821 15:08:28.235944 910 periodic.cc:174] PeriodicTimer callback delayed by 1.336s
I0821 15:08:30.624632 141157 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:08:32.106655 141157 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:08:33.605862 141157 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 7.570s
W0821 15:08:33.425740 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:08:35.612946 911 periodic.cc:174] PeriodicTimer callback delayed by 2.925s
W0821 15:08:36.909793 916 inbound_call.cc:111] 0x0000504037bf1400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:43499 => 10.0.0.2:7100 (request call id 5041016): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:43499 => 10.0.0.2:7100 (request call id 5041016) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:08:39.628791 912 periodic.cc:174] PeriodicTimer callback delayed by 1.765s
W0821 15:08:41.816785 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:08:35.282613 141157 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:08:45.667400 913 periodic.cc:174] PeriodicTimer callback delayed by 3.802s
W0821 15:08:45.681350 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403e0a6a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313185, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112597.911s (30.987s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112618.308s (10.590s ago), now: 112628.899s, connection: Connection (0x0000504038ebe560) client 10.0.0.2:39125 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:08:45.683919 141157 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:08:46.967723 141157 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:08:48.566594 914 thread.cc:888] Master-high-pri_553_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.746s user 0.000s sys 0.033s
W0821 15:08:49.770190 914 thread.cc:844] Master-high-pri_553_worker (Master-high-pri) Time spent starting thread: real 2.780s user 0.000s sys 0.067s
W0821 15:08:49.494081 912 periodic.cc:174] PeriodicTimer callback delayed by 1.036s
W0821 15:08:48.785187 141173 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313185) to 10.0.0.4:7100 timed out after 9.585s
W0821 15:08:49.920328 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bf6ea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313180, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112521.605s (113.082s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112609.758s (24.929s ago), now: 112634.688s, connection: Connection (0x000050403b8176e0) client 10.0.0.2:37229 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:08:53.864650 911 periodic.cc:174] PeriodicTimer callback delayed by 1.079s
W0821 15:08:53.867308 913 thread.cc:844] CatalogManagerBGTasks_6287_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.329s user 0.000s sys 0.026s
W0821 15:08:50.630203 141173 outbound_call.cc:526] RPC callback for RPC call 0x000050403e0a6a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313185, state: TIMED_OUT, transfer_state: PENDING took 1.853s
W0821 15:08:55.787643 141178 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313180) to 10.0.0.3:7100 timed out after 3.013s
W0821 15:08:55.910609 915 thread.cc:844] Master-high-pri_554_worker (Master-high-pri) Time spent starting thread: real 2.042s user 0.000s sys 0.043s
I0821 15:08:57.181689 141178 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:08:57.938831 910 periodic.cc:174] PeriodicTimer callback delayed by 0.839s
W0821 15:09:00.443212 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403ce37c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313184, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112596.803s (45.215s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112625.020s (16.997s ago), now: 112642.018s, connection: Connection (0x000050403b902720) client 10.0.0.2:33219 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:09:01.618167 141182 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:09:01.745967 141184 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 14.201s
W0821 15:09:02.200227 910 thread.cc:844] raft_19763_worker (raft) Time spent starting thread: real 1.624s user 0.001s sys 0.022s
W0821 15:09:02.200338 141178 thread.cc:844] raft_19762_worker (raft) Time spent starting thread: real 1.624s user 0.000s sys 0.022s
I0821 15:09:04.109261 141182 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:09:04.085664 141184 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:09:03.588251 141173 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313184) to 10.0.0.3:7100 timed out after 9.585s
I0821 15:09:04.130095 141182 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.215s
I0821 15:09:04.610435 141173 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:09:04.126603 141178 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf6ea80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313180, state: TIMED_OUT, transfer_state: PENDING took 8.370s
W0821 15:09:08.945080 141173 thread.cc:844] raft_19764_worker (raft) Time spent starting thread: real 1.233s user 0.000s sys 0.021s
W0821 15:09:09.548454 910 periodic.cc:174] PeriodicTimer callback delayed by 8.029s
W0821 15:09:10.025913 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403d9ee000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313183, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112571.354s (81.271s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112625.521s (27.104s ago), now: 112652.625s, connection: Connection (0x0000504038ebf8a0) client 10.0.0.2:46195 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:09:13.124439 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403e2ac000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313186, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112606.612s (46.521s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112642.826s (10.307s ago), now: 112653.133s, connection: Connection (0x000050403b97e720) client 10.0.0.2:40179 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:09:12.311730 141178 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313183) to 10.0.0.4:7100 timed out after 13.213s
W0821 15:09:15.305385 912 periodic.cc:174] PeriodicTimer callback delayed by 1.824s
W0821 15:09:15.014667 141178 outbound_call.cc:526] RPC callback for RPC call 0x000050403d9ee000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313183, state: TIMED_OUT, transfer_state: PENDING took 2.604s
W0821 15:09:18.990223 914 thread.cc:888] Master-high-pri_555_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.856s user 0.000s sys 0.026s
W0821 15:09:18.833670 141195 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313186) to 10.0.0.3:7100 timed out after 14.554s
W0821 15:09:20.442741 914 thread.cc:844] Master-high-pri_555_worker (Master-high-pri) Time spent starting thread: real 3.805s user 0.000s sys 0.084s
W0821 15:09:16.392320 141173 outbound_call.cc:526] RPC callback for RPC call 0x000050403ce37c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313184, state: TIMED_OUT, transfer_state: PENDING took 12.826s
W0821 15:09:21.773736 911 periodic.cc:174] PeriodicTimer callback delayed by 2.624s
W0821 15:09:21.486287 141195 outbound_call.cc:526] RPC callback for RPC call 0x000050403e2ac000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313186, state: TIMED_OUT, transfer_state: PENDING took 2.524s
W0821 15:09:27.337459 911 periodic.cc:174] PeriodicTimer callback delayed by 1.640s
W0821 15:09:28.185595 912 thread.cc:850] CatalogManagerBGTasks_6288_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.660s user 0.001s sys 0.015s
W0821 15:09:28.803797 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403dcaf180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313182, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112571.237s (97.434s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112651.989s (16.681s ago), now: 112668.671s, connection: Connection (0x0000504038eeba60) client 10.0.0.2:41119 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:09:30.374675 912 thread.cc:844] CatalogManagerBGTasks_6288_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.217s user 0.002s sys 0.068s
W0821 15:09:31.406688 141195 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313182) to 10.0.0.3:7100 timed out after 13.213s
W0821 15:09:34.147116 912 periodic.cc:174] PeriodicTimer callback delayed by 1.813s
I0821 15:09:34.537315 141195 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:09:04.867070 141182 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:09:38.699295 141195 thread.cc:844] raft_19765_worker (raft) Time spent starting thread: real 1.323s user 0.001s sys 0.027s
W0821 15:09:40.937996 911 periodic.cc:174] PeriodicTimer callback delayed by 2.041s
I0821 15:09:40.867290 141182 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:09:41.166153 141190 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 12.383s
I0821 15:09:43.304394 141190 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:09:43.364439 141206 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 14.497s
I0821 15:09:45.955399 141206 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:09:48.265210 912 periodic.cc:174] PeriodicTimer callback delayed by 1.930s
I0821 15:09:51.090706 141182 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:09:52.282881 912 periodic.cc:174] PeriodicTimer callback delayed by 1.717s
W0821 15:09:56.970185 912 periodic.cc:174] PeriodicTimer callback delayed by 2.485s
W0821 15:09:53.870658 141195 outbound_call.cc:526] RPC callback for RPC call 0x000050403dcaf180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313182, state: TIMED_OUT, transfer_state: PENDING took 22.360s
W0821 15:10:01.544361 910 periodic.cc:174] PeriodicTimer callback delayed by 2.354s
W0821 15:10:03.501525 912 thread.cc:888] CatalogManagerBGTasks_6289_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.046s user 0.007s sys 0.042s
W0821 15:10:05.335983 912 thread.cc:844] CatalogManagerBGTasks_6289_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.953s user 0.009s sys 0.076s
W0821 15:10:07.379878 910 periodic.cc:174] PeriodicTimer callback delayed by 2.235s
W0821 15:10:14.056226 912 periodic.cc:174] PeriodicTimer callback delayed by 1.682s
W0821 15:10:17.247501 912 thread.cc:850] raft_19766_worker (raft) Time spent creating pthread: real 0.656s user 0.000s sys 0.015s
W0821 15:10:20.042155 912 thread.cc:844] raft_19766_worker (raft) Time spent starting thread: real 3.263s user 0.000s sys 0.074s
I0821 15:10:19.119073 141227 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:10:22.523904 141227 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:10:27.714684 912 periodic.cc:174] PeriodicTimer callback delayed by 9.246s
I0821 15:10:27.125136 141227 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.647s
W0821 15:10:33.939281 913 periodic.cc:174] PeriodicTimer callback delayed by 2.331s
W0821 15:10:40.373456 913 periodic.cc:174] PeriodicTimer callback delayed by 4.250s
W0821 15:10:39.666250 912 thread.cc:844] CatalogManagerBGTasks_6290_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.960s user 0.001s sys 0.040s
W0821 15:10:43.731098 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x00005040379ef500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313189, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112638.771s (99.674s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112665.591s (72.853s ago), now: 112738.445s, connection: Connection (0x000050403bbf9de0) client 10.0.0.2:34245 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:10:46.855665 913 periodic.cc:174] PeriodicTimer callback delayed by 1.671s
W0821 15:10:51.629823 916 thread.cc:844] Master-high-pri_556_worker (Master-high-pri) Time spent starting thread: real 0.643s user 0.000s sys 0.014s
W0821 15:10:50.917073 141230 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313189) to 10.0.0.4:7100 timed out after 7.570s
W0821 15:10:54.834839 910 periodic.cc:174] PeriodicTimer callback delayed by 2.594s
W0821 15:10:54.620453 141230 outbound_call.cc:526] RPC callback for RPC call 0x00005040379ef500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313189, state: TIMED_OUT, transfer_state: PENDING took 3.482s
W0821 15:11:02.652429 911 periodic.cc:174] PeriodicTimer callback delayed by 2.886s
W0821 15:11:10.131886 910 periodic.cc:174] PeriodicTimer callback delayed by 2.368s
W0821 15:11:16.102883 913 thread.cc:850] CatalogManagerBGTasks_6291_worker (CatalogManagerBGTasks) Time spent creating pthread: real 2.978s user 0.000s sys 0.069s
W0821 15:11:16.476310 912 periodic.cc:174] PeriodicTimer callback delayed by 2.207s
W0821 15:11:19.302992 913 thread.cc:844] CatalogManagerBGTasks_6291_worker (CatalogManagerBGTasks) Time spent starting thread: real 7.265s user 0.001s sys 0.159s
W0821 15:11:23.864972 912 periodic.cc:174] PeriodicTimer callback delayed by 2.860s
W0821 15:11:29.726555 911 periodic.cc:174] PeriodicTimer callback delayed by 1.903s
W0821 15:11:37.727510 911 periodic.cc:174] PeriodicTimer callback delayed by 3.394s
I0821 15:10:30.299988 141227 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:11:44.047677 910 periodic.cc:174] PeriodicTimer callback delayed by 2.462s
I0821 15:11:43.995252 141227 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:11:44.905153 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf29500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313190, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112692.170s (102.936s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112718.407s (76.698s ago), now: 112795.106s, connection: Connection (0x000050403b1fd6e0) client 10.0.0.2:35279 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:11:52.811652 916 thread.cc:844] Master-high-pri_557_worker (Master-high-pri) Time spent starting thread: real 0.959s user 0.000s sys 0.024s
W0821 15:11:52.811961 911 thread.cc:844] CatalogManagerBGTasks_6292_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.959s user 0.002s sys 0.025s
W0821 15:11:52.102289 141254 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313190) to 10.0.0.3:7100 timed out after 4.215s
W0821 15:11:53.550611 912 periodic.cc:174] PeriodicTimer callback delayed by 4.113s
I0821 15:11:54.741228 141227 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:11:54.127573 141254 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf29500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 1 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313190, state: TIMED_OUT, transfer_state: PENDING took 2.043s
W0821 15:11:58.546835 910 periodic.cc:174] PeriodicTimer callback delayed by 1.312s
W0821 15:12:07.232247 913 periodic.cc:174] PeriodicTimer callback delayed by 3.320s
I0821 15:12:10.482872 141270 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:12:09.908495 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403d92b180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313188, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112634.696s (191.539s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112660.294s (165.941s ago), now: 112826.236s, connection: Connection (0x0000504039e82560) client 10.0.0.2:45649 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:12:10.888257 141270 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:12:12.519392 141270 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 18.015s
W0821 15:12:13.750917 915 thread.cc:850] Master-high-pri_558_worker (Master-high-pri) Time spent creating pthread: real 0.589s user 0.000s sys 0.009s
W0821 15:12:15.266942 910 periodic.cc:174] PeriodicTimer callback delayed by 3.606s
W0821 15:12:15.642071 915 thread.cc:844] Master-high-pri_558_worker (Master-high-pri) Time spent starting thread: real 3.009s user 0.000s sys 0.039s
W0821 15:12:15.195147 141274 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313188) to 10.0.0.3:7100 timed out after 7.570s
I0821 15:12:17.499795 141274 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:12:20.726071 910 periodic.cc:174] PeriodicTimer callback delayed by 3.234s
W0821 15:12:22.757246 141274 thread.cc:844] raft_19768_worker (raft) Time spent starting thread: real 1.998s user 0.001s sys 0.041s
W0821 15:12:25.749715 913 periodic.cc:174] PeriodicTimer callback delayed by 2.702s
W0821 15:12:26.954187 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403d9ef880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313187, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112608.080s (237.710s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112663.873s (181.917s ago), now: 112845.790s, connection: Connection (0x000050403b99d1a0) client 10.0.0.2:45781 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:12:27.787894 911 thread.cc:888] CatalogManagerBGTasks_6293_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.706s user 0.000s sys 0.016s
W0821 15:12:29.601508 911 thread.cc:844] CatalogManagerBGTasks_6293_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.640s user 0.000s sys 0.044s
W0821 15:12:29.584226 141274 outbound_call.cc:526] RPC callback for RPC call 0x000050403d92b180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313188, state: TIMED_OUT, transfer_state: PENDING took 14.346s
W0821 15:12:33.003762 910 periodic.cc:174] PeriodicTimer callback delayed by 5.080s
W0821 15:12:34.436149 915 thread.cc:844] Master-high-pri_559_worker (Master-high-pri) Time spent starting thread: real 1.484s user 0.000s sys 0.031s
W0821 15:12:34.282907 141290 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313187) to 10.0.0.4:7100 timed out after 14.554s
I0821 15:12:36.653456 141290 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:12:13.789114 141270 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:12:39.858453 912 periodic.cc:174] PeriodicTimer callback delayed by 2.302s
W0821 15:12:41.189954 141290 thread.cc:888] raft_19769_worker (raft) Time spent waiting for new thread to publish its TID: real 0.687s user 0.007s sys 0.025s
I0821 15:12:41.748644 141270 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:12:43.418927 141290 thread.cc:844] raft_19769_worker (raft) Time spent starting thread: real 3.028s user 0.007s sys 0.060s
I0821 15:12:42.003723 141282 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 17.566s
I0821 15:12:46.158134 141282 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:12:46.223719 141294 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.867s
W0821 15:12:48.006839 913 periodic.cc:174] PeriodicTimer callback delayed by 3.765s
I0821 15:12:49.016161 141270 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 15:12:50.304721 141294 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:12:53.689788 911 periodic.cc:174] PeriodicTimer callback delayed by 3.421s
W0821 15:12:51.288710 141290 outbound_call.cc:526] RPC callback for RPC call 0x000050403d9ef880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313187, state: TIMED_OUT, transfer_state: PENDING took 16.935s
W0821 15:13:00.910894 913 thread.cc:850] CatalogManagerBGTasks_6294_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.522s user 0.000s sys 0.011s
W0821 15:13:03.574401 912 periodic.cc:174] PeriodicTimer callback delayed by 5.099s
W0821 15:13:05.768383 913 thread.cc:844] CatalogManagerBGTasks_6294_worker (CatalogManagerBGTasks) Time spent starting thread: real 6.135s user 0.003s sys 0.117s
W0821 15:13:09.386667 912 thread.cc:850] raft_19770_worker (raft) Time spent creating pthread: real 0.900s user 0.000s sys 0.019s
I0821 15:13:10.951573 141305 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:13:11.913795 912 thread.cc:844] raft_19770_worker (raft) Time spent starting thread: real 3.634s user 0.001s sys 0.067s
I0821 15:13:12.718086 141305 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:13:14.118512 141305 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 18.090s
I0821 15:13:15.549610 141305 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:13:17.609516 141305 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:13:20.690259 911 periodic.cc:174] PeriodicTimer callback delayed by 11.495s
I0821 15:13:24.487998 141305 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:13:27.995558 910 periodic.cc:174] PeriodicTimer callback delayed by 5.101s
W0821 15:13:33.458416 910 periodic.cc:174] PeriodicTimer callback delayed by 3.206s
W0821 15:13:36.494766 911 periodic.cc:174] PeriodicTimer callback delayed by 2.865s
W0821 15:13:39.892409 912 thread.cc:850] CatalogManagerBGTasks_6295_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.715s user 0.000s sys 0.009s
W0821 15:13:39.892577 911 thread.cc:850] raft_19771_worker (raft) Time spent creating pthread: real 0.715s user 0.000s sys 0.009s
I0821 15:13:42.059883 141331 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:13:42.630716 912 thread.cc:844] CatalogManagerBGTasks_6295_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.128s user 0.005s sys 0.056s
W0821 15:13:42.630790 911 thread.cc:844] raft_19771_worker (raft) Time spent starting thread: real 3.627s user 0.001s sys 0.047s
I0821 15:13:43.725178 141331 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:13:44.882377 141331 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.616s
I0821 15:13:46.315433 141331 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:13:46.837508 141331 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:13:50.827795 911 periodic.cc:174] PeriodicTimer callback delayed by 9.830s
I0821 15:13:50.834271 141331 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:13:54.635937 913 periodic.cc:174] PeriodicTimer callback delayed by 1.514s
W0821 15:13:56.045701 911 periodic.cc:174] PeriodicTimer callback delayed by 1.159s
I0821 15:13:59.577617 141345 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:14:01.059082 911 thread.cc:844] raft_19772_worker (raft) Time spent starting thread: real 0.937s user 0.000s sys 0.025s
I0821 15:14:01.617013 141345 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:14:03.115190 141345 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 12.339s
W0821 15:14:08.301216 911 periodic.cc:174] PeriodicTimer callback delayed by 8.236s
W0821 15:14:13.482829 910 periodic.cc:174] PeriodicTimer callback delayed by 2.957s
W0821 15:14:16.909222 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403ce37c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313195, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112880.113s (77.694s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112923.046s (34.761s ago), now: 112957.808s, connection: Connection (0x0000504038e743a0) client 10.0.0.2:37883 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:14:16.694212 911 thread.cc:844] CatalogManagerBGTasks_6296_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.251s user 0.001s sys 0.031s
W0821 15:14:17.887456 912 periodic.cc:174] PeriodicTimer callback delayed by 2.404s
W0821 15:14:23.762699 910 periodic.cc:174] PeriodicTimer callback delayed by 1.965s
W0821 15:14:23.826274 915 thread.cc:850] Master-high-pri_560_worker (Master-high-pri) Time spent creating pthread: real 0.710s user 0.000s sys 0.015s
W0821 15:14:25.825614 915 thread.cc:844] Master-high-pri_560_worker (Master-high-pri) Time spent starting thread: real 2.889s user 0.000s sys 0.054s
W0821 15:14:25.967024 141363 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313195) to 10.0.0.4:7100 timed out after 18.015s
W0821 15:14:31.088775 913 periodic.cc:174] PeriodicTimer callback delayed by 2.048s
W0821 15:14:29.559496 141363 outbound_call.cc:526] RPC callback for RPC call 0x000050403ce37c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313195, state: TIMED_OUT, transfer_state: PENDING took 3.558s
I0821 15:14:03.115407 141345 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:14:36.622079 141345 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:14:37.315145 913 periodic.cc:174] PeriodicTimer callback delayed by 1.869s
W0821 15:14:44.481621 910 periodic.cc:174] PeriodicTimer callback delayed by 2.293s
I0821 15:14:43.482197 141345 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:14:51.761678 912 periodic.cc:174] PeriodicTimer callback delayed by 3.247s
W0821 15:14:52.347157 913 thread.cc:844] CatalogManagerBGTasks_6297_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.322s user 0.003s sys 0.028s
W0821 15:14:59.482142 912 periodic.cc:174] PeriodicTimer callback delayed by 2.810s
W0821 15:15:05.261955 912 thread.cc:844] raft_19773_worker (raft) Time spent starting thread: real 1.132s user 0.000s sys 0.024s
I0821 15:15:04.550566 141383 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:15:07.741971 141383 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:15:10.696277 141383 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.900s
W0821 15:15:12.301942 912 periodic.cc:174] PeriodicTimer callback delayed by 8.449s
W0821 15:15:20.226490 911 periodic.cc:174] PeriodicTimer callback delayed by 3.738s
W0821 15:15:26.835567 910 periodic.cc:174] PeriodicTimer callback delayed by 3.134s
W0821 15:15:31.823987 913 thread.cc:844] CatalogManagerBGTasks_6298_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.503s user 0.001s sys 0.031s
W0821 15:15:34.576849 910 periodic.cc:174] PeriodicTimer callback delayed by 3.628s
I0821 15:15:14.360164 141383 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:15:46.493242 910 periodic.cc:174] PeriodicTimer callback delayed by 6.709s
I0821 15:15:47.740087 141383 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:15:51.803323 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403e2ac000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313196, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 112907.555s (135.633s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 112956.704s (86.484s ago), now: 113043.188s, connection: Connection (0x0000504038e44020) client 10.0.0.2:43885 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:15:54.205762 910 periodic.cc:174] PeriodicTimer callback delayed by 3.759s
I0821 15:15:56.382911 141383 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:15:59.002247 915 thread.cc:888] Master-high-pri_561_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.525s user 0.000s sys 0.014s
W0821 15:15:59.280426 141405 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313196) to 10.0.0.3:7100 timed out after 18.090s
W0821 15:16:00.974238 915 thread.cc:844] Master-high-pri_561_worker (Master-high-pri) Time spent starting thread: real 2.470s user 0.000s sys 0.046s
W0821 15:16:01.793685 911 periodic.cc:174] PeriodicTimer callback delayed by 3.354s
W0821 15:16:01.946593 141405 outbound_call.cc:526] RPC callback for RPC call 0x000050403e2ac000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313196, state: TIMED_OUT, transfer_state: PENDING took 2.686s
W0821 15:16:09.483592 913 thread.cc:888] CatalogManagerBGTasks_6299_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.827s user 0.000s sys 0.021s
W0821 15:16:09.813000 912 periodic.cc:174] PeriodicTimer callback delayed by 3.258s
W0821 15:16:11.510658 913 thread.cc:844] CatalogManagerBGTasks_6299_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.594s user 0.000s sys 0.065s
W0821 15:16:14.694376 912 thread.cc:850] raft_19774_worker (raft) Time spent creating pthread: real 0.668s user 0.000s sys 0.012s
I0821 15:16:16.757301 141412 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:16:18.209110 912 thread.cc:844] raft_19774_worker (raft) Time spent starting thread: real 4.784s user 0.000s sys 0.069s
I0821 15:16:19.136960 141412 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:16:21.411661 141412 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.989s
W0821 15:16:25.940721 912 periodic.cc:174] PeriodicTimer callback delayed by 10.823s
W0821 15:16:31.406029 913 periodic.cc:174] PeriodicTimer callback delayed by 3.227s
I0821 15:16:24.317250 141412 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:16:36.715747 911 periodic.cc:174] PeriodicTimer callback delayed by 3.087s
I0821 15:16:38.197165 141412 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:16:39.589218 913 periodic.cc:174] PeriodicTimer callback delayed by 1.284s
W0821 15:16:43.946274 913 thread.cc:888] CatalogManagerBGTasks_6300_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.593s user 0.002s sys 0.021s
W0821 15:16:45.779909 913 thread.cc:844] CatalogManagerBGTasks_6300_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.934s user 0.002s sys 0.055s
W0821 15:16:46.529364 911 periodic.cc:174] PeriodicTimer callback delayed by 2.882s
I0821 15:16:47.229496 141412 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:16:54.661828 913 periodic.cc:174] PeriodicTimer callback delayed by 2.881s
W0821 15:16:59.642668 913 periodic.cc:174] PeriodicTimer callback delayed by 1.438s
I0821 15:17:05.440078 141447 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:17:05.633206 913 thread.cc:844] raft_19775_worker (raft) Time spent starting thread: real 2.282s user 0.002s sys 0.046s
I0821 15:17:07.980373 141447 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:17:09.837478 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.783s
W0821 15:17:12.668224 912 periodic.cc:174] PeriodicTimer callback delayed by 9.003s
W0821 15:17:12.686494 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403f890a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313203, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113066.940s (63.207s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113099.379s (30.769s ago), now: 113130.148s, connection: Connection (0x0000504038e443a0) client 10.0.0.2:35601 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:17:17.709239 913 periodic.cc:174] PeriodicTimer callback delayed by 2.653s
W0821 15:17:18.068588 912 thread.cc:850] CatalogManagerBGTasks_6301_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.816s user 0.000s sys 0.018s
W0821 15:17:19.626540 915 thread.cc:844] Master-high-pri_562_worker (Master-high-pri) Time spent starting thread: real 1.960s user 0.000s sys 0.035s
W0821 15:17:19.520378 141454 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313203) to 10.0.0.4:7100 timed out after 3.900s
W0821 15:17:21.937736 141454 outbound_call.cc:526] RPC callback for RPC call 0x000050403f890a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313203, state: TIMED_OUT, transfer_state: PENDING took 2.446s
W0821 15:17:21.081231 912 thread.cc:844] CatalogManagerBGTasks_6301_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.285s user 0.002s sys 0.074s
I0821 15:17:12.274919 141447 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:17:21.944940 141447 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:17:21.945139 141447 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:17:21.921660 911 periodic.cc:174] PeriodicTimer callback delayed by 2.171s
W0821 15:17:21.931453 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403e0a6380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313204, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113110.620s (40.313s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113137.789s (13.144s ago), now: 113150.934s, connection: Connection (0x0000504038f82020) client 10.0.0.2:33747 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:17:21.976022 141454 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313204) to 10.0.0.3:7100 timed out after 13.989s
I0821 15:17:21.982738 141447 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:17:21.982767 141447 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:17:21.982805 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 18.701s
I0821 15:17:21.982817 141447 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:17:21.982863 141447 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:17:21.982882 141447 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 15:17:22.028384 141454 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)
I0821 15:17:22.028401 141454 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:17:22.029367 141494 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)
I0821 15:17:22.029378 141494 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:17:22.081624 141494 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)
I0821 15:17:22.081640 141494 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:17:22.086622 141494 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)
I0821 15:17:22.086638 141494 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30200. Preceding OpId from leader: 30.30223. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:17:22.096685 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221887375.728s
I0821 15:17:22.096717 141454 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.096751 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.111s
I0821 15:17:22.096764 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.096849 141494 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221887375.728s
I0821 15:17:22.096868 141494 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.096889 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 12.283s
I0821 15:17:22.096897 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.097973 141494 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.098016 141494 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221887375.725s
I0821 15:17:22.104549 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.131549 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.831s
I0821 15:17:22.131601 141504 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.131714 141504 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.131724 141504 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.131803 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.319s
I0821 15:17:22.131834 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.132058 141504 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.132071 141504 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.132082 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.355s
I0821 15:17:22.132093 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.132114 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221887375.694s
I0821 15:17:22.132119 141454 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.132134 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.854s
I0821 15:17:22.132138 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.132807 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221887375.684s
I0821 15:17:22.132822 141454 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.134567 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 14.579s
I0821 15:17:22.134582 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.134620 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.134629 141454 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.134639 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 17.679s
I0821 15:17:22.134644 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.148552 141454 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.809s
I0821 15:17:22.148586 141454 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.148607 141504 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.148618 141504 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.148770 141506 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.579s
I0821 15:17:22.148797 141506 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.148833 141494 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:17:22.148877 141494 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:17:22.148850 141447 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.908s
I0821 15:17:22.148914 141447 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:17:22.148936 141507 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.972s
I0821 15:17:22.148942 141507 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:17:26.381138 911 periodic.cc:174] PeriodicTimer callback delayed by 0.785s
W0821 15:17:26.660009 901 long_operation_tracker.cc:155] Log callback took a long time: 1.969s
W0821 15:17:26.335858 141494 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33257 => 10.0.0.2:7100 (request call id 5042560) took 2761ms (client timeout 3000ms).
W0821 15:17:28.295562 913 periodic.cc:174] PeriodicTimer callback delayed by 1.233s
I0821 15:17:30.913312 141543 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:17:32.013710 141543 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:17:32.019320 141543 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 18.250s
W0821 15:17:35.208019 911 periodic.cc:174] PeriodicTimer callback delayed by 3.180s
W0821 15:17:37.410998 141494 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:44909 => 10.0.0.2:7100 (request call id 5042584) took 5301ms (client timeout 3000ms).
W0821 15:17:39.358521 913 periodic.cc:174] PeriodicTimer callback delayed by 1.911s
W0821 15:17:42.956599 911 periodic.cc:174] PeriodicTimer callback delayed by 1.371s
W0821 15:17:46.831373 911 periodic.cc:174] PeriodicTimer callback delayed by 1.617s
W0821 15:17:50.488121 912 periodic.cc:174] PeriodicTimer callback delayed by 1.360s
W0821 15:17:57.520385 912 periodic.cc:174] PeriodicTimer callback delayed by 2.896s
W0821 15:17:59.797272 911 thread.cc:888] CatalogManagerBGTasks_6302_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.821s user 0.003s sys 0.040s
W0821 15:18:02.110124 911 thread.cc:844] CatalogManagerBGTasks_6302_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.968s user 0.006s sys 0.093s
W0821 15:18:03.997938 913 periodic.cc:174] PeriodicTimer callback delayed by 2.087s
W0821 15:18:09.857914 913 periodic.cc:174] PeriodicTimer callback delayed by 2.127s
W0821 15:18:10.045905 914 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:17:32.024729 141543 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:18:14.694481 141543 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:18:16.449880 141454 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30254. (index mismatch)
W0821 15:18:16.450716 910 periodic.cc:174] PeriodicTimer callback delayed by 1.947s
W0821 15:18:16.378376 917 yb_rpc.cc:553] Connection (0x0000504039f418a0) client 10.0.0.2:40473 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 30.543s, timeout: 15.000s, now: 113202.530s, last_read_time_: 113171.986s
W0821 15:18:16.457213 914 yb_rpc.cc:553] Connection (0x000050403b0991a0) client 10.0.0.2:41295 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.954s, timeout: 15.000s, now: 113191.303s, last_read_time_: 113156.349s
I0821 15:18:16.458134 141454 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30254. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 15:18:16.517416 141454 long_operation_tracker.cc:155] UpdateReplica took a long time: 44.298s
W0821 15:18:19.018306 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:18:20.759002 911 periodic.cc:174] PeriodicTimer callback delayed by 1.376s
I0821 15:18:20.188431 141543 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:18:19.260600 141454 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36815 => 10.0.0.2:7100 (request call id 5042599) took 47155ms (client timeout 3000ms).
W0821 15:18:27.793989 910 periodic.cc:174] PeriodicTimer callback delayed by 3.045s
I0821 15:18:28.768671 141543 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:18:28.738893 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:18:31.456327 141543 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:18:33.850867 910 thread.cc:850] CatalogManagerBGTasks_6303_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.623s user 0.000s sys 0.014s
I0821 15:18:32.664760 141543 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 7.983s
W0821 15:18:35.729316 912 periodic.cc:174] PeriodicTimer callback delayed by 3.618s
W0821 15:18:36.613155 910 thread.cc:844] CatalogManagerBGTasks_6303_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.250s user 0.001s sys 0.071s
W0821 15:18:42.598664 911 periodic.cc:174] PeriodicTimer callback delayed by 1.916s
I0821 15:18:37.775394 141543 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:18:46.888381 912 periodic.cc:174] PeriodicTimer callback delayed by 2.225s
W0821 15:18:46.192001 914 yb_rpc.cc:553] Connection (0x000050403b9cb1a0) client 10.0.0.2:44703 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.954s, timeout: 15.000s, now: 113191.303s, last_read_time_: 113156.349s
I0821 15:18:49.092926 141543 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:18:52.145179 915 thread.cc:844] Master-high-pri_565_worker (Master-high-pri) Time spent starting thread: real 2.137s user 0.000s sys 0.048s
W0821 15:18:52.348798 913 periodic.cc:174] PeriodicTimer callback delayed by 1.560s
W0821 15:18:49.964000 917 yb_rpc.cc:553] Connection (0x000050403b974fe0) client 10.0.0.2:39545 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 30.543s, timeout: 15.000s, now: 113202.530s, last_read_time_: 113171.986s
W0821 15:18:50.009045 910 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33257 => 10.0.0.2:7100 (request call id 5042629) took 7783ms (client timeout 3000ms).
I0821 15:18:56.762656 141543 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:18:59.381742 913 periodic.cc:174] PeriodicTimer callback delayed by 2.374s
W0821 15:19:02.266145 915 inbound_call.cc:111] 0x000050403875d400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33257 => 10.0.0.2:7100 (request call id 5042629): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33257 => 10.0.0.2:7100 (request call id 5042629) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:19:02.456691 141596 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:39123 => 10.0.0.2:7100 (request call id 5042616) took 6422ms (client timeout 3000ms).
W0821 15:19:08.663723 912 periodic.cc:174] PeriodicTimer callback delayed by 4.213s
W0821 15:19:10.314564 910 thread.cc:850] CatalogManagerBGTasks_6304_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.610s user 0.000s sys 0.011s
I0821 15:19:11.639817 141609 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:19:11.805342 912 thread.cc:888] raft_19779_worker (raft) Time spent waiting for new thread to publish its TID: real 0.643s user 0.002s sys 0.023s
W0821 15:19:12.136495 910 thread.cc:844] CatalogManagerBGTasks_6304_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.386s user 0.001s sys 0.036s
W0821 15:19:13.777508 912 thread.cc:844] raft_19779_worker (raft) Time spent starting thread: real 3.315s user 0.002s sys 0.064s
I0821 15:19:13.675552 141609 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:19:14.289162 141609 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 15.532s
W0821 15:19:14.411487 914 yb_rpc.cc:553] Connection (0x000050403b926aa0) client 10.0.0.2:39907 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.954s, timeout: 15.000s, now: 113191.303s, last_read_time_: 113156.349s
W0821 15:19:18.449442 910 periodic.cc:174] PeriodicTimer callback delayed by 5.941s
W0821 15:19:23.288131 910 periodic.cc:174] PeriodicTimer callback delayed by 2.586s
W0821 15:19:27.863441 915 inbound_call.cc:111] 0x000050403dad0c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:39123 => 10.0.0.2:7100 (request call id 5042616): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:39123 => 10.0.0.2:7100 (request call id 5042616) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:19:31.611410 911 periodic.cc:174] PeriodicTimer callback delayed by 6.030s
W0821 15:19:31.277207 917 yb_rpc.cc:553] Connection (0x000050403b8ca3a0) client 10.0.0.2:39667 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 30.543s, timeout: 15.000s, now: 113202.530s, last_read_time_: 113171.986s
W0821 15:19:39.018541 910 periodic.cc:174] PeriodicTimer callback delayed by 2.602s
W0821 15:19:47.495600 913 thread.cc:844] CatalogManagerBGTasks_6305_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.473s user 0.001s sys 0.023s
I0821 15:19:14.412187 141609 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:19:47.988071 910 periodic.cc:174] PeriodicTimer callback delayed by 3.657s
I0821 15:19:50.453363 141609 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:19:57.195176 912 periodic.cc:174] PeriodicTimer callback delayed by 3.929s
W0821 15:20:01.016613 910 periodic.cc:174] PeriodicTimer callback delayed by 0.790s
I0821 15:20:00.828209 141609 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:20:06.882728 913 periodic.cc:174] PeriodicTimer callback delayed by 1.917s
W0821 15:20:03.431445 914 yb_rpc.cc:553] Connection (0x000050403b0cd360) client 10.0.0.2:39289 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.954s, timeout: 15.000s, now: 113191.303s, last_read_time_: 113156.349s
W0821 15:20:13.933637 912 periodic.cc:174] PeriodicTimer callback delayed by 1.950s
I0821 15:20:18.833659 141647 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:20:20.833349 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403d843500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313212, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113240.322s (66.008s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113283.997s (22.333s ago), now: 113306.331s, connection: Connection (0x0000504038ff0aa0) client 10.0.0.2:39763 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:20:21.001955 912 thread.cc:844] raft_19780_worker (raft) Time spent starting thread: real 3.173s user 0.000s sys 0.067s
I0821 15:20:21.728204 141647 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:20:24.018188 141647 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.256s
W0821 15:20:26.090445 913 thread.cc:844] CatalogManagerBGTasks_6306_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.796s user 0.000s sys 0.100s
W0821 15:20:27.778270 911 periodic.cc:174] PeriodicTimer callback delayed by 9.436s
W0821 15:20:26.939323 141649 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313212) to 10.0.0.3:7100 timed out after 7.983s
I0821 15:20:26.183959 141647 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:20:28.905524 917 yb_rpc.cc:553] Connection (0x0000504038ed6e20) client 10.0.0.2:38451 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 30.543s, timeout: 15.000s, now: 113202.530s, last_read_time_: 113171.986s
W0821 15:20:29.773630 141649 outbound_call.cc:526] RPC callback for RPC call 0x000050403d843500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313212, state: TIMED_OUT, transfer_state: PENDING took 2.680s
W0821 15:20:30.391429 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403e0a6380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313211, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113212.213s (123.677s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113291.859s (44.031s ago), now: 113335.890s, connection: Connection (0x0000504038feec60) client 10.0.0.2:44043 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:20:30.395720 141649 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313211) to 10.0.0.4:7100 timed out after 18.250s
I0821 15:20:30.398711 141647 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:20:30.415064 141647 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:20:30.402654 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:20:32.233111 910 periodic.cc:174] PeriodicTimer callback delayed by 2.350s
W0821 15:20:35.973758 911 periodic.cc:174] PeriodicTimer callback delayed by 1.488s
W0821 15:20:38.561403 915 thread.cc:850] Master-high-pri_567_worker (Master-high-pri) Time spent creating pthread: real 0.669s user 0.000s sys 0.013s
W0821 15:20:40.425203 915 thread.cc:844] Master-high-pri_567_worker (Master-high-pri) Time spent starting thread: real 2.869s user 0.000s sys 0.045s
W0821 15:20:37.082613 141649 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33069 => 10.0.0.2:7100 (request call id 5042607) took 4709ms (client timeout 3000ms).
W0821 15:20:40.935437 910 periodic.cc:174] PeriodicTimer callback delayed by 2.694s
W0821 15:20:38.483937 912 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36815 => 10.0.0.2:7100 (request call id 5042645) took 6173ms (client timeout 3000ms).
W0821 15:20:42.476969 915 inbound_call.cc:111] 0x000050403cbbc400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36815 => 10.0.0.2:7100 (request call id 5042599): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36815 => 10.0.0.2:7100 (request call id 5042599) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:20:44.521556 912 periodic.cc:174] PeriodicTimer callback delayed by 2.105s
I0821 15:20:47.626431 141683 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:20:48.602326 912 thread.cc:844] raft_19781_worker (raft) Time spent starting thread: real 1.237s user 0.007s sys 0.048s
I0821 15:20:49.143699 141683 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:20:51.414441 141683 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.615s
W0821 15:20:56.084966 910 periodic.cc:174] PeriodicTimer callback delayed by 7.363s
W0821 15:21:02.306686 910 periodic.cc:174] PeriodicTimer callback delayed by 3.912s
W0821 15:21:04.637817 913 thread.cc:888] CatalogManagerBGTasks_6307_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.904s user 0.005s sys 0.043s
W0821 15:21:05.771492 913 thread.cc:844] CatalogManagerBGTasks_6307_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.018s user 0.007s sys 0.086s
W0821 15:21:09.352720 913 periodic.cc:174] PeriodicTimer callback delayed by 2.724s
W0821 15:21:15.713326 912 periodic.cc:174] PeriodicTimer callback delayed by 2.631s
W0821 15:21:21.387189 911 periodic.cc:174] PeriodicTimer callback delayed by 1.634s
W0821 15:21:23.791785 915 inbound_call.cc:111] 0x000050403e486800 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36815 => 10.0.0.2:7100 (request call id 5042645): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:36815 => 10.0.0.2:7100 (request call id 5042645) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:21:28.005007 910 periodic.cc:174] PeriodicTimer callback delayed by 1.995s
I0821 15:20:54.079071 141683 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:21:32.089875 141683 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:21:37.605697 912 periodic.cc:174] PeriodicTimer callback delayed by 5.296s
W0821 15:21:40.528409 911 thread.cc:844] CatalogManagerBGTasks_6308_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.891s user 0.001s sys 0.040s
I0821 15:21:40.696466 141683 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:21:47.138073 911 periodic.cc:174] PeriodicTimer callback delayed by 4.720s
W0821 15:21:50.482196 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403f878000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313215, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113312.951s (85.131s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113356.577s (41.505s ago), now: 113398.082s, connection: Connection (0x0000504038f9b1a0) client 10.0.0.2:33851 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:21:53.884214 912 periodic.cc:174] PeriodicTimer callback delayed by 1.940s
W0821 15:21:54.163746 141728 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313215) to 10.0.0.4:7100 timed out after 15.532s
W0821 15:21:58.749091 911 periodic.cc:174] PeriodicTimer callback delayed by 1.874s
W0821 15:21:55.669279 141728 outbound_call.cc:526] RPC callback for RPC call 0x000050403f878000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313215, state: TIMED_OUT, transfer_state: PENDING took 1.489s
W0821 15:22:02.392699 911 thread.cc:850] raft_19782_worker (raft) Time spent creating pthread: real 1.086s user 0.000s sys 0.023s
I0821 15:22:04.030937 141735 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:22:04.598893 911 thread.cc:844] raft_19782_worker (raft) Time spent starting thread: real 3.265s user 0.000s sys 0.055s
W0821 15:22:03.402326 914 inbound_call.cc:111] 0x000050403af3f800 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33069 => 10.0.0.2:7100 (request call id 5042607): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33069 => 10.0.0.2:7100 (request call id 5042607) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:22:05.659751 141735 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:22:06.639313 141735 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 6.367s
W0821 15:22:10.322211 912 periodic.cc:174] PeriodicTimer callback delayed by 7.716s
W0821 15:22:16.526396 911 periodic.cc:174] PeriodicTimer callback delayed by 3.949s
W0821 15:22:18.341545 912 thread.cc:888] CatalogManagerBGTasks_6309_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.686s user 0.004s sys 0.029s
W0821 15:22:19.881009 912 thread.cc:844] CatalogManagerBGTasks_6309_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.259s user 0.005s sys 0.070s
W0821 15:22:19.977437 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403e2ebc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313216, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113339.407s (100.215s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113401.541s (38.080s ago), now: 113439.622s, connection: Connection (0x0000504038f7c3a0) client 10.0.0.2:44385 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:22:23.257443 912 periodic.cc:174] PeriodicTimer callback delayed by 1.951s
I0821 15:22:08.669540 141735 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:22:26.744547 141735 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:22:27.500653 917 thread.cc:844] Master-high-pri_569_worker (Master-high-pri) Time spent starting thread: real 1.973s user 0.000s sys 0.046s
W0821 15:22:26.708186 141748 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313216) to 10.0.0.3:7100 timed out after 16.256s
W0821 15:22:30.222844 911 periodic.cc:174] PeriodicTimer callback delayed by 2.438s
W0821 15:22:29.692735 141748 outbound_call.cc:526] RPC callback for RPC call 0x000050403e2ebc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313216, state: TIMED_OUT, transfer_state: PENDING took 2.929s
I0821 15:22:34.595880 141735 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:22:38.147825 913 periodic.cc:174] PeriodicTimer callback delayed by 2.856s
W0821 15:22:45.657281 912 periodic.cc:174] PeriodicTimer callback delayed by 2.330s
W0821 15:22:52.091977 913 periodic.cc:174] PeriodicTimer callback delayed by 2.165s
I0821 15:22:52.124650 141735 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:22:54.066406 141735 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:22:56.819398 911 periodic.cc:174] PeriodicTimer callback delayed by 2.079s
I0821 15:22:56.578559 141735 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.170s
W0821 15:23:02.796356 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403d8e6a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313217, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113339.421s (141.686s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113372.237s (108.869s ago), now: 113481.107s, connection: Connection (0x000050403b0cd6e0) client 10.0.0.2:44761 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:23:05.663627 912 periodic.cc:174] PeriodicTimer callback delayed by 3.255s
W0821 15:23:08.805399 914 thread.cc:850] Master-high-pri_570_worker (Master-high-pri) Time spent creating pthread: real 0.534s user 0.000s sys 0.010s
W0821 15:23:10.205346 913 periodic.cc:174] PeriodicTimer callback delayed by 2.256s
W0821 15:23:11.372792 914 thread.cc:844] Master-high-pri_570_worker (Master-high-pri) Time spent starting thread: real 3.618s user 0.000s sys 0.071s
W0821 15:23:10.868052 141763 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313217) to 10.0.0.4:7100 timed out after 16.256s
I0821 15:23:13.733573 141763 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:23:17.093786 912 periodic.cc:174] PeriodicTimer callback delayed by 2.140s
W0821 15:23:17.271193 141763 thread.cc:844] raft_19783_worker (raft) Time spent starting thread: real 1.286s user 0.001s sys 0.031s
W0821 15:23:23.773562 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403bf6e000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313210, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113205.471s (296.899s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113372.240s (130.130s ago), now: 113502.371s, connection: Connection (0x0000504038f9b520) client 10.0.0.2:35285 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:23:24.185871 911 periodic.cc:174] PeriodicTimer callback delayed by 2.601s
W0821 15:23:26.369215 913 thread.cc:844] CatalogManagerBGTasks_6311_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.890s user 0.002s sys 0.035s
W0821 15:23:24.698946 141763 outbound_call.cc:526] RPC callback for RPC call 0x000050403d8e6a80: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313217, state: TIMED_OUT, transfer_state: PENDING took 13.831s
W0821 15:23:29.804399 912 periodic.cc:174] PeriodicTimer callback delayed by 1.916s
W0821 15:23:31.344673 914 thread.cc:844] Master-high-pri_571_worker (Master-high-pri) Time spent starting thread: real 2.167s user 0.000s sys 0.050s
W0821 15:23:30.314445 141781 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313210) to 10.0.0.3:7100 timed out after 18.250s
I0821 15:23:32.996459 141781 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:22:59.793463 141735 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:23:36.683045 141735 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:23:36.948567 913 periodic.cc:174] PeriodicTimer callback delayed by 2.689s
I0821 15:23:36.698882 141769 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.728s
W0821 15:23:37.307556 141781 thread.cc:844] raft_19784_worker (raft) Time spent starting thread: real 0.569s user 0.000s sys 0.010s
I0821 15:23:38.607707 141769 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:23:39.821219 141787 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 15.054s
I0821 15:23:41.746062 141787 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:23:42.855165 911 periodic.cc:174] PeriodicTimer callback delayed by 1.867s
I0821 15:23:43.301815 141735 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:23:44.293447 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403d9ee000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313218, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113403.365s (118.623s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113442.862s (79.126s ago), now: 113521.988s, connection: Connection (0x0000504038fbc560) client 10.0.0.2:43559 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:23:42.846000 141781 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf6e000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313210, state: TIMED_OUT, transfer_state: PENDING took 12.512s
W0821 15:23:48.805368 911 periodic.cc:174] PeriodicTimer callback delayed by 3.656s
W0821 15:23:50.140595 914 thread.cc:844] Master-high-pri_572_worker (Master-high-pri) Time spent starting thread: real 2.206s user 0.000s sys 0.052s
W0821 15:23:49.877275 141796 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313218) to 10.0.0.3:7100 timed out after 4.615s
W0821 15:23:53.023299 913 periodic.cc:174] PeriodicTimer callback delayed by 1.985s
W0821 15:23:53.178925 141796 outbound_call.cc:526] RPC callback for RPC call 0x000050403d9ee000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313218, state: TIMED_OUT, transfer_state: PENDING took 3.286s
W0821 15:23:57.858136 913 periodic.cc:174] PeriodicTimer callback delayed by 2.423s
I0821 15:24:03.506358 141803 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:24:04.938552 913 thread.cc:844] raft_19785_worker (raft) Time spent starting thread: real 3.519s user 0.000s sys 0.065s
I0821 15:24:05.305979 141803 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:24:06.594142 911 thread.cc:844] CatalogManagerBGTasks_6312_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.594s user 0.005s sys 0.031s
I0821 15:24:08.357105 141803 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.281s
W0821 15:24:13.065364 912 periodic.cc:174] PeriodicTimer callback delayed by 9.823s
W0821 15:24:17.434793 913 periodic.cc:174] PeriodicTimer callback delayed by 2.168s
W0821 15:24:24.080624 911 periodic.cc:174] PeriodicTimer callback delayed by 2.073s
W0821 15:24:31.313498 911 periodic.cc:174] PeriodicTimer callback delayed by 2.372s
W0821 15:24:40.433295 910 periodic.cc:174] PeriodicTimer callback delayed by 4.214s
W0821 15:24:40.433303 911 thread.cc:850] CatalogManagerBGTasks_6313_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.683s user 0.000s sys 0.008s
W0821 15:24:42.577867 911 thread.cc:844] CatalogManagerBGTasks_6313_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.857s user 0.000s sys 0.034s
I0821 15:24:10.995354 141803 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:24:47.035679 913 periodic.cc:174] PeriodicTimer callback delayed by 1.777s
I0821 15:24:47.689954 141803 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:24:53.320677 910 periodic.cc:174] PeriodicTimer callback delayed by 1.957s
I0821 15:24:55.693579 141803 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:25:00.262084 913 periodic.cc:174] PeriodicTimer callback delayed by 1.782s
W0821 15:25:06.596748 911 periodic.cc:174] PeriodicTimer callback delayed by 1.706s
W0821 15:25:10.764791 911 thread.cc:844] raft_19786_worker (raft) Time spent starting thread: real 1.380s user 0.000s sys 0.020s
I0821 15:25:10.763728 141833 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:25:13.607632 141833 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:25:15.474962 141833 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.156s
W0821 15:25:19.000047 913 periodic.cc:174] PeriodicTimer callback delayed by 7.035s
W0821 15:25:20.292865 910 thread.cc:850] CatalogManagerBGTasks_6314_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.552s user 0.000s sys 0.013s
W0821 15:25:23.610848 910 thread.cc:844] CatalogManagerBGTasks_6314_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.562s user 0.000s sys 0.072s
W0821 15:25:26.920192 911 periodic.cc:174] PeriodicTimer callback delayed by 5.328s
I0821 15:25:17.141894 141833 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:25:34.019629 910 periodic.cc:174] PeriodicTimer callback delayed by 2.546s
I0821 15:25:34.750304 141833 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:25:39.436371 141833 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:25:41.512830 910 periodic.cc:174] PeriodicTimer callback delayed by 2.644s
W0821 15:25:49.271001 910 periodic.cc:174] PeriodicTimer callback delayed by 2.862s
W0821 15:25:51.723979 910 thread.cc:844] raft_19787_worker (raft) Time spent starting thread: real 0.533s user 0.001s sys 0.011s
I0821 15:25:51.723934 141862 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:25:51.848691 141862 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:25:53.768287 141862 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.536s
W0821 15:25:55.480430 910 thread.cc:844] CatalogManagerBGTasks_6315_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.233s user 0.000s sys 0.020s
W0821 15:25:56.074958 912 periodic.cc:174] PeriodicTimer callback delayed by 3.157s
W0821 15:26:00.160504 911 periodic.cc:174] PeriodicTimer callback delayed by 1.759s
W0821 15:26:02.228878 910 periodic.cc:174] PeriodicTimer callback delayed by 2.054s
W0821 15:26:10.119336 913 periodic.cc:174] PeriodicTimer callback delayed by 2.961s
W0821 15:26:11.531049 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403ce37c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313226, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113606.957s (61.013s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113638.329s (29.642s ago), now: 113667.971s, connection: Connection (0x0000504038fbc8e0) client 10.0.0.2:38383 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:26:16.747579 913 periodic.cc:174] PeriodicTimer callback delayed by 1.841s
W0821 15:26:20.234714 914 thread.cc:850] Master-high-pri_573_worker (Master-high-pri) Time spent creating pthread: real 0.550s user 0.000s sys 0.012s
W0821 15:26:23.990239 914 thread.cc:844] Master-high-pri_573_worker (Master-high-pri) Time spent starting thread: real 4.265s user 0.000s sys 0.094s
W0821 15:26:22.569326 141882 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313226) to 10.0.0.4:7100 timed out after 4.281s
I0821 15:25:54.638173 141862 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:26:26.163623 911 periodic.cc:174] PeriodicTimer callback delayed by 4.028s
I0821 15:26:28.150949 141862 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:26:25.922434 141882 outbound_call.cc:526] RPC callback for RPC call 0x000050403ce37c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313226, state: TIMED_OUT, transfer_state: PENDING took 3.241s
W0821 15:26:30.031719 912 thread.cc:844] CatalogManagerBGTasks_6316_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.408s user 0.005s sys 0.038s
W0821 15:26:32.588258 910 periodic.cc:174] PeriodicTimer callback delayed by 2.063s
I0821 15:26:35.713781 141862 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:26:39.400607 913 periodic.cc:174] PeriodicTimer callback delayed by 2.909s
W0821 15:26:46.182209 911 periodic.cc:174] PeriodicTimer callback delayed by 2.134s
I0821 15:26:54.437553 141889 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:26:54.995221 911 thread.cc:888] raft_19788_worker (raft) Time spent waiting for new thread to publish its TID: real 1.060s user 0.003s sys 0.031s
W0821 15:26:57.601214 911 thread.cc:844] raft_19788_worker (raft) Time spent starting thread: real 5.846s user 0.006s sys 0.136s
I0821 15:26:57.041683 141889 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:27:02.950120 141889 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.937s
W0821 15:27:05.498338 911 periodic.cc:174] PeriodicTimer callback delayed by 14.408s
W0821 15:27:05.713244 912 thread.cc:888] CatalogManagerBGTasks_6317_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.966s user 0.001s sys 0.037s
W0821 15:27:07.546192 912 thread.cc:844] CatalogManagerBGTasks_6317_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.384s user 0.003s sys 0.082s
W0821 15:27:13.612593 910 periodic.cc:174] PeriodicTimer callback delayed by 3.008s
W0821 15:27:17.891969 913 periodic.cc:174] PeriodicTimer callback delayed by 2.056s
W0821 15:27:25.360488 912 periodic.cc:174] PeriodicTimer callback delayed by 5.244s
W0821 15:27:32.575968 912 periodic.cc:174] PeriodicTimer callback delayed by 2.459s
W0821 15:27:40.330894 910 periodic.cc:174] PeriodicTimer callback delayed by 2.630s
W0821 15:27:40.532553 911 thread.cc:888] CatalogManagerBGTasks_6318_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.811s user 0.005s sys 0.062s
W0821 15:27:43.637216 911 thread.cc:844] CatalogManagerBGTasks_6318_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.707s user 0.007s sys 0.120s
W0821 15:27:46.573843 912 periodic.cc:174] PeriodicTimer callback delayed by 2.378s
W0821 15:27:53.876778 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403e2ebc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313227, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113646.055s (120.542s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113662.521s (104.076s ago), now: 113766.598s, connection: Connection (0x0000504038eca720) client 10.0.0.2:35225 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:27:53.880663 913 periodic.cc:174] PeriodicTimer callback delayed by 2.132s
W0821 15:27:56.906335 914 thread.cc:888] Master-high-pri_574_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.684s user 0.000s sys 0.031s
W0821 15:27:57.304026 141913 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313227) to 10.0.0.3:7100 timed out after 9.156s
W0821 15:27:58.650576 914 thread.cc:844] Master-high-pri_574_worker (Master-high-pri) Time spent starting thread: real 2.806s user 0.000s sys 0.068s
W0821 15:28:00.602139 911 periodic.cc:174] PeriodicTimer callback delayed by 2.088s
W0821 15:27:59.175328 141913 outbound_call.cc:526] RPC callback for RPC call 0x000050403e2ebc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313227, state: TIMED_OUT, transfer_state: PENDING took 1.881s
I0821 15:27:08.150956 141889 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:28:04.853133 910 periodic.cc:174] PeriodicTimer callback delayed by 1.336s
I0821 15:28:06.332414 141889 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:28:10.985381 912 periodic.cc:174] PeriodicTimer callback delayed by 2.014s
W0821 15:28:14.509141 910 thread.cc:850] CatalogManagerBGTasks_6319_worker (CatalogManagerBGTasks) Time spent creating pthread: real 1.245s user 0.002s sys 0.026s
I0821 15:28:16.284925 141889 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:28:17.123672 910 thread.cc:844] CatalogManagerBGTasks_6319_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.812s user 0.004s sys 0.075s
W0821 15:28:19.095119 911 periodic.cc:174] PeriodicTimer callback delayed by 2.706s
W0821 15:28:23.028239 912 periodic.cc:174] PeriodicTimer callback delayed by 0.718s
W0821 15:28:28.383060 912 periodic.cc:174] PeriodicTimer callback delayed by 1.166s
I0821 15:28:32.376116 141934 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:28:33.470386 912 thread.cc:888] raft_19789_worker (raft) Time spent waiting for new thread to publish its TID: real 0.653s user 0.002s sys 0.013s
W0821 15:28:35.494581 912 thread.cc:844] raft_19789_worker (raft) Time spent starting thread: real 4.880s user 0.003s sys 0.074s
I0821 15:28:35.838990 141934 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:28:37.945395 141934 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.492s
W0821 15:28:42.103909 911 periodic.cc:174] PeriodicTimer callback delayed by 9.671s
W0821 15:28:48.176599 910 periodic.cc:174] PeriodicTimer callback delayed by 3.815s
W0821 15:28:51.530031 913 thread.cc:850] CatalogManagerBGTasks_6320_worker (CatalogManagerBGTasks) Time spent creating pthread: real 1.106s user 0.000s sys 0.024s
W0821 15:28:52.368989 911 periodic.cc:174] PeriodicTimer callback delayed by 4.063s
W0821 15:28:54.520224 913 thread.cc:844] CatalogManagerBGTasks_6320_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.148s user 0.002s sys 0.081s
W0821 15:29:02.181605 911 periodic.cc:174] PeriodicTimer callback delayed by 5.877s
I0821 15:28:39.787920 141934 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:29:09.931547 912 periodic.cc:174] PeriodicTimer callback delayed by 2.439s
I0821 15:29:10.447017 141934 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:29:17.119158 912 periodic.cc:174] PeriodicTimer callback delayed by 2.545s
I0821 15:29:18.073598 141934 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:29:25.996884 913 periodic.cc:174] PeriodicTimer callback delayed by 3.733s
W0821 15:29:26.657682 910 thread.cc:850] CatalogManagerBGTasks_6321_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.578s user 0.000s sys 0.012s
W0821 15:29:28.282024 910 thread.cc:844] CatalogManagerBGTasks_6321_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.523s user 0.000s sys 0.050s
W0821 15:29:29.712401 913 thread.cc:888] raft_19790_worker (raft) Time spent waiting for new thread to publish its TID: real 1.448s user 0.004s sys 0.049s
I0821 15:29:29.500244 141971 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:29:31.462512 913 thread.cc:844] raft_19790_worker (raft) Time spent starting thread: real 3.858s user 0.005s sys 0.087s
I0821 15:29:32.100914 141971 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:29:33.857649 141971 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.848s
W0821 15:29:37.596109 911 periodic.cc:174] PeriodicTimer callback delayed by 8.104s
W0821 15:29:41.881103 911 periodic.cc:174] PeriodicTimer callback delayed by 2.017s
W0821 15:29:46.167421 911 periodic.cc:174] PeriodicTimer callback delayed by 2.036s
W0821 15:29:48.317405 910 periodic.cc:174] PeriodicTimer callback delayed by 1.452s
I0821 15:29:35.954124 141971 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:29:54.007151 911 periodic.cc:174] PeriodicTimer callback delayed by 1.731s
I0821 15:29:55.015839 141971 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:30:01.410219 911 periodic.cc:174] PeriodicTimer callback delayed by 2.437s
W0821 15:30:01.863090 910 thread.cc:844] CatalogManagerBGTasks_6322_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.499s user 0.001s sys 0.040s
I0821 15:30:04.522805 141971 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:30:08.189170 912 periodic.cc:174] PeriodicTimer callback delayed by 2.075s
W0821 15:30:14.894707 910 periodic.cc:174] PeriodicTimer callback delayed by 2.834s
W0821 15:30:18.219453 910 thread.cc:844] raft_19791_worker (raft) Time spent starting thread: real 0.884s user 0.001s sys 0.013s
I0821 15:30:18.219553 142013 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:30:19.662421 142013 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:30:21.848575 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 7.236s
W0821 15:30:24.561326 911 periodic.cc:174] PeriodicTimer callback delayed by 5.072s
W0821 15:30:28.860037 911 periodic.cc:174] PeriodicTimer callback delayed by 2.129s
W0821 15:30:33.078402 912 periodic.cc:174] PeriodicTimer callback delayed by 2.005s
W0821 15:30:35.940896 910 thread.cc:850] CatalogManagerBGTasks_6323_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.598s user 0.000s sys 0.013s
W0821 15:30:38.859854 910 thread.cc:844] CatalogManagerBGTasks_6323_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.123s user 0.001s sys 0.065s
W0821 15:30:39.715780 911 periodic.cc:174] PeriodicTimer callback delayed by 2.428s
W0821 15:30:47.376477 910 periodic.cc:174] PeriodicTimer callback delayed by 2.785s
I0821 15:30:24.106514 142013 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:30:54.494426 911 periodic.cc:174] PeriodicTimer callback delayed by 2.020s
I0821 15:30:55.811439 142013 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:31:03.423328 913 periodic.cc:174] PeriodicTimer callback delayed by 4.004s
I0821 15:31:02.628568 142013 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:31:11.211735 913 periodic.cc:174] PeriodicTimer callback delayed by 2.712s
I0821 15:31:12.930397 142013 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:31:12.861604 911 thread.cc:888] CatalogManagerBGTasks_6324_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.066s user 0.003s sys 0.032s
W0821 15:31:15.141038 911 thread.cc:844] CatalogManagerBGTasks_6324_worker (CatalogManagerBGTasks) Time spent starting thread: real 5.180s user 0.004s sys 0.105s
I0821 15:31:15.071348 142013 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:31:17.193563 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.984s
W0821 15:31:19.159116 910 periodic.cc:174] PeriodicTimer callback delayed by 3.350s
W0821 15:31:25.055795 912 periodic.cc:174] PeriodicTimer callback delayed by 1.710s
W0821 15:31:31.035930 912 periodic.cc:174] PeriodicTimer callback delayed by 3.726s
W0821 15:31:35.233057 913 periodic.cc:174] PeriodicTimer callback delayed by 1.979s
W0821 15:31:39.132133 911 periodic.cc:174] PeriodicTimer callback delayed by 1.652s
W0821 15:31:41.261811 912 periodic.cc:174] PeriodicTimer callback delayed by 1.115s
W0821 15:31:49.527341 910 periodic.cc:174] PeriodicTimer callback delayed by 4.393s
W0821 15:31:53.360978 912 thread.cc:844] CatalogManagerBGTasks_6325_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.695s user 0.001s sys 0.045s
W0821 15:31:56.570727 911 periodic.cc:174] PeriodicTimer callback delayed by 2.388s
I0821 15:31:20.797079 142013 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:32:01.294518 142013 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:32:01.294586 142013 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:32:01.296022 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403da31500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313234, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 113869.283s (161.018s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 113886.386s (143.915s ago), now: 114030.302s, connection: Connection (0x0000504038fb58a0) client 10.0.0.2:39495 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:32:01.305619 916 yb_rpc.cc:553] Connection (0x0000504038e44aa0) client 10.0.0.2:39703 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
W0821 15:32:01.313052 142119 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313234) to 10.0.0.4:7100 timed out after 8.492s
W0821 15:32:01.314493 142120 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
I0821 15:32:01.314513 142120 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:32:01.314577 916 yb_rpc.cc:553] Connection (0x0000504038e631a0) client 10.0.0.2:36731 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
W0821 15:32:01.314720 142119 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
I0821 15:32:01.315016 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.614s
I0821 15:32:01.315029 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:32:01.315994 142121 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
W0821 15:32:01.316038 916 yb_rpc.cc:553] Connection (0x0000504038e34c60) client 10.0.0.2:33329 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
W0821 15:32:01.316077 916 yb_rpc.cc:553] Connection (0x0000504038e621e0) client 10.0.0.2:46015 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 864.638s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113165.610s
W0821 15:32:01.316828 142119 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
W0821 15:32:01.317073 142122 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 864.638s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113165.610s
I0821 15:32:01.317095 142122 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.317114 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 18.092s
I0821 15:32:01.317121 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:32:01.317164 142120 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 857.797s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113172.451s
W0821 15:32:01.318642 142120 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 864.638s, timeout: 15.000s, now: 114030.249s, last_read_time_: 113165.610s
W0821 15:32:01.335659 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403dfc2700: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313219, state: TIMED_OUT, transfer_state: PENDING, start_time: 113411.471s (618.871s ago), sent_time: -inf, trigger_callback_time: 114030.340s (0.002s ago), invoke_callback_time: -inf, expiration_time: 113432.868s (597.473s ago), now: 114030.342s, connection: Connection (0x0000504038ffac60) client 10.0.0.2:37703 => 10.0.0.4:7100, active_call_state: kResetOnExpiration
W0821 15:32:01.337805 142122 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313219) to 10.0.0.4:7100 timed out after 4.615s
I0821 15:32:01.337829 142122 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.340279 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.658s
I0821 15:32:01.340302 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:32:01.343005 142120 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x0000504038ffac60) client 10.0.0.2:37703 => 10.0.0.4:7100, passed: 602.867s, timeout: 15.000s: kConnectFailed (network error 1)
I0821 15:32:01.354588 142120 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886496.464s
I0821 15:32:01.354616 142120 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.354645 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.149s
I0821 15:32:01.354656 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.354666 142122 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:32:01.354735 142119 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886496.464s
I0821 15:32:01.354758 142119 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.354771 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 15.994s
I0821 15:32:01.354780 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.356877 142122 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)
I0821 15:32:01.356894 142122 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:01.356946 142119 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)
I0821 15:32:01.356956 142119 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:01.369592 142120 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)
I0821 15:32:01.369622 142120 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:01.369678 142119 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)
I0821 15:32:01.369684 142119 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:01.369712 142122 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)
I0821 15:32:01.369719 142122 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30245. Preceding OpId from leader: 30.30266. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:01.371470 142122 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886496.445s
I0821 15:32:01.371491 142122 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.371511 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.558s
I0821 15:32:01.371520 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.393342 142122 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.742s
I0821 15:32:01.393376 142122 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.393410 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.157s
I0821 15:32:01.393420 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.393431 142119 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.738s
I0821 15:32:01.393438 142119 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.393456 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.291s
I0821 15:32:01.393460 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.393599 142120 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.740s
I0821 15:32:01.393615 142120 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.393641 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.113s
I0821 15:32:01.393648 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.394971 142122 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.740s
I0821 15:32:01.394991 142122 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.395013 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.077s
I0821 15:32:01.395027 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.395035 142120 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:32:01.395042 142120 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.395056 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.840s
I0821 15:32:01.395061 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.395069 142121 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:32:01.395078 142121 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.395097 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.608s
I0821 15:32:01.395104 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:32:01.395114 142119 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.740s
I0821 15:32:01.395123 142119 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:32:01.395143 142013 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.277s
I0821 15:32:01.395149 142013 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:32:03.698298 913 periodic.cc:174] PeriodicTimer callback delayed by 1.886s
W0821 15:32:08.111951 913 periodic.cc:174] PeriodicTimer callback delayed by 2.242s
I0821 15:32:13.012635 142166 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:32:13.050271 142166 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:32:13.062178 142166 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 14.400s
I0821 15:32:13.070999 142166 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:32:13.501906 142166 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:32:13.522948 142120 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30288. Preceding OpId from leader: 30.30291. (index mismatch)
I0821 15:32:13.572088 142166 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 15:32:13.559286 142120 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30288. Preceding OpId from leader: 30.30291. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:13.749370 142119 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30288. Preceding OpId from leader: 30.30293. (index mismatch)
I0821 15:32:13.754149 142119 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30288. Preceding OpId from leader: 30.30293. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:13.832994 142121 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30288. Preceding OpId from leader: 30.30291. (index mismatch)
I0821 15:32:13.841171 142121 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30288. Preceding OpId from leader: 30.30291. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:32:13.845968 142119 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886483.978s
W0821 15:32:17.905191 910 periodic.cc:174] PeriodicTimer callback delayed by 5.361s
I0821 15:32:20.841364 142231 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:32:20.842257 910 thread.cc:844] raft_19793_worker (raft) Time spent starting thread: real 1.164s user 0.005s sys 0.031s
W0821 15:32:26.488581 912 periodic.cc:174] PeriodicTimer callback delayed by 3.985s
W0821 15:32:29.040704 910 thread.cc:844] CatalogManagerBGTasks_6326_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.014s user 0.000s sys 0.037s
W0821 15:32:23.437639 142119 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:34951 => 10.0.0.2:7100 (request call id 5044614) took 6236ms (client timeout 3000ms).
W0821 15:32:33.462348 913 periodic.cc:174] PeriodicTimer callback delayed by 2.067s
W0821 15:32:39.719600 913 periodic.cc:174] PeriodicTimer callback delayed by 2.126s
W0821 15:32:44.536613 142121 long_operation_tracker.cc:155] UpdateReplica took a long time: 30.651s
I0821 15:32:44.554112 142231 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:32:44.743494 917 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:32:45.801963 911 periodic.cc:174] PeriodicTimer callback delayed by 2.020s
I0821 15:32:45.810453 142231 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.926s
W0821 15:32:45.812409 142121 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:37331 => 10.0.0.2:7100 (request call id 5044606) took 31970ms (client timeout 3000ms).
W0821 15:32:45.812876 917 yb_rpc.cc:553] Connection (0x0000504038e57520) client 10.0.0.2:46839 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 16.753s, timeout: 15.000s, now: 114062.605s, last_read_time_: 114045.851s
W0821 15:32:45.830782 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:32:45.887729 916 yb_rpc.cc:553] Connection (0x0000504038fe56e0) client 10.0.0.2:44903 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.688s, timeout: 15.000s, now: 114069.540s, last_read_time_: 114045.851s
W0821 15:32:49.462563 911 periodic.cc:174] PeriodicTimer callback delayed by 0.314s
W0821 15:32:54.031428 911 periodic.cc:174] PeriodicTimer callback delayed by 2.330s
W0821 15:32:49.410575 917 yb_rpc.cc:553] Connection (0x0000504038e58fe0) client 10.0.0.2:41431 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 16.753s, timeout: 15.000s, now: 114062.605s, last_read_time_: 114045.851s
I0821 15:32:45.810505 142231 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:32:59.124922 913 periodic.cc:174] PeriodicTimer callback delayed by 2.852s
I0821 15:32:59.191658 142231 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:32:59.202064 916 yb_rpc.cc:553] Connection (0x0000504038f9a020) client 10.0.0.2:43671 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 26.693s, timeout: 15.000s, now: 114069.540s, last_read_time_: 114042.846s
W0821 15:32:59.202008 917 yb_rpc.cc:553] Connection (0x0000504038e53360) client 10.0.0.2:38903 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 16.753s, timeout: 15.000s, now: 114062.605s, last_read_time_: 114045.851s
W0821 15:32:59.303628 915 yb_rpc.cc:553] Connection (0x0000504038e528e0) client 10.0.0.2:39691 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 42.431s, timeout: 15.000s, now: 114088.280s, last_read_time_: 114045.848s
I0821 15:32:59.366971 142231 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 15:32:59.974550 142231 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:32:59.627857 917 yb_rpc.cc:553] Connection (0x0000504038e58aa0) client 10.0.0.2:36597 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 16.753s, timeout: 15.000s, now: 114062.605s, last_read_time_: 114045.851s
I0821 15:33:00.805897 142231 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:33:00.855135 142231 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.649s
I0821 15:33:00.863623 142231 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:33:00.877353 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:33:00.877357 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:33:00.882236 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:33:00.882261 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Connection reset by peer (system error 104)
W0821 15:33:00.882274 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:33:00.882298 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:33:00.878324 142231 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:33:00.884829 142231 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:33:00.882370 915 yb_rpc.cc:553] Connection (0x0000504038ffd360) client 10.0.0.2:42801 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 42.431s, timeout: 15.000s, now: 114088.280s, last_read_time_: 114045.848s
W0821 15:33:00.907125 915 yb_rpc.cc:553] Connection (0x0000504038e523a0) client 10.0.0.2:44149 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 45.430s, timeout: 15.000s, now: 114088.280s, last_read_time_: 114042.849s
I0821 15:33:00.959081 142291 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30304. (index mismatch)
I0821 15:33:00.969451 142291 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30304. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 15:33:03.398481 142291 long_operation_tracker.cc:155] UpdateReplica took a long time: 2.460s
W0821 15:33:03.294888 910 periodic.cc:174] PeriodicTimer callback delayed by 0.408s [suppressed 1 similar messages]
W0821 15:33:07.949928 910 periodic.cc:174] PeriodicTimer callback delayed by 2.369s
W0821 15:33:05.547668 142291 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:40951 => 10.0.0.2:7100 (request call id 5044661) took 4643ms (client timeout 3000ms).
W0821 15:33:05.546969 142293 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42585 => 10.0.0.2:7100 (request call id 5044676) took 4572ms (client timeout 3000ms).
W0821 15:33:12.571623 913 periodic.cc:174] PeriodicTimer callback delayed by 2.403s
W0821 15:33:18.981433 142293 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:35265 => 10.0.0.2:7100 (request call id 5044666) took 10489ms (client timeout 3000ms).
W0821 15:33:21.539324 913 thread.cc:850] raft_19794_worker (raft) Time spent creating pthread: real 0.962s user 0.000s sys 0.026s
W0821 15:33:24.894675 913 thread.cc:844] raft_19794_worker (raft) Time spent starting thread: real 3.811s user 0.000s sys 0.103s
I0821 15:33:23.172181 142336 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:33:28.594605 142336 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:33:28.435250 142293 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33053 => 10.0.0.2:7100 (request call id 5044620) took 7425ms (client timeout 3000ms).
W0821 15:33:28.654232 915 inbound_call.cc:111] 0x000050403cb90c00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:37331 => 10.0.0.2:7100 (request call id 5044606): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:37331 => 10.0.0.2:7100 (request call id 5044606) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
I0821 15:33:28.880046 142336 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 12.158s
W0821 15:33:32.028106 913 periodic.cc:174] PeriodicTimer callback delayed by 15.183s
W0821 15:33:33.316984 912 thread.cc:888] CatalogManagerBGTasks_6328_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.021s user 0.009s sys 0.055s
W0821 15:33:34.995187 912 thread.cc:844] CatalogManagerBGTasks_6328_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.765s user 0.009s sys 0.106s
W0821 15:33:36.548760 913 periodic.cc:174] PeriodicTimer callback delayed by 2.258s
W0821 15:33:41.787405 913 periodic.cc:174] PeriodicTimer callback delayed by 2.896s
W0821 15:33:43.546928 915 inbound_call.cc:111] 0x000050403aec1800 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:35265 => 10.0.0.2:7100 (request call id 5044666): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:35265 => 10.0.0.2:7100 (request call id 5044666) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:33:49.095588 913 periodic.cc:174] PeriodicTimer callback delayed by 3.301s
W0821 15:33:59.402887 913 periodic.cc:174] PeriodicTimer callback delayed by 4.968s
W0821 15:34:07.046584 913 periodic.cc:174] PeriodicTimer callback delayed by 2.946s
W0821 15:34:09.428418 910 thread.cc:888] CatalogManagerBGTasks_6329_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.775s user 0.001s sys 0.023s
W0821 15:34:10.920250 910 thread.cc:844] CatalogManagerBGTasks_6329_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.224s user 0.002s sys 0.084s
I0821 15:33:28.922878 142336 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:34:13.077762 915 inbound_call.cc:111] 0x000050403e5f6400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33053 => 10.0.0.2:7100 (request call id 5044620): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:33053 => 10.0.0.2:7100 (request call id 5044620) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:34:13.812806 913 periodic.cc:174] PeriodicTimer callback delayed by 1.874s
I0821 15:34:14.288084 142336 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:34:18.630960 912 periodic.cc:174] PeriodicTimer callback delayed by 1.368s
I0821 15:34:19.082441 142336 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:34:23.911548 910 periodic.cc:174] PeriodicTimer callback delayed by 1.328s
W0821 15:34:27.854504 910 thread.cc:850] raft_19795_worker (raft) Time spent creating pthread: real 0.835s user 0.000s sys 0.015s
W0821 15:34:29.595911 910 thread.cc:844] raft_19795_worker (raft) Time spent starting thread: real 3.308s user 0.000s sys 0.052s
I0821 15:34:29.595803 142355 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:34:32.786643 142355 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:34:34.782426 142355 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.557s
W0821 15:34:35.858556 910 periodic.cc:174] PeriodicTimer callback delayed by 7.422s
W0821 15:34:39.584115 912 periodic.cc:174] PeriodicTimer callback delayed by 1.054s
W0821 15:34:42.149024 910 thread.cc:888] CatalogManagerBGTasks_6330_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.773s user 0.003s sys 0.033s
W0821 15:34:43.379079 911 periodic.cc:174] PeriodicTimer callback delayed by 1.548s
W0821 15:34:44.181082 910 thread.cc:844] CatalogManagerBGTasks_6330_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.875s user 0.003s sys 0.071s
I0821 15:34:36.558921 142355 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:34:44.224089 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403e2ac000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313245, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114088.328s (96.552s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114141.816s (43.064s ago), now: 114184.881s, connection: Connection (0x000050403b2336e0) client 10.0.0.2:39669 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:34:44.725437 142355 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:34:47.035264 915 thread.cc:844] Master-high-pri_581_worker (Master-high-pri) Time spent starting thread: real 0.569s user 0.000s sys 0.009s
W0821 15:34:47.130323 910 periodic.cc:174] PeriodicTimer callback delayed by 1.483s
W0821 15:34:47.008176 142374 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313245) to 10.0.0.3:7100 timed out after 13.926s
I0821 15:34:48.350474 142355 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:34:48.608032 142374 outbound_call.cc:526] RPC callback for RPC call 0x000050403e2ac000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 4 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313245, state: TIMED_OUT, transfer_state: PENDING took 1.543s
W0821 15:34:51.116593 910 periodic.cc:174] PeriodicTimer callback delayed by 1.712s
W0821 15:34:55.306278 910 periodic.cc:174] PeriodicTimer callback delayed by 2.753s
I0821 15:35:00.626434 142381 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:35:00.682909 910 thread.cc:844] raft_19796_worker (raft) Time spent starting thread: real 1.163s user 0.001s sys 0.027s
I0821 15:35:00.732319 142381 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:35:00.758689 142381 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 6.491s
W0821 15:35:06.529347 910 periodic.cc:174] PeriodicTimer callback delayed by 7.217s
W0821 15:35:10.833604 911 periodic.cc:174] PeriodicTimer callback delayed by 2.185s
W0821 15:35:16.849975 913 periodic.cc:174] PeriodicTimer callback delayed by 2.051s
W0821 15:35:17.212821 910 thread.cc:844] CatalogManagerBGTasks_6331_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.449s user 0.001s sys 0.021s
W0821 15:35:23.824659 912 periodic.cc:174] PeriodicTimer callback delayed by 2.602s
W0821 15:35:33.710808 911 periodic.cc:174] PeriodicTimer callback delayed by 4.563s
I0821 15:35:02.182550 142381 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:35:41.684177 912 periodic.cc:174] PeriodicTimer callback delayed by 2.860s
I0821 15:35:42.747332 142381 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:35:49.172000 913 periodic.cc:174] PeriodicTimer callback delayed by 2.790s
I0821 15:35:50.971072 142381 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:35:53.602191 910 thread.cc:844] CatalogManagerBGTasks_6332_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.488s user 0.000s sys 0.023s
W0821 15:35:56.070436 910 periodic.cc:174] PeriodicTimer callback delayed by 1.783s
W0821 15:36:05.339363 912 periodic.cc:174] PeriodicTimer callback delayed by 3.869s
I0821 15:36:08.356518 142432 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:36:11.356421 912 thread.cc:844] raft_19797_worker (raft) Time spent starting thread: real 2.277s user 0.001s sys 0.050s
I0821 15:36:11.347932 142432 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:36:13.795986 142432 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.953s
W0821 15:36:17.306187 912 periodic.cc:174] PeriodicTimer callback delayed by 7.640s
W0821 15:36:21.611519 912 periodic.cc:174] PeriodicTimer callback delayed by 2.039s
W0821 15:36:26.122865 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403bf29500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313252, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114199.794s (74.249s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114235.251s (38.792s ago), now: 114274.043s, connection: Connection (0x0000504038fe9de0) client 10.0.0.2:46595 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:36:26.840269 913 periodic.cc:174] PeriodicTimer callback delayed by 2.955s
W0821 15:36:29.333135 912 thread.cc:844] CatalogManagerBGTasks_6333_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.680s user 0.000s sys 0.045s
W0821 15:36:30.715337 913 periodic.cc:174] PeriodicTimer callback delayed by 1.696s
W0821 15:36:31.707636 142444 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313252) to 10.0.0.4:7100 timed out after 16.557s
W0821 15:36:32.398887 915 thread.cc:888] Master-high-pri_582_worker (Master-high-pri) Time spent waiting for new thread to publish its TID: real 0.712s user 0.000s sys 0.020s
W0821 15:36:34.647200 915 thread.cc:844] Master-high-pri_582_worker (Master-high-pri) Time spent starting thread: real 4.372s user 0.000s sys 0.093s
W0821 15:36:35.234467 910 periodic.cc:174] PeriodicTimer callback delayed by 2.257s
W0821 15:36:34.305683 142444 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf29500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313252, state: TIMED_OUT, transfer_state: PENDING took 2.463s
W0821 15:36:37.188354 913 periodic.cc:174] PeriodicTimer callback delayed by 1.502s
I0821 15:36:15.744398 142432 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:36:40.466351 142432 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:36:43.208380 911 periodic.cc:174] PeriodicTimer callback delayed by 2.066s
I0821 15:36:45.416890 142432 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:36:50.257030 911 periodic.cc:174] PeriodicTimer callback delayed by 2.425s
W0821 15:36:57.269974 910 periodic.cc:174] PeriodicTimer callback delayed by 1.953s
W0821 15:36:57.456308 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403da31500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313254, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114253.757s (61.927s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114273.936s (41.748s ago), now: 114315.685s, connection: Connection (0x000050403b2fa020) client 10.0.0.2:42641 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:36:59.374621 142463 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313254) to 10.0.0.3:7100 timed out after 6.491s
I0821 15:36:59.374176 142460 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:36:59.675583 142460 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:37:01.433007 142460 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.023s
W0821 15:36:59.767273 142463 outbound_call.cc:526] RPC callback for RPC call 0x000050403da31500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313254, state: TIMED_OUT, transfer_state: PENDING took 0.293s
W0821 15:37:04.588951 912 thread.cc:844] CatalogManagerBGTasks_6334_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.793s user 0.000s sys 0.056s
W0821 15:37:04.724393 910 periodic.cc:174] PeriodicTimer callback delayed by 3.789s
W0821 15:37:05.906167 914 thread.cc:844] Master-high-pri_584_worker (Master-high-pri) Time spent starting thread: real 0.673s user 0.000s sys 0.016s
W0821 15:37:05.565461 142472 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/util/net/socket.cc:536): recvmsg got EOF from remote (system error 108)
I0821 15:37:07.538522 142472 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:37:08.273463 912 periodic.cc:174] PeriodicTimer callback delayed by 1.258s
W0821 15:37:12.719612 142472 thread.cc:844] raft_19799_worker (raft) Time spent starting thread: real 1.846s user 0.000s sys 0.033s
W0821 15:37:14.623576 913 periodic.cc:174] PeriodicTimer callback delayed by 4.129s
W0821 15:37:19.196655 910 periodic.cc:174] PeriodicTimer callback delayed by 2.343s
W0821 15:37:21.147993 911 periodic.cc:174] PeriodicTimer callback delayed by 1.944s
W0821 15:37:17.664925 142472 outbound_call.cc:526] RPC callback for RPC call 0x000050403e23c000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 3 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313243, state: FINISHED_ERROR, transfer_state: ABORTED took 12.114s
W0821 15:37:24.986783 911 periodic.cc:174] PeriodicTimer callback delayed by 0.789s
I0821 15:37:03.232728 142460 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:37:28.175091 142460 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:37:28.687402 142476 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.407s
I0821 15:37:31.260314 142476 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:37:32.169442 913 periodic.cc:174] PeriodicTimer callback delayed by 2.103s
W0821 15:37:37.045938 911 thread.cc:844] CatalogManagerBGTasks_6335_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.140s user 0.000s sys 0.019s
I0821 15:37:35.969388 142460 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:37:37.863375 910 periodic.cc:174] PeriodicTimer callback delayed by 3.436s
W0821 15:37:42.905893 910 periodic.cc:174] PeriodicTimer callback delayed by 2.817s
I0821 15:37:44.409538 142460 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:37:46.361641 142460 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:37:49.055145 912 periodic.cc:174] PeriodicTimer callback delayed by 2.166s
I0821 15:37:48.364883 142460 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 12.543s
W0821 15:37:57.379060 912 periodic.cc:174] PeriodicTimer callback delayed by 3.311s
W0821 15:38:01.562114 912 periodic.cc:174] PeriodicTimer callback delayed by 1.719s
W0821 15:38:05.856185 913 periodic.cc:174] PeriodicTimer callback delayed by 2.381s
W0821 15:38:13.445381 911 periodic.cc:174] PeriodicTimer callback delayed by 3.189s
W0821 15:38:15.408941 913 thread.cc:888] CatalogManagerBGTasks_6336_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.502s user 0.001s sys 0.015s
I0821 15:37:50.755051 142460 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:38:17.192548 913 thread.cc:844] CatalogManagerBGTasks_6336_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.766s user 0.003s sys 0.052s
I0821 15:38:18.085049 142460 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:38:19.243048 913 periodic.cc:174] PeriodicTimer callback delayed by 1.484s
I0821 15:38:24.231033 142460 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:38:25.644634 910 periodic.cc:174] PeriodicTimer callback delayed by 1.961s
W0821 15:38:35.828130 911 periodic.cc:174] PeriodicTimer callback delayed by 5.707s
I0821 15:38:39.129729 142516 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:38:39.912204 911 thread.cc:844] raft_19800_worker (raft) Time spent starting thread: real 1.704s user 0.000s sys 0.032s
I0821 15:38:40.488440 142516 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:38:42.238897 142516 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.743s
W0821 15:38:46.082279 911 periodic.cc:174] PeriodicTimer callback delayed by 6.349s
W0821 15:38:52.082856 912 thread.cc:844] CatalogManagerBGTasks_6337_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.547s user 0.000s sys 0.029s
W0821 15:38:55.660266 911 periodic.cc:174] PeriodicTimer callback delayed by 7.223s
I0821 15:38:44.878088 142516 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:39:00.205852 912 periodic.cc:174] PeriodicTimer callback delayed by 2.411s
I0821 15:39:01.217566 142516 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:39:03.173923 910 periodic.cc:174] PeriodicTimer callback delayed by 0.546s
W0821 15:39:06.511219 912 periodic.cc:174] PeriodicTimer callback delayed by 1.897s
I0821 15:39:06.642657 142516 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:39:09.298296 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403bf34380: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313250, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114170.261s (288.036s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114404.676s (53.622s ago), now: 114458.298s, connection: Connection (0x0000504038feb1a0) client 10.0.0.2:41877 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:39:09.301486 916 inbound_call.cc:111] 0x000050403ca3cc00 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42585 => 10.0.0.2:7100 (request call id 5044676): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42585 => 10.0.0.2:7100 (request call id 5044676) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:39:09.309156 142547 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313250) to 10.0.0.4:7100 timed out after 12.158s
W0821 15:39:09.309185 142548 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313246) to 10.0.0.4:7100 timed out after 13.926s
I0821 15:39:09.309266 142548 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:39:09.311103 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf25880: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313247, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114089.889s (368.427s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114100.547s (357.769s ago), now: 114458.316s, connection: Connection (0x0000504038fe9520) client 10.0.0.2:37081 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:39:09.311267 142551 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313247) to 10.0.0.3:7100 timed out after 10.649s
W0821 15:39:09.311439 142551 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x000050403b232c60) client 10.0.0.2:41887 => 10.0.0.4:7100, passed: 368.416s, timeout: 15.000s: kConnectFailed (network error 1)
W0821 15:39:09.314779 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403daeae00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313251, state: SENT, transfer_state: FINISHED, start_time: 114195.521s (262.799s ago), sent_time: 114418.574s (39.746s ago), trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114422.371s (35.949s ago), now: 114458.320s, connection: Connection (0x0000504038fdc560) client 10.0.0.2:46001 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:39:09.314842 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403e23c000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 2 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313259, state: SENT, transfer_state: FINISHED, start_time: 114367.685s (90.634s ago), sent_time: 114425.060s (33.260s ago), trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114429.540s (28.780s ago), now: 114458.320s, connection: Connection (0x0000504038fe98a0) client 10.0.0.2:44425 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:39:09.320621 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.831s
I0821 15:39:09.320868 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:39:09.329782 142551 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313251) to 10.0.0.3:7100 timed out after 16.557s
I0821 15:39:09.329811 142551 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.329861 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.818s
I0821 15:39:09.329872 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:39:09.329897 142553 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313259) to 10.0.0.4:7100 timed out after 16.023s
W0821 15:39:09.330183 914 yb_rpc.cc:553] Connection (0x0000504038fdc560) client 10.0.0.2:46001 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 390.554s, timeout: 15.000s, now: 114432.048s, last_read_time_: 114041.494s
W0821 15:39:09.331212 914 yb_rpc.cc:553] Connection (0x0000504038fe98a0) client 10.0.0.2:44425 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 390.554s, timeout: 15.000s, now: 114432.048s, last_read_time_: 114041.494s
W0821 15:39:09.401038 917 reactor.cc:692] Master_R003: Stuck OutboundCall: RPC call 0x000050403bf6fc00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 5 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313248, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114089.891s (368.515s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114100.547s (357.859s ago), now: 114458.406s, connection: Connection (0x0000504038feac60) client 10.0.0.2:38969 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:39:09.401453 142548 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)
I0821 15:39:09.401463 142548 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:39:09.401680 142570 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)
I0821 15:39:09.401691 142570 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:39:09.401714 142547 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)
I0821 15:39:09.401718 142547 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:39:09.401731 142551 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30305. (index mismatch)
I0821 15:39:09.401733 142551 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30305. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 15:39:09.401952 142574 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313248) to 10.0.0.4:7100 timed out after 10.649s
I0821 15:39:09.401968 142574 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.401998 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.794s
I0821 15:39:09.402005 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:39:09.404961 142574 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886068.414s
I0821 15:39:09.404979 142574 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.407066 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 6.978s
I0821 15:39:09.407086 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:39:09.441309 142551 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886068.377s
I0821 15:39:09.441602 142574 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)
I0821 15:39:09.441617 142574 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30293. Preceding OpId from leader: 30.30307. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:39:09.446664 142551 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221886068.370s
I0821 15:39:09.584043 142574 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:39:09.584074 142574 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.584095 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 3.864s
I0821 15:39:09.584105 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:39:09.584959 142574 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:39:09.584976 142574 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.585386 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.232s
I0821 15:39:09.585397 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:39:09.588323 142574 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.915s
I0821 15:39:09.588465 142574 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.588398 142551 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.915s
I0821 15:39:09.588414 142547 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.915s
I0821 15:39:09.588482 142547 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.588553 142551 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:39:09.588570 142620 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 15.491s
I0821 15:39:09.588581 142620 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:39:09.588586 142549 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.647s
I0821 15:39:09.588593 142549 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:39:09.588719 142621 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 13.391s
I0821 15:39:09.588929 142621 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:39:12.488988 912 periodic.cc:174] PeriodicTimer callback delayed by 1.435s
W0821 15:39:13.837608 913 periodic.cc:174] PeriodicTimer callback delayed by 0.167s
W0821 15:39:16.750185 910 periodic.cc:174] PeriodicTimer callback delayed by 1.191s
I0821 15:39:20.926419 142670 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:39:21.157835 910 thread.cc:844] raft_19804_worker (raft) Time spent starting thread: real 0.700s user 0.001s sys 0.017s
W0821 15:39:22.054382 142551 long_operation_tracker.cc:155] UpdateReplica took a long time: 2.844s
I0821 15:39:22.614152 142670 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:39:23.688705 142670 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.817s
W0821 15:39:23.689208 142551 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:42271 => 10.0.0.2:7100 (request call id 5045610) took 9495ms (client timeout 3000ms).
I0821 15:39:24.385418 142670 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:39:26.752128 142670 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:39:27.029457 142670 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:39:27.051427 915 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:39:27.051369 916 yb_rpc.cc:249] Could not queue an inbound connection heartbeat message: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:39:27.567044 912 periodic.cc:174] PeriodicTimer callback delayed by 5.988s
W0821 15:39:32.499303 910 periodic.cc:174] PeriodicTimer callback delayed by 2.620s
W0821 15:39:35.574584 911 periodic.cc:174] PeriodicTimer callback delayed by 0.866s
W0821 15:39:39.115059 913 periodic.cc:174] PeriodicTimer callback delayed by 1.340s
W0821 15:39:41.360029 910 periodic.cc:174] PeriodicTimer callback delayed by 0.184s
W0821 15:39:41.369695 916 yb_rpc.cc:553] Connection (0x000050403b294e20) client 10.0.0.2:35175 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.296s, timeout: 15.000s, now: 114488.881s, last_read_time_: 114465.584s
I0821 15:39:42.182453 142729 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:39:43.215564 142729 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:39:43.236651 142729 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 16.544s
W0821 15:39:46.760129 910 periodic.cc:174] PeriodicTimer callback delayed by 1.137s
W0821 15:39:51.301962 911 periodic.cc:174] PeriodicTimer callback delayed by 2.297s
W0821 15:39:56.147334 912 periodic.cc:174] PeriodicTimer callback delayed by 2.537s
W0821 15:39:57.540287 911 thread.cc:850] CatalogManagerBGTasks_6339_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.541s user 0.000s sys 0.011s
W0821 15:39:59.450863 911 thread.cc:844] CatalogManagerBGTasks_6339_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.542s user 0.001s sys 0.047s
W0821 15:39:57.538548 916 yb_rpc.cc:553] Connection (0x0000504038fe56e0) client 10.0.0.2:38841 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 23.296s, timeout: 15.000s, now: 114488.881s, last_read_time_: 114465.584s
W0821 15:40:00.315856 912 periodic.cc:174] PeriodicTimer callback delayed by 1.973s
W0821 15:40:02.048290 911 periodic.cc:174] PeriodicTimer callback delayed by 1.628s
W0821 15:40:07.799660 912 periodic.cc:174] PeriodicTimer callback delayed by 1.927s
I0821 15:39:43.851002 142729 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:40:07.815423 914 reactor.cc:692] Master_R000: Stuck OutboundCall: RPC call 0x000050403be27500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313267, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114476.036s (33.198s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114492.866s (16.368s ago), now: 114509.234s, connection: Connection (0x000050403b295de0) client 10.0.0.2:35911 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:40:08.606663 142729 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:40:09.954284 142728 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30350. (index mismatch)
W0821 15:40:11.695673 914 thread.cc:844] Master-high-pri_592_worker (Master-high-pri) Time spent starting thread: real 1.221s user 0.000s sys 0.019s
W0821 15:40:10.889613 911 periodic.cc:174] PeriodicTimer callback delayed by 0.445s
W0821 15:40:10.309842 142777 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313267) to 10.0.0.4:7100 timed out after 16.817s
I0821 15:40:11.753290 142728 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30350. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
W0821 15:40:13.317402 142728 long_operation_tracker.cc:155] UpdateReplica took a long time: 30.022s
I0821 15:40:11.957086 142729 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:40:11.880029 915 yb_rpc.cc:553] Connection (0x000050403b272aa0) client 10.0.0.2:45611 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.954s, timeout: 15.000s, now: 114492.223s, last_read_time_: 114466.268s
W0821 15:40:12.793448 142777 outbound_call.cc:526] RPC callback for RPC call 0x000050403be27500: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 6 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313267, state: TIMED_OUT, transfer_state: PENDING took 2.467s
W0821 15:40:14.516304 914 yb_rpc.cc:553] Connection (0x0000504038fea3a0) client 10.0.0.2:44241 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 34.602s, timeout: 15.000s, now: 114509.185s, last_read_time_: 114474.583s
W0821 15:40:16.297577 913 periodic.cc:174] PeriodicTimer callback delayed by 1.619s
W0821 15:40:15.548476 142728 yb_rpc.cc:388] Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38195 => 10.0.0.2:7100 (request call id 5045633) took 35678ms (client timeout 3000ms).
W0821 15:40:23.258282 911 periodic.cc:174] PeriodicTimer callback delayed by 2.078s
W0821 15:40:28.131503 911 thread.cc:850] raft_19806_worker (raft) Time spent creating pthread: real 0.711s user 0.000s sys 0.015s
W0821 15:40:30.128583 911 thread.cc:844] raft_19806_worker (raft) Time spent starting thread: real 2.811s user 0.001s sys 0.047s
I0821 15:40:29.497627 142797 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:40:32.679632 912 thread.cc:850] CatalogManagerBGTasks_6340_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.567s user 0.000s sys 0.011s
I0821 15:40:32.719422 142797 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:40:34.406733 912 thread.cc:844] CatalogManagerBGTasks_6340_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.205s user 0.000s sys 0.059s
I0821 15:40:34.726584 142797 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.054s
W0821 15:40:37.477218 911 periodic.cc:174] PeriodicTimer callback delayed by 8.493s
W0821 15:40:43.206595 910 periodic.cc:174] PeriodicTimer callback delayed by 3.635s
W0821 15:40:47.563061 911 periodic.cc:174] PeriodicTimer callback delayed by 2.120s
W0821 15:40:51.402230 912 periodic.cc:174] PeriodicTimer callback delayed by 1.608s
W0821 15:40:51.222698 915 yb_rpc.cc:553] Connection (0x000050403b271520) client 10.0.0.2:38883 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 25.954s, timeout: 15.000s, now: 114492.223s, last_read_time_: 114466.268s
W0821 15:40:55.262926 913 periodic.cc:174] PeriodicTimer callback delayed by 1.574s
W0821 15:40:57.771896 910 periodic.cc:174] PeriodicTimer callback delayed by 1.799s
I0821 15:40:36.907476 142797 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:41:03.543372 911 periodic.cc:174] PeriodicTimer callback delayed by 1.722s
I0821 15:41:06.177608 142797 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:41:08.348942 913 thread.cc:844] CatalogManagerBGTasks_6341_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.160s user 0.001s sys 0.048s
W0821 15:41:10.141942 910 periodic.cc:174] PeriodicTimer callback delayed by 1.644s
I0821 15:41:14.071200 142797 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:41:17.823561 911 periodic.cc:174] PeriodicTimer callback delayed by 2.252s
W0821 15:41:23.826954 913 periodic.cc:174] PeriodicTimer callback delayed by 1.891s
I0821 15:41:27.928714 142826 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:41:28.623240 913 thread.cc:844] raft_19807_worker (raft) Time spent starting thread: real 2.079s user 0.000s sys 0.059s
I0821 15:41:29.789832 142826 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
W0821 15:41:27.842080 916 yb_rpc.cc:553] Connection (0x000050403b2541e0) client 10.0.0.2:43259 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 90.249s, timeout: 15.000s, now: 114566.299s, last_read_time_: 114476.050s
I0821 15:41:31.423820 142826 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 6.659s
W0821 15:41:35.734378 912 periodic.cc:174] PeriodicTimer callback delayed by 6.689s
W0821 15:41:35.695029 915 inbound_call.cc:111] 0x000050403875d400 -> Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38195 => 10.0.0.2:7100 (request call id 5045633): Connection torn down before Call yb.consensus.ConsensusService.UpdateConsensus 10.0.0.4:38195 => 10.0.0.2:7100 (request call id 5045633) could send its response: Network error (yb/util/net/socket.cc:449): sendmsg error: Broken pipe (system error 32)
W0821 15:41:40.041685 910 periodic.cc:174] PeriodicTimer callback delayed by 2.173s
W0821 15:41:40.545661 913 thread.cc:850] CatalogManagerBGTasks_6342_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.726s user 0.000s sys 0.022s
W0821 15:41:42.967569 911 periodic.cc:174] PeriodicTimer callback delayed by 2.362s
W0821 15:41:43.163686 913 thread.cc:844] CatalogManagerBGTasks_6342_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.599s user 0.000s sys 0.116s
W0821 15:41:50.040494 911 periodic.cc:174] PeriodicTimer callback delayed by 2.558s
W0821 15:41:58.365127 910 periodic.cc:174] PeriodicTimer callback delayed by 2.651s
W0821 15:42:05.901496 913 periodic.cc:174] PeriodicTimer callback delayed by 2.099s
W0821 15:42:13.763993 913 periodic.cc:174] PeriodicTimer callback delayed by 3.124s
W0821 15:42:14.970624 912 thread.cc:850] CatalogManagerBGTasks_6343_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.756s user 0.001s sys 0.014s
W0821 15:42:16.812832 912 thread.cc:844] CatalogManagerBGTasks_6343_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.771s user 0.003s sys 0.068s
W0821 15:42:19.429286 912 periodic.cc:174] PeriodicTimer callback delayed by 1.236s
I0821 15:41:33.934499 142826 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:42:24.964380 910 periodic.cc:174] PeriodicTimer callback delayed by 1.867s
I0821 15:42:22.850857 142826 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:42:31.363168 911 periodic.cc:174] PeriodicTimer callback delayed by 1.868s
I0821 15:42:32.437680 142826 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:42:38.776700 910 periodic.cc:174] PeriodicTimer callback delayed by 2.918s
I0821 15:42:40.266767 142826 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:42:42.727372 916 reactor.cc:692] Master_R002: Stuck OutboundCall: RPC call 0x000050403bf27c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313271, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114585.433s (76.970s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114656.010s (6.393s ago), now: 114662.403s, connection: Connection (0x000050403b2943a0) client 10.0.0.2:46321 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:42:41.902724 142826 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:42:42.765125 142826 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.293s
W0821 15:42:42.795281 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403e23c000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313269, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114524.169s (144.689s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114618.229s (50.630s ago), now: 114668.859s, connection: Connection (0x000050403b2958a0) client 10.0.0.2:41331 => 10.0.0.4:7100, active_call_state: kAdded (forcing a timeout)
W0821 15:42:42.931114 142839 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313271) to 10.0.0.4:7100 timed out after 19.054s
W0821 15:42:45.559780 912 periodic.cc:174] PeriodicTimer callback delayed by 1.817s
W0821 15:42:45.480820 142843 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313269) to 10.0.0.4:7100 timed out after 16.544s
W0821 15:42:47.077361 915 thread.cc:844] Master-high-pri_594_worker (Master-high-pri) Time spent starting thread: real 3.084s user 0.000s sys 0.083s
W0821 15:42:43.929365 142839 outbound_call.cc:526] RPC callback for RPC call 0x000050403bf27c00: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313271, state: TIMED_OUT, transfer_state: PENDING took 0.939s
W0821 15:42:50.086833 912 periodic.cc:174] PeriodicTimer callback delayed by 2.177s
W0821 15:42:48.649838 142843 outbound_call.cc:526] RPC callback for RPC call 0x000050403e23c000: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.4:7100 idx: 7 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313269, state: TIMED_OUT, transfer_state: PENDING took 3.149s
W0821 15:42:52.012005 910 thread.cc:888] CatalogManagerBGTasks_6344_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.887s user 0.010s sys 0.050s
W0821 15:42:53.315727 912 periodic.cc:174] PeriodicTimer callback delayed by 2.185s
W0821 15:42:54.208416 910 thread.cc:844] CatalogManagerBGTasks_6344_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.375s user 0.010s sys 0.110s
W0821 15:42:55.722491 915 reactor.cc:692] Master_R001: Stuck OutboundCall: RPC call 0x000050403d905180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313270, state: ON_OUTBOUND_QUEUE, transfer_state: PENDING, start_time: 114577.224s (101.058s ago), sent_time: -inf, trigger_callback_time: -inf, invoke_callback_time: -inf, expiration_time: 114623.347s (54.935s ago), now: 114678.282s, connection: Connection (0x000050403b2718a0) client 10.0.0.2:41129 => 10.0.0.3:7100, active_call_state: kAdded (forcing a timeout)
I0821 15:42:42.803668 142826 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:42:55.744158 142843 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313270) to 10.0.0.3:7100 timed out after 19.054s
I0821 15:42:55.895545 142843 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:42:55.803690 142826 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:42:57.066646 913 periodic.cc:174] PeriodicTimer callback delayed by 0.794s
W0821 15:42:59.246909 142843 thread.cc:888] raft_19808_worker (raft) Time spent waiting for new thread to publish its TID: real 1.380s user 0.009s sys 0.058s
I0821 15:42:59.557976 142852 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.046s
W0821 15:43:00.365538 142843 thread.cc:844] raft_19808_worker (raft) Time spent starting thread: real 2.762s user 0.009s sys 0.084s
I0821 15:43:00.033030 142826 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
I0821 15:43:00.605983 142852 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:43:02.432343 911 periodic.cc:174] PeriodicTimer callback delayed by 1.267s
W0821 15:43:01.942034 142843 outbound_call.cc:526] RPC callback for RPC call 0x000050403d905180: yb.consensus.ConsensusService.RequestConsensusVote -> { remote: 10.0.0.3:7100 idx: 0 protocol: 0x00006051e9dd2af0 -> tcpc }, id: 313270, state: TIMED_OUT, transfer_state: PENDING took 6.102s
W0821 15:43:06.649156 912 periodic.cc:174] PeriodicTimer callback delayed by 1.846s
W0821 15:43:10.066668 910 periodic.cc:174] PeriodicTimer callback delayed by 1.454s
W0821 15:43:11.091295 910 thread.cc:844] raft_19809_worker (raft) Time spent starting thread: real 0.657s user 0.000s sys 0.017s
I0821 15:43:11.048688 142888 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:43:12.294360 142888 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:43:14.162130 142888 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.325s
W0821 15:43:15.365814 911 periodic.cc:174] PeriodicTimer callback delayed by 2.238s
I0821 15:43:17.094385 142888 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:43:20.423476 913 periodic.cc:174] PeriodicTimer callback delayed by 1.169s
I0821 15:43:21.629601 142888 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:43:24.728215 912 periodic.cc:174] PeriodicTimer callback delayed by 2.028s
W0821 15:43:25.660934 911 thread.cc:844] CatalogManagerBGTasks_6345_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.067s user 0.000s sys 0.061s
I0821 15:43:28.838039 142888 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:43:31.394505 910 periodic.cc:174] PeriodicTimer callback delayed by 4.398s
W0821 15:43:36.675522 912 periodic.cc:174] PeriodicTimer callback delayed by 1.593s
I0821 15:43:40.972298 142911 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:43:42.265154 912 thread.cc:844] raft_19810_worker (raft) Time spent starting thread: real 2.977s user 0.000s sys 0.058s
I0821 15:43:43.144806 142911 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:43:44.890076 142911 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.676s
W0821 15:43:48.205235 911 periodic.cc:174] PeriodicTimer callback delayed by 7.447s
I0821 15:43:48.089874 142911 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:43:50.074707 142911 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:43:51.909449 912 periodic.cc:174] PeriodicTimer callback delayed by 1.445s
I0821 15:43:53.070899 142911 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:43:55.597332 913 periodic.cc:174] PeriodicTimer callback delayed by 1.500s
W0821 15:43:58.545434 910 thread.cc:888] CatalogManagerBGTasks_6346_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.652s user 0.005s sys 0.031s
W0821 15:43:59.377645 912 periodic.cc:174] PeriodicTimer callback delayed by 1.611s
W0821 15:44:00.096285 910 thread.cc:844] CatalogManagerBGTasks_6346_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.226s user 0.005s sys 0.077s
I0821 15:44:01.726159 142920 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:44:02.898790 912 thread.cc:844] raft_19811_worker (raft) Time spent starting thread: real 1.105s user 0.000s sys 0.031s
I0821 15:44:03.818437 142920 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:44:06.148586 142920 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.810s
W0821 15:44:08.415796 913 periodic.cc:174] PeriodicTimer callback delayed by 5.511s
W0821 15:44:15.137740 913 periodic.cc:174] PeriodicTimer callback delayed by 2.382s
W0821 15:44:19.137090 913 periodic.cc:174] PeriodicTimer callback delayed by 0.424s
W0821 15:44:22.425232 912 periodic.cc:174] PeriodicTimer callback delayed by 0.587s
W0821 15:44:28.892434 910 periodic.cc:174] PeriodicTimer callback delayed by 2.180s
I0821 15:44:10.138466 142920 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:44:33.342067 913 thread.cc:888] CatalogManagerBGTasks_6347_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.837s user 0.002s sys 0.043s
I0821 15:44:33.342084 142920 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:44:33.764675 911 periodic.cc:174] PeriodicTimer callback delayed by 1.552s
W0821 15:44:35.149377 913 thread.cc:844] CatalogManagerBGTasks_6347_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.878s user 0.003s sys 0.074s
I0821 15:44:37.607942 142920 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:44:40.480374 911 periodic.cc:174] PeriodicTimer callback delayed by 1.662s
W0821 15:44:47.179603 911 periodic.cc:174] PeriodicTimer callback delayed by 2.197s
W0821 15:44:52.456214 911 thread.cc:850] raft_19812_worker (raft) Time spent creating pthread: real 0.522s user 0.001s sys 0.012s
I0821 15:44:54.003582 142943 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:44:54.892683 911 thread.cc:844] raft_19812_worker (raft) Time spent starting thread: real 2.848s user 0.001s sys 0.071s
I0821 15:44:55.972611 142943 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:44:58.055150 142943 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.766s
W0821 15:45:01.296916 913 periodic.cc:174] PeriodicTimer callback delayed by 9.221s
W0821 15:45:05.763154 912 periodic.cc:174] PeriodicTimer callback delayed by 2.262s
I0821 15:44:59.613253 142943 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:45:06.795279 913 thread.cc:844] CatalogManagerBGTasks_6348_worker (CatalogManagerBGTasks) Time spent starting thread: real 0.955s user 0.000s sys 0.014s
I0821 15:45:08.381916 142943 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:45:09.492631 910 periodic.cc:174] PeriodicTimer callback delayed by 1.474s
W0821 15:45:14.969617 911 periodic.cc:174] PeriodicTimer callback delayed by 3.225s
I0821 15:45:15.341842 142943 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:45:19.778497 910 periodic.cc:174] PeriodicTimer callback delayed by 2.579s
W0821 15:45:25.968633 913 periodic.cc:174] PeriodicTimer callback delayed by 2.506s
W0821 15:45:29.360904 913 thread.cc:850] raft_19813_worker (raft) Time spent creating pthread: real 0.809s user 0.000s sys 0.018s
I0821 15:45:31.166277 142965 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:45:32.057716 913 thread.cc:844] raft_19813_worker (raft) Time spent starting thread: real 3.241s user 0.002s sys 0.059s
I0821 15:45:33.211164 142965 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:45:35.219231 142965 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 7.709s
W0821 15:45:38.806150 913 periodic.cc:174] PeriodicTimer callback delayed by 7.947s
W0821 15:45:40.681213 910 thread.cc:888] CatalogManagerBGTasks_6349_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.578s user 0.000s sys 0.022s
W0821 15:45:42.017259 910 thread.cc:844] CatalogManagerBGTasks_6349_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.360s user 0.000s sys 0.053s
W0821 15:45:43.048520 911 periodic.cc:174] PeriodicTimer callback delayed by 1.875s
W0821 15:45:47.380649 911 periodic.cc:174] PeriodicTimer callback delayed by 2.120s
W0821 15:45:54.650820 913 periodic.cc:174] PeriodicTimer callback delayed by 3.389s
I0821 15:45:37.459592 142965 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:46:00.146207 142965 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:46:02.067513 911 periodic.cc:174] PeriodicTimer callback delayed by 2.396s
I0821 15:46:08.268702 142965 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:46:11.125234 911 periodic.cc:174] PeriodicTimer callback delayed by 5.347s
W0821 15:46:16.140412 912 thread.cc:888] CatalogManagerBGTasks_6350_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 1.030s user 0.004s sys 0.039s
W0821 15:46:18.582108 912 thread.cc:844] CatalogManagerBGTasks_6350_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.494s user 0.005s sys 0.076s
W0821 15:46:21.936573 910 periodic.cc:174] PeriodicTimer callback delayed by 5.429s
W0821 15:46:26.483620 910 thread.cc:850] raft_19814_worker (raft) Time spent creating pthread: real 0.741s user 0.000s sys 0.019s
I0821 15:46:28.582103 143002 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:46:30.639212 910 thread.cc:844] raft_19814_worker (raft) Time spent starting thread: real 4.118s user 0.000s sys 0.073s
I0821 15:46:30.908975 143002 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:46:32.708447 143002 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.921s
W0821 15:46:37.241075 911 periodic.cc:174] PeriodicTimer callback delayed by 10.917s
W0821 15:46:41.554360 911 periodic.cc:174] PeriodicTimer callback delayed by 2.009s
W0821 15:46:50.709872 910 periodic.cc:174] PeriodicTimer callback delayed by 3.724s
W0821 15:46:54.089056 913 thread.cc:844] CatalogManagerBGTasks_6351_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.135s user 0.000s sys 0.022s
W0821 15:46:57.414449 912 periodic.cc:174] PeriodicTimer callback delayed by 2.371s
I0821 15:46:34.673416 143002 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:47:03.829450 143002 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:47:04.575517 910 periodic.cc:174] PeriodicTimer callback delayed by 2.391s
W0821 15:47:13.573808 912 periodic.cc:174] PeriodicTimer callback delayed by 3.985s
I0821 15:47:13.206372 143002 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:47:19.751695 913 periodic.cc:174] PeriodicTimer callback delayed by 2.382s
W0821 15:47:27.413070 913 periodic.cc:174] PeriodicTimer callback delayed by 2.752s
W0821 15:47:28.199569 911 thread.cc:850] CatalogManagerBGTasks_6352_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.632s user 0.000s sys 0.013s
W0821 15:47:31.024415 911 thread.cc:844] CatalogManagerBGTasks_6352_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.885s user 0.000s sys 0.084s
I0821 15:47:31.480014 143028 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:47:34.331816 913 thread.cc:844] raft_19815_worker (raft) Time spent starting thread: real 3.631s user 0.001s sys 0.074s
I0821 15:47:34.710844 143028 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:47:38.387924 143028 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 19.289s
W0821 15:47:41.019676 911 periodic.cc:174] PeriodicTimer callback delayed by 9.608s
W0821 15:47:46.107380 911 periodic.cc:174] PeriodicTimer callback delayed by 2.822s
W0821 15:47:51.243956 913 periodic.cc:174] PeriodicTimer callback delayed by 2.929s
W0821 15:47:56.527961 911 periodic.cc:174] PeriodicTimer callback delayed by 2.968s
W0821 15:48:05.051321 912 periodic.cc:174] PeriodicTimer callback delayed by 6.265s
W0821 15:48:08.778138 910 thread.cc:844] CatalogManagerBGTasks_6353_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.250s user 0.000s sys 0.049s
W0821 15:48:12.818182 911 periodic.cc:174] PeriodicTimer callback delayed by 3.497s
W0821 15:48:20.608247 913 periodic.cc:174] PeriodicTimer callback delayed by 2.270s
W0821 15:48:24.156832 911 periodic.cc:174] PeriodicTimer callback delayed by 0.756s
W0821 15:48:33.257115 912 periodic.cc:174] PeriodicTimer callback delayed by 3.968s
W0821 15:48:40.463416 912 periodic.cc:174] PeriodicTimer callback delayed by 3.385s
W0821 15:48:42.195570 913 thread.cc:850] CatalogManagerBGTasks_6354_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.507s user 0.000s sys 0.009s
W0821 15:48:45.272301 913 thread.cc:844] CatalogManagerBGTasks_6354_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.620s user 0.001s sys 0.071s
W0821 15:48:48.717949 912 periodic.cc:174] PeriodicTimer callback delayed by 3.376s
W0821 15:48:58.587245 910 periodic.cc:174] PeriodicTimer callback delayed by 4.791s
W0821 15:49:05.933804 912 periodic.cc:174] PeriodicTimer callback delayed by 2.201s
I0821 15:47:40.534945 143028 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:49:13.685945 143028 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:49:15.782186 913 periodic.cc:174] PeriodicTimer callback delayed by 5.576s
W0821 15:49:23.354223 913 periodic.cc:174] PeriodicTimer callback delayed by 2.492s
W0821 15:49:23.033277 912 thread.cc:844] CatalogManagerBGTasks_6355_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.090s user 0.001s sys 0.075s
I0821 15:49:23.801671 143028 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:49:30.209983 911 periodic.cc:174] PeriodicTimer callback delayed by 2.359s
W0821 15:49:40.098937 912 periodic.cc:174] PeriodicTimer callback delayed by 4.671s
W0821 15:49:45.384022 912 thread.cc:844] raft_19816_worker (raft) Time spent starting thread: real 1.878s user 0.000s sys 0.050s
I0821 15:49:45.337268 143062 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:49:49.166872 143062 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:49:52.164237 143062 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 12.286s
W0821 15:49:53.204136 910 periodic.cc:174] PeriodicTimer callback delayed by 8.925s
W0821 15:50:00.745864 910 thread.cc:888] CatalogManagerBGTasks_6356_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.968s user 0.001s sys 0.022s
W0821 15:50:01.632829 913 periodic.cc:174] PeriodicTimer callback delayed by 3.113s
W0821 15:50:02.795475 910 thread.cc:844] CatalogManagerBGTasks_6356_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.253s user 0.001s sys 0.082s
W0821 15:50:10.139261 910 periodic.cc:174] PeriodicTimer callback delayed by 6.224s
W0821 15:50:17.464982 911 periodic.cc:174] PeriodicTimer callback delayed by 2.704s
W0821 15:50:25.699613 911 periodic.cc:174] PeriodicTimer callback delayed by 2.669s
I0821 15:49:55.546933 143062 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:50:32.721072 911 periodic.cc:174] PeriodicTimer callback delayed by 2.780s
I0821 15:50:34.962936 143062 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:50:36.126336 910 thread.cc:844] CatalogManagerBGTasks_6357_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.587s user 0.000s sys 0.031s
W0821 15:50:39.192483 910 periodic.cc:174] PeriodicTimer callback delayed by 2.040s
I0821 15:50:45.085567 143062 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:50:46.653190 911 periodic.cc:174] PeriodicTimer callback delayed by 2.422s
W0821 15:50:54.606901 913 periodic.cc:174] PeriodicTimer callback delayed by 3.234s
I0821 15:50:58.868546 143095 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:51:00.754573 913 thread.cc:844] raft_19817_worker (raft) Time spent starting thread: real 2.495s user 0.000s sys 0.063s
I0821 15:51:01.213469 143095 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:51:03.511394 143095 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.661s
W0821 15:51:09.953825 912 periodic.cc:174] PeriodicTimer callback delayed by 10.961s
W0821 15:51:12.655495 910 thread.cc:888] CatalogManagerBGTasks_6358_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.734s user 0.002s sys 0.018s
W0821 15:51:15.154503 911 periodic.cc:174] PeriodicTimer callback delayed by 2.941s
W0821 15:51:15.283303 910 thread.cc:844] CatalogManagerBGTasks_6358_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.951s user 0.003s sys 0.082s
W0821 15:51:21.452020 910 periodic.cc:174] PeriodicTimer callback delayed by 2.305s
I0821 15:51:06.475384 143095 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:51:28.610994 910 periodic.cc:174] PeriodicTimer callback delayed by 2.387s
I0821 15:51:31.877493 143095 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:51:37.075155 912 periodic.cc:174] PeriodicTimer callback delayed by 3.581s
I0821 15:51:41.375176 143095 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:51:44.931569 911 periodic.cc:174] PeriodicTimer callback delayed by 2.564s
W0821 15:51:47.309653 913 thread.cc:844] CatalogManagerBGTasks_6359_worker (CatalogManagerBGTasks) Time spent starting thread: real 1.649s user 0.000s sys 0.042s
W0821 15:51:52.310119 912 periodic.cc:174] PeriodicTimer callback delayed by 2.370s
I0821 15:51:56.586016 143124 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:51:58.061645 912 thread.cc:844] raft_19818_worker (raft) Time spent starting thread: real 2.135s user 0.000s sys 0.055s
I0821 15:51:59.853749 143124 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:52:02.551483 143124 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.829s
W0821 15:52:08.289666 910 periodic.cc:174] PeriodicTimer callback delayed by 10.152s
W0821 15:52:12.772925 912 periodic.cc:174] PeriodicTimer callback delayed by 2.398s
W0821 15:52:18.836301 911 periodic.cc:174] PeriodicTimer callback delayed by 3.666s
W0821 15:52:20.528666 913 thread.cc:850] CatalogManagerBGTasks_6360_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.642s user 0.000s sys 0.017s
W0821 15:52:22.770110 913 thread.cc:844] CatalogManagerBGTasks_6360_worker (CatalogManagerBGTasks) Time spent starting thread: real 2.874s user 0.000s sys 0.080s
W0821 15:52:25.211848 913 periodic.cc:174] PeriodicTimer callback delayed by 2.297s
I0821 15:52:05.604265 143124 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:52:29.319753 143124 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:52:31.889101 912 periodic.cc:174] PeriodicTimer callback delayed by 1.865s
I0821 15:52:34.959468 143124 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:52:38.250190 911 periodic.cc:174] PeriodicTimer callback delayed by 1.750s
W0821 15:52:45.776921 913 periodic.cc:174] PeriodicTimer callback delayed by 2.634s
I0821 15:52:51.015359 143157 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
W0821 15:52:51.024786 913 thread.cc:888] raft_19819_worker (raft) Time spent waiting for new thread to publish its TID: real 0.507s user 0.000s sys 0.014s
W0821 15:52:53.181460 913 thread.cc:844] raft_19819_worker (raft) Time spent starting thread: real 3.838s user 0.002s sys 0.094s
I0821 15:52:53.115916 143157 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:52:55.064682 143157 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.987s
W0821 15:52:55.785362 911 thread.cc:850] CatalogManagerBGTasks_6361_worker (CatalogManagerBGTasks) Time spent creating pthread: real 0.560s user 0.000s sys 0.017s
W0821 15:52:58.796839 911 thread.cc:844] CatalogManagerBGTasks_6361_worker (CatalogManagerBGTasks) Time spent starting thread: real 3.295s user 0.000s sys 0.076s
W0821 15:53:00.818440 913 periodic.cc:174] PeriodicTimer callback delayed by 9.758s
W0821 15:53:03.829852 912 periodic.cc:174] PeriodicTimer callback delayed by 2.020s
W0821 15:53:10.969944 912 periodic.cc:174] PeriodicTimer callback delayed by 2.173s
I0821 15:52:56.846185 143157 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
W0821 15:53:18.291929 910 periodic.cc:174] PeriodicTimer callback delayed by 2.600s
I0821 15:53:19.734653 143157 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
W0821 15:53:25.732662 912 periodic.cc:174] PeriodicTimer callback delayed by 2.691s
I0821 15:53:26.711879 143157 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:53:32.593636 910 thread.cc:888] CatalogManagerBGTasks_6362_worker (CatalogManagerBGTasks) Time spent waiting for new thread to publish its TID: real 0.872s user 0.000s sys 0.016s
W0821 15:53:34.300060 910 thread.cc:844] CatalogManagerBGTasks_6362_worker (CatalogManagerBGTasks) Time spent starting thread: real 4.034s user 0.000s sys 0.088s
W0821 15:53:34.287485 912 periodic.cc:174] PeriodicTimer callback delayed by 2.781s
I0821 15:53:34.300776 143177 raft_consensus.cc:1070] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: ReportFailDetected: Starting NORMAL_ELECTION...
I0821 15:53:34.303153 143177 raft_consensus.cc:656] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Fail or stepdown of leader 64fedbe39ec148ab8f9e982b6d19f6d2 detected. Triggering leader pre-election, mode=NORMAL_ELECTION
I0821 15:53:34.303181 143177 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 14.191s
I0821 15:53:34.303188 143177 raft_consensus.cc:704] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Starting pre-election with config: opid_index: -1 peers { permanent_uuid: "55d29a1415af4e44ac228b2c8a82f866" member_type: VOTER last_known_private_addr { host: "10.0.0.2" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care1" } } peers { permanent_uuid: "a5110a40aa5b46eaa183f0022d8d9063" member_type: VOTER last_known_private_addr { host: "10.0.0.3" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care2" } } peers { permanent_uuid: "64fedbe39ec148ab8f9e982b6d19f6d2" member_type: VOTER last_known_private_addr { host: "10.0.0.4" port: 7100 } cloud_info { placement_cloud: "hetzner" placement_region: "fsn" placement_zone: "care3" } }
I0821 15:53:34.303313 143177 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer a5110a40aa5b46eaa183f0022d8d9063
I0821 15:53:34.303344 143177 leader_election.cc:214] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Requesting vote from peer 64fedbe39ec148ab8f9e982b6d19f6d2
W0821 15:53:34.321091 143193 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313277) to 10.0.0.4:7100 timed out after 11.325s
W0821 15:53:34.322690 143197 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313266) to 10.0.0.3:7100 timed out after 16.817s
I0821 15:53:34.322824 143197 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.330662 143177 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.450s
I0821 15:53:34.330689 143177 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:53:34.337620 143197 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x000050403b392020) client 10.0.0.2:36493 => 10.0.0.4:7100, passed: 599.002s, timeout: 15.000s: kConnectFailed (network error 1)
W0821 15:53:34.346299 917 yb_rpc.cc:553] Connection (0x000050403b2551a0) client 10.0.0.2:34097 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 847.272s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114476.058s
W0821 15:53:34.346657 143197 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 847.272s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114476.058s
W0821 15:53:34.347375 917 yb_rpc.cc:553] Connection (0x000050403b254720) client 10.0.0.2:33873 => 10.0.0.2:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.348023 917 yb_rpc.cc:553] Connection (0x000050403b237c20) client 10.0.0.2:40105 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.348251 143197 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.348325 143197 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.346974 143209 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 847.272s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114476.058s
W0821 15:53:34.348574 917 yb_rpc.cc:553] Connection (0x000050403b296aa0) client 10.0.0.2:35915 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.348783 143209 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.348840 143209 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 851.832s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114471.498s
W0821 15:53:34.348906 917 yb_rpc.cc:553] Connection (0x000050403b254c60) client 10.0.0.2:37441 => 10.0.0.4:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 847.272s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114476.058s
W0821 15:53:34.348973 143209 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 847.272s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114476.058s
W0821 15:53:34.348995 143209 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer 64fedbe39ec148ab8f9e982b6d19f6d2: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 847.272s, timeout: 15.000s, now: 115323.330s, last_read_time_: 114476.058s
W0821 15:53:34.351503 143193 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313272) to 10.0.0.3:7100 timed out after 6.659s
I0821 15:53:34.351577 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
W0821 15:53:34.355590 143193 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Timed out (yb/rpc/outbound_call.cc:685): RequestConsensusVote RPC (request call id 313268) to 10.0.0.3:7100 timed out after 16.544s
I0821 15:53:34.355638 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.355726 143177 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 15.755s
I0821 15:53:34.355737 143177 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:34.358681 143213 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 11.859s
I0821 15:53:34.358697 143213 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:53:34.394479 143193 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x000050403b296020) client 10.0.0.2:43631 => 10.0.0.3:7100, passed: 745.635s, timeout: 15.000s: kConnectFailed (network error 1)
W0821 15:53:34.400664 143193 leader_election.cc:275] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: RPC error from VoteRequest() call to peer a5110a40aa5b46eaa183f0022d8d9063: Network error (yb/rpc/connection.cc:268): Connect timeout Connection (0x000050403b2541e0) client 10.0.0.2:44705 => 10.0.0.3:7100, passed: 611.577s, timeout: 15.000s: kConnectFailed (network error 1)
I0821 15:53:34.400856 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.400899 143213 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.638s
I0821 15:53:34.400938 143213 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:34.461584 143197 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.358s
I0821 15:53:34.462121 143209 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.358s
I0821 15:53:34.465561 143209 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.356s
I0821 15:53:34.466130 143193 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)
I0821 15:53:34.466143 143193 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:53:34.469832 143193 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)
I0821 15:53:34.469852 143193 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:53:34.470072 143193 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)
I0821 15:53:34.470080 143193 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:53:34.470644 143193 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.346s
I0821 15:53:34.470660 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.470701 143193 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.346s
I0821 15:53:34.470713 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.470705 143213 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.354s
I0821 15:53:34.470777 143177 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 5.191s
I0821 15:53:34.470784 143177 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:34.470795 143213 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:34.470796 143193 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.346s
I0821 15:53:34.470806 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.470824 143213 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 4.576s
I0821 15:53:34.470830 143213 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:34.470836 143193 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:53:34.470842 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.470857 143213 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 7.107s
I0821 15:53:34.470865 143213 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:34.478681 143193 raft_consensus.cc:1780] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Refusing update from remote peer 64fedbe39ec148ab8f9e982b6d19f6d2: Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)
I0821 15:53:34.478713 143193 raft_consensus.cc:1996] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Returning from UpdateConsensus because of error: error { code: PRECEDING_ENTRY_DIDNT_MATCH status { code: ILLEGAL_STATE message: "Log matching property violated. Preceding OpId in replica: 30.30346. Preceding OpId from leader: 30.30357. (index mismatch)" source_file: "../../src/yb/consensus/raft_consensus.cc" source_line: 1778 errors: "00" } }
I0821 15:53:34.485570 143209 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer 64fedbe39ec148ab8f9e982b6d19f6d2. Message: Invalid argument (yb/consensus/raft_consensus.cc:3147): T 00000000000000000000000000000000 P 64fedbe39ec148ab8f9e982b6d19f6d2 [term 30 LEADER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 9221885203.332s
I0821 15:53:34.485611 143209 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:34.489711 143213 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 7.129s
I0821 15:53:34.489750 143213 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
W0821 15:53:35.973320 142932 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313301) to 10.0.0.3:7100 timed out after 1.500s
I0821 15:53:37.809491 143209 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:53:37.809562 143209 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:37.809670 143197 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:53:37.809684 143197 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:37.809743 143467 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3089): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica has last-logged OpId of term: 30 index: 30358, which is greater than that of the candidate, which has last-logged OpId of term: 30 index: 30346.
I0821 15:53:37.809748 143468 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 18.301s
I0821 15:53:37.809796 143468 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:37.809819 143467 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:37.809839 143468 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 9.782s
I0821 15:53:37.809847 143468 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:37.809861 143466 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 6.436s
I0821 15:53:37.809871 143466 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:37.812561 143193 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:53:37.812606 143193 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:37.812640 143466 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.199s
I0821 15:53:37.812661 143466 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:39.038488 143467 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.851s
I0821 15:53:39.585449 143467 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Invalid argument (yb/consensus/raft_consensus.cc:3089): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica has last-logged OpId of term: 30 index: 30358, which is greater than that of the candidate, which has last-logged OpId of term: 30 index: 30346.
I0821 15:53:39.585517 143467 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:39.585718 143495 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.745s
I0821 15:53:39.585814 143495 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:39.689839 143467 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:53:39.689903 143467 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:39.689997 143495 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 10.094s
I0821 15:53:39.690045 143495 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:53:40.102703 143467 leader_election.cc:387] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Vote denied by peer a5110a40aa5b46eaa183f0022d8d9063. Message: Service unavailable (yb/consensus/raft_consensus.cc:3161): T 00000000000000000000000000000000 P a5110a40aa5b46eaa183f0022d8d9063 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 55d29a1415af4e44ac228b2c8a82f866 for term 31 because replica is already servicing an update from a current leader or another vote.
I0821 15:53:40.102756 143467 leader_election.cc:240] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [CANDIDATE]: Term 31 pre-election: Election decided. Result: candidate lost.
I0821 15:53:40.102955 143495 raft_consensus.cc:3634] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Snoozing leader timeout detection for 8.548s
I0821 15:53:40.102986 143495 raft_consensus.cc:3480] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader Pre-election lost for term 31. Reason: None given. Originator:
I0821 15:56:16.660137 143467 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.874s
I0821 15:56:16.663756 143467 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.870s
I0821 15:57:14.570151 143467 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.991s
I0821 15:57:14.664958 143467 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.997s
I0821 15:59:31.104369 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.696s
I0821 15:59:50.934655 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.963s
I0821 16:00:06.433911 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.641s
I0821 16:00:10.282382 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.554s
I0821 16:00:10.771950 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.612s
I0821 16:00:34.435094 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.619s
I0821 16:01:07.818943 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.567s
I0821 16:02:30.749512 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.535s
I0821 16:02:40.159488 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.730s
I0821 16:05:14.933525 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.509s
I0821 16:07:33.561846 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.652s
I0821 16:07:48.796044 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.682s
I0821 16:08:27.025864 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.852s
I0821 16:09:45.903945 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.708s
I0821 16:09:49.966189 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.814s
I0821 16:10:27.071218 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.712s
I0821 16:11:17.430188 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.656s
I0821 16:13:29.797727 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.944s
I0821 16:13:41.823169 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.557s
I0821 16:13:49.627432 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.795s
I0821 16:14:51.262301 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.751s
I0821 16:15:01.399240 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.840s
I0821 16:15:38.469218 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.555s
I0821 16:16:11.168466 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.588s
I0821 16:16:50.251945 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.764s
I0821 16:17:02.975272 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.683s
I0821 16:19:04.666023 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.559s
I0821 16:19:45.058353 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.676s
I0821 16:19:48.522006 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.699s
I0821 16:20:44.028719 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.980s
I0821 16:23:22.258761 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.875s
I0821 16:23:28.026877 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.879s
I0821 16:24:47.894318 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.553s
I0821 16:24:52.499584 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.658s
I0821 16:26:04.190996 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.932s
I0821 16:27:11.837412 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.737s
I0821 16:27:15.724723 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.950s
I0821 16:27:20.302796 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.794s
I0821 16:27:35.251392 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.631s
I0821 16:27:46.788931 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.983s
I0821 16:27:53.986914 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.942s
I0821 16:28:03.260872 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.829s
I0821 16:28:09.685441 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.993s
I0821 16:30:47.811012 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.826s
I0821 16:31:28.813786 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.599s
I0821 16:31:47.722554 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.636s
I0821 16:32:00.096619 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.509s
I0821 16:32:13.966706 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.433s
I0821 16:32:58.220968 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.961s
I0821 16:35:07.729588 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.604s
I0821 16:36:03.349252 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.518s
I0821 16:36:51.805505 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.959s
I0821 16:37:19.217131 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.997s
I0821 16:37:29.684691 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.712s
I0821 16:39:18.164769 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.781s
I0821 16:40:12.002286 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.869s
I0821 16:41:27.360632 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.505s
I0821 16:41:39.812523 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.757s
I0821 16:41:43.729736 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.666s
I0821 16:42:33.356925 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.904s
I0821 16:42:40.437927 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.873s
I0821 16:43:33.538668 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.549s
I0821 16:44:24.340186 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.551s
I0821 16:44:34.468760 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.874s
I0821 16:47:53.437352 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.783s
I0821 16:48:00.045074 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.601s
I0821 16:49:16.380790 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.829s
I0821 16:50:27.312837 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.800s
I0821 16:50:34.845422 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.610s
I0821 16:50:49.174410 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.942s
I0821 16:51:06.193173 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.705s
I0821 16:51:36.032842 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.872s
I0821 16:51:44.165616 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.401s
I0821 16:52:41.442965 910 pg_client_service.cc:2621] Fetching set of database oids
I0821 16:52:41.443919 910 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0821 16:52:41.443936 910 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0821 16:53:00.847640 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.689s
I0821 16:53:26.203279 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.747s
I0821 16:53:38.613338 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.699s
I0821 16:55:57.593917 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.824s
I0821 16:57:44.736164 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.456s
I0821 16:57:47.236712 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.503s
I0821 16:59:10.527494 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.766s
I0821 17:00:21.930722 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.485s
I0821 17:00:25.756793 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.605s
I0821 17:01:11.919721 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.984s
I0821 17:02:40.735818 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.951s
I0821 17:02:56.668953 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.697s
I0821 17:03:00.110428 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.627s
I0821 17:04:53.582834 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.635s
I0821 17:05:10.123266 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.663s
I0821 17:05:44.107311 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.817s
I0821 17:05:48.207614 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.712s
I0821 17:06:34.144124 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.725s
I0821 17:06:39.876634 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.808s
I0821 17:08:00.569892 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.644s
I0821 17:08:39.553022 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.475s
I0821 17:08:47.433712 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.814s
I0821 17:09:49.496816 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.957s
W0821 17:11:34.689574 308130 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313308) to 10.0.0.3:7100 timed out after 1.500s
I0821 17:11:36.803139 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.544s
I0821 17:12:08.821452 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.914s
I0821 17:12:55.248497 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.694s
I0821 17:13:02.065562 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.895s
I0821 17:13:15.204022 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.590s
I0821 17:13:34.551263 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.853s
I0821 17:14:30.180747 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.882s
I0821 17:15:11.945348 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.860s
I0821 17:15:48.339861 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.832s
I0821 17:17:21.619091 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.669s
I0821 17:18:05.668851 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.557s
I0821 17:18:56.339727 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.746s
I0821 17:19:04.597803 914 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:34387 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
I0821 17:20:21.398010 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.732s
I0821 17:20:43.353433 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.853s
I0821 17:20:54.379288 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.718s
I0821 17:21:41.337855 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.704s
I0821 17:21:55.188711 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.892s
I0821 17:22:30.376655 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.826s
I0821 17:23:54.482775 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.872s
I0821 17:23:58.184351 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.862s
I0821 17:24:23.651588 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.501s
I0821 17:25:46.661424 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.571s
I0821 17:26:05.309238 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.561s
I0821 17:26:35.650367 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.712s
I0821 17:26:57.774955 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.839s
I0821 17:27:31.205356 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.947s
I0821 17:27:39.766117 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.904s
I0821 17:28:09.313989 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.645s
I0821 17:28:30.664511 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.712s
I0821 17:29:00.214433 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.673s
I0821 17:29:09.075519 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.700s
I0821 17:29:25.070168 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.738s
I0821 17:30:13.489826 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.744s
I0821 17:30:21.011318 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.643s
I0821 17:30:57.944125 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.594s
I0821 17:31:25.425765 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.503s
I0821 17:33:05.370071 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.664s
I0821 17:33:21.609906 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.964s
I0821 17:33:52.510586 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.859s
I0821 17:35:15.255383 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.591s
I0821 17:35:26.036695 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.776s
I0821 17:35:49.799579 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.919s
I0821 17:35:57.663416 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.881s
I0821 17:36:55.805564 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.906s
I0821 17:38:58.223071 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.884s
I0821 17:39:39.334726 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.728s
I0821 17:39:56.487711 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.840s
I0821 17:40:18.491925 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.613s
I0821 17:41:04.933650 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.941s
W0821 17:42:04.223757 371912 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313311) to 10.0.0.3:7100 timed out after 1.500s
I0821 17:42:08.416098 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.638s
W0821 17:42:17.543408 916 yb_rpc.cc:553] Connection (0x000050403b391360) client 10.0.0.2:41871 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.098s, timeout: 15.000s, now: 121846.547s, last_read_time_: 121831.449s
I0821 17:42:19.212893 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.935s
W0821 17:42:34.158129 915 yb_rpc.cc:553] Connection (0x000050403b394c60) client 10.0.0.2:44693 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.000s, timeout: 15.000s, now: 121863.127s, last_read_time_: 121848.126s
I0821 17:42:36.426728 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.653s
I0821 17:42:37.470161 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.709s
I0821 17:42:41.099807 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.982s
I0821 17:43:40.732504 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.835s
I0821 17:44:50.014983 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.989s
I0821 17:44:51.446691 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.990s
I0821 17:45:03.790280 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.919s
W0821 17:46:41.621523 915 yb_rpc.cc:553] Connection (0x000050403b392fe0) client 10.0.0.2:36729 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 122110.626s, last_read_time_: 122095.527s
W0821 17:46:44.791843 914 yb_rpc.cc:553] Connection (0x000050403b33d520) client 10.0.0.2:38611 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.000s, timeout: 15.000s, now: 122113.761s, last_read_time_: 122098.760s
I0821 17:46:44.914419 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.934s
W0821 17:46:50.782351 914 yb_rpc.cc:553] Connection (0x000050403b98d520) client 10.0.0.2:36369 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.000s, timeout: 15.000s, now: 122119.761s, last_read_time_: 122104.760s
W0821 17:46:53.105176 381922 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313314) to 10.0.0.3:7100 timed out after 1.500s
I0821 17:46:53.320724 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.805s
I0821 17:46:54.217437 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.485s
I0821 17:47:26.943219 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.618s
I0821 17:48:34.866107 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.761s
I0821 17:49:02.378962 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.819s
I0821 17:49:23.688468 387248 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.999s
I0821 17:49:29.945214 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.827s
I0821 17:49:35.014693 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.560s
I0821 17:49:36.992327 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.546s
I0821 17:50:21.672269 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.694s
I0821 17:50:52.365478 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.738s
I0821 17:51:17.598317 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.816s
I0821 17:51:21.687708 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.761s
I0821 17:51:38.268064 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.896s
I0821 17:52:01.000733 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.998s
I0821 17:52:30.885329 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.972s
I0821 17:52:37.739643 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.636s
I0821 17:52:56.024876 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.990s
I0821 17:53:27.073918 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.925s
I0821 17:54:12.437568 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.512s
I0821 17:54:54.060240 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.532s
I0821 17:56:08.490202 916 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:35335 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
W0821 17:56:34.730620 402156 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313317) to 10.0.0.3:7100 timed out after 1.500s
I0821 17:57:16.508565 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.817s
I0821 17:57:25.067237 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.445s
I0821 17:57:43.133155 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.746s
I0821 17:58:33.645357 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.597s
I0821 18:00:32.167775 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.653s
I0821 18:01:52.131922 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.708s
I0821 18:01:59.807384 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.912s
I0821 18:02:05.232326 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.723s
I0821 18:04:24.030759 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.600s
I0821 18:05:25.903949 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.975s
I0821 18:06:29.559708 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.803s
I0821 18:06:38.991492 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.648s
I0821 18:10:11.069502 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.575s
I0821 18:11:18.875038 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.912s
I0821 18:11:29.701985 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.683s
I0821 18:12:18.428380 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.490s
I0821 18:14:01.641570 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.559s
I0821 18:14:07.521317 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.749s
W0821 18:14:08.254746 914 yb_rpc.cc:553] Connection (0x000050403b343360) client 10.0.0.2:41217 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 123757.260s, last_read_time_: 123742.161s
W0821 18:14:08.343225 916 yb_rpc.cc:553] Connection (0x000050403b462e20) client 10.0.0.2:37577 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 123757.348s, last_read_time_: 123742.249s
I0821 18:14:21.535578 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.991s
I0821 18:14:28.467226 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.646s
I0821 18:14:41.929301 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.742s
I0821 18:14:53.067237 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.633s
I0821 18:15:03.287068 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.610s
I0821 18:15:38.810760 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.876s
I0821 18:15:44.529934 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.625s
I0821 18:16:24.857354 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.633s
I0821 18:16:33.798077 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.827s
I0821 18:17:33.498246 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.594s
I0821 18:19:45.286846 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.666s
I0821 18:21:09.263783 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.601s
I0821 18:21:53.355829 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.671s
I0821 18:22:21.812834 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.940s
I0821 18:24:45.528821 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.614s
I0821 18:26:16.653344 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.985s
I0821 18:28:52.015910 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.744s
I0821 18:29:07.761386 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.575s
I0821 18:30:00.133635 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.493s
I0821 18:31:54.233464 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.598s
I0821 18:33:54.485060 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.531s
I0821 18:34:01.538599 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.989s
I0821 18:34:15.153210 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.708s
I0821 18:34:32.152017 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.483s
I0821 18:34:37.746855 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.994s
I0821 18:36:56.198311 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.774s
I0821 18:39:04.489054 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.541s
I0821 18:39:20.871776 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.853s
I0821 18:39:35.270315 917 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:36659 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
I0821 18:40:26.746351 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.849s
I0821 18:41:26.620879 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.787s
I0821 18:41:36.810604 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.642s
I0821 18:41:49.075079 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.858s
I0821 18:42:06.926860 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.972s
W0821 18:42:20.221997 915 yb_rpc.cc:553] Connection (0x000050403b329a60) client 10.0.0.2:45165 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.098s, timeout: 15.000s, now: 125449.226s, last_read_time_: 125434.128s
I0821 18:42:21.617331 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.733s
I0821 18:42:36.153802 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.988s
I0821 18:42:45.928812 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.874s
I0821 18:42:47.653040 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.835s
I0821 18:43:24.587893 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.849s
I0821 18:43:32.658304 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.985s
I0821 18:43:35.560446 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.942s
I0821 18:44:06.861195 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.903s
I0821 18:44:27.502820 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.889s
I0821 18:46:57.627434 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.641s
I0821 18:47:03.829711 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.929s
I0821 18:47:15.500916 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.466s
I0821 18:47:18.612777 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.732s
I0821 18:47:30.491489 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.878s
I0821 18:47:33.546734 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.691s
I0821 18:47:39.054639 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.941s
I0821 18:48:00.870184 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.548s
I0821 18:48:19.000155 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.659s
I0821 18:48:34.205926 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.773s
I0821 18:49:34.382622 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.565s
I0821 18:49:35.288496 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.632s
I0821 18:49:51.625648 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.828s
I0821 18:50:19.553995 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.934s
I0821 18:50:23.471218 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.868s
I0821 18:52:31.231559 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.827s
I0821 18:53:13.885929 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.471s
I0821 18:53:19.365068 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.634s
I0821 18:53:23.914148 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.966s
I0821 18:53:38.080484 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.715s
I0821 18:54:14.682677 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.689s
I0821 18:54:23.958142 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.552s
I0821 18:56:16.791869 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.965s
I0821 18:56:46.722565 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.732s
I0821 18:57:21.703631 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.932s
I0821 18:57:31.867982 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.972s
I0821 18:58:27.656890 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.786s
I0821 18:58:53.629643 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.814s
I0821 18:59:08.554358 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.779s
I0821 18:59:11.989969 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.769s
I0821 18:59:31.209741 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.967s
I0821 18:59:35.973362 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.852s
I0821 19:00:23.555203 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.926s
I0821 19:01:51.268453 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.717s
I0821 19:02:21.295809 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.940s
I0821 19:02:43.109441 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.973s
I0821 19:03:00.327113 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.665s
I0821 19:03:06.397243 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.888s
I0821 19:03:26.096130 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.482s
I0821 19:05:09.064605 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.711s
I0821 19:05:13.046808 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.844s
W0821 19:05:38.895648 546077 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313320) to 10.0.0.3:7100 timed out after 1.500s
I0821 19:05:56.913779 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.498s
I0821 19:06:01.479365 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.824s
I0821 19:06:41.773500 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.962s
I0821 19:09:44.983952 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.426s
I0821 19:10:05.209916 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.594s
I0821 19:10:29.565128 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.772s
I0821 19:10:45.469962 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.916s
I0821 19:11:00.030858 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.550s
I0821 19:12:58.901235 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.977s
I0821 19:13:20.030712 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.506s
I0821 19:14:09.173858 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.731s
I0821 19:14:28.950474 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.738s
I0821 19:14:38.835599 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.731s
I0821 19:15:00.723522 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.610s
I0821 19:15:01.778227 917 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:37727 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
I0821 19:15:28.262492 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.682s
I0821 19:15:35.507787 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.678s
I0821 19:15:55.376205 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.793s
I0821 19:16:01.578133 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.836s
I0821 19:16:09.743098 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.678s
I0821 19:16:26.389022 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.816s
I0821 19:16:54.532280 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.962s
I0821 19:17:08.248274 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.943s
I0821 19:17:17.232908 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.744s
I0821 19:17:25.713554 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.600s
W0821 19:17:27.642833 916 yb_rpc.cc:553] Connection (0x0000504038e05520) client 10.0.0.2:36163 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 127556.648s, last_read_time_: 127541.549s
I0821 19:17:35.227443 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.790s
I0821 19:17:47.930346 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.658s
I0821 19:18:16.491112 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.777s
I0821 19:18:18.718241 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.938s
I0821 19:18:31.538332 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.503s
I0821 19:18:34.407011 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.875s
I0821 19:19:03.183202 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.524s
I0821 19:19:03.367709 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.849s
I0821 19:19:30.965200 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.891s
I0821 19:19:49.461567 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.730s
I0821 19:19:56.725674 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.735s
I0821 19:21:26.066586 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.733s
I0821 19:21:30.602005 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.867s
I0821 19:21:35.587714 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.490s
I0821 19:22:49.520188 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.458s
I0821 19:23:59.358376 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.610s
I0821 19:24:02.778318 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.944s
I0821 19:25:27.954993 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.540s
I0821 19:25:42.908505 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.897s
I0821 19:26:08.376147 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.691s
I0821 19:26:58.736960 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.736s
I0821 19:27:58.752769 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.558s
I0821 19:28:15.452781 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.773s
I0821 19:28:21.595795 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.662s
I0821 19:28:32.772800 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.747s
I0821 19:28:39.253393 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.903s
I0821 19:30:19.964383 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.787s
I0821 19:31:33.711807 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.558s
I0821 19:31:38.939471 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.509s
W0821 19:34:56.178231 607053 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313326) to 10.0.0.3:7100 timed out after 1.500s
I0821 19:35:05.858385 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.691s
I0821 19:35:40.064213 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.684s
I0821 19:37:40.048053 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.752s
I0821 19:37:56.902968 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.694s
I0821 19:39:33.485870 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.808s
I0821 19:39:40.686154 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.549s
I0821 19:39:56.577410 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.927s
I0821 19:40:36.051496 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.745s
I0821 19:41:20.227792 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.692s
I0821 19:42:39.177928 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.462s
I0821 19:42:47.175117 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.917s
I0821 19:43:07.885924 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.563s
I0821 19:43:35.245316 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.988s
I0821 19:43:42.854686 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.912s
I0821 19:44:57.343703 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.809s
I0821 19:45:09.204102 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.734s
I0821 19:45:15.116437 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.760s
I0821 19:45:15.702167 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.657s
I0821 19:45:37.445520 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.503s
I0821 19:46:20.487627 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.612s
I0821 19:46:24.665064 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.783s
I0821 19:47:28.302838 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.772s
I0821 19:47:57.422137 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.721s
I0821 19:48:24.790975 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.580s
W0821 19:48:32.137068 635220 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313329) to 10.0.0.3:7100 timed out after 1.500s
I0821 19:49:33.537587 914 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:38273 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
W0821 19:50:03.264943 638357 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313332) to 10.0.0.3:7100 timed out after 1.500s
I0821 19:52:12.846309 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.667s
I0821 19:52:29.512938 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.986s
I0821 19:52:41.444074 913 pg_client_service.cc:2621] Fetching set of database oids
I0821 19:52:41.445653 913 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0821 19:52:41.445688 913 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0821 19:52:44.336848 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.881s
I0821 19:53:14.947618 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.799s
I0821 19:53:16.879753 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.726s
I0821 19:55:23.482580 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.574s
I0821 19:55:28.283226 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.773s
I0821 19:56:05.912721 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.711s
I0821 19:56:23.037345 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.783s
I0821 19:57:01.360164 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.838s
I0821 19:57:09.072912 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.930s
I0821 19:57:16.515154 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.533s
I0821 19:57:25.378386 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.606s
I0821 19:57:29.816973 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.827s
I0821 19:58:27.867141 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.621s
I0821 19:58:38.394739 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.758s
I0821 19:58:55.113394 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.634s
W0821 19:59:25.760288 657840 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313336) to 10.0.0.3:7100 timed out after 1.500s
I0821 19:59:34.153653 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.755s
I0821 20:00:07.124325 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.907s
I0821 20:02:43.108176 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.475s
I0821 20:03:00.727633 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.781s
I0821 20:03:08.145391 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.937s
I0821 20:05:28.039611 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.971s
I0821 20:07:36.634173 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.628s
I0821 20:07:44.657974 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.666s
I0821 20:08:06.383384 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.891s
I0821 20:08:28.397291 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.706s
I0821 20:09:15.661494 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.810s
I0821 20:09:25.392169 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.770s
I0821 20:09:30.585798 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.959s
I0821 20:11:13.909446 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.684s
I0821 20:11:53.978487 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.848s
I0821 20:11:58.413252 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.905s
I0821 20:13:11.645866 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.779s
W0821 20:14:22.868934 688978 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313351) to 10.0.0.3:7100 timed out after 1.500s
I0821 20:14:26.467008 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.952s
I0821 20:14:40.934114 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.509s
I0821 20:15:52.420388 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.941s
I0821 20:16:00.515632 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.639s
W0821 20:16:31.462807 693462 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313354) to 10.0.0.3:7100 timed out after 1.500s
I0821 20:17:33.712623 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.668s
I0821 20:17:53.718232 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.721s
I0821 20:17:53.835814 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.603s
I0821 20:18:29.657861 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.831s
I0821 20:19:11.311312 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.897s
I0821 20:20:23.877247 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.972s
I0821 20:21:42.894577 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.844s
I0821 20:21:50.479303 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.745s
I0821 20:21:56.180956 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.456s
I0821 20:22:05.488199 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.740s
I0821 20:22:21.445079 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.905s
I0821 20:22:40.018356 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.669s
I0821 20:23:47.201463 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.499s
W0821 20:24:08.255846 914 yb_rpc.cc:553] Connection (0x0000504039e0ae20) client 10.0.0.2:41547 => 10.0.0.3:7100: Network error (yb/rpc/yb_rpc.cc:552): Rpc timeout, passed: 15.099s, timeout: 15.000s, now: 131557.260s, last_read_time_: 131542.161s
I0821 20:24:09.093060 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.995s
I0821 20:24:26.341496 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.792s
I0821 20:24:58.310257 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.628s
I0821 20:25:58.517942 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.928s
I0821 20:29:12.291942 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.570s
I0821 20:29:46.091311 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.797s
I0821 20:30:19.090050 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.727s
I0821 20:32:26.017452 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.784s
I0821 20:32:28.031431 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.748s
I0821 20:32:28.720140 916 tcp_stream.cc:329] { local: 10.0.0.2:7100 remote: 10.0.0.3:34443 }: Recv failed: Network error (yb/util/net/socket.cc:542): recvmsg error: Connection reset by peer (system error 104)
I0821 20:32:39.431984 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.959s
I0821 20:32:50.353722 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.872s
I0821 20:32:55.065826 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.978s
I0821 20:33:14.229154 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.637s
W0821 20:34:52.858655 731699 master.cc:552] ListMasters: Timed out (yb/rpc/outbound_call.cc:685): Unable to get registration information for peer ([10.0.0.3:7100]) id (a5110a40aa5b46eaa183f0022d8d9063): GetMasterRegistration RPC (request call id 313357) to 10.0.0.3:7100 timed out after 1.500s
I0821 20:36:06.101348 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.759s
I0821 20:36:14.737119 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.957s
I0821 20:36:37.348163 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.631s
I0821 20:36:51.505460 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.995s
I0821 20:36:56.518725 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.586s
I0821 20:37:10.585952 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.551s
I0821 20:37:26.424504 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.623s
I0821 20:37:47.130385 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.640s
I0821 20:39:53.071214 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.657s
I0821 20:41:09.294793 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.602s
I0821 20:42:33.504117 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.670s
I0821 20:43:52.000905 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.649s
I0821 20:44:38.536216 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.709s
I0821 20:45:46.568017 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.927s
I0821 20:45:52.251847 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.675s
I0821 20:46:29.465901 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.596s
I0821 20:48:17.698280 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.709s
I0821 20:48:32.458948 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.762s
I0821 20:48:56.003732 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.741s
I0821 20:50:07.878640 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.853s
I0821 20:50:12.785135 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.775s
I0821 20:50:46.637905 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.945s
I0821 20:52:47.594213 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.475s
I0821 20:53:12.727764 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.889s
I0821 20:53:29.550770 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.885s
I0821 20:55:00.741914 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.702s
I0821 20:55:55.415787 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.699s
I0821 20:56:00.572340 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.879s
I0821 20:56:14.149115 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.550s
I0821 20:57:16.060707 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.827s
I0821 20:58:08.410429 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.717s
I0821 20:58:16.418396 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.753s
I0821 20:58:40.930151 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.978s
I0821 20:58:54.714826 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.682s
I0821 20:59:07.596943 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.863s
I0821 20:59:17.624503 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.479s
I0821 20:59:37.999512 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.610s
I0821 20:59:55.399425 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.848s
I0821 21:00:44.403275 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.713s
I0821 21:01:47.293627 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.604s
I0821 21:03:26.797258 154948 raft_consensus.cc:3146] T 00000000000000000000000000000000 P 55d29a1415af4e44ac228b2c8a82f866 [term 30 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a5110a40aa5b46eaa183f0022d8d9063 for term 31 because replica believes a valid leader to be alive (leader uuid: 64fedbe39ec148ab8f9e982b6d19f6d2). Time left: 2.605s
I0821 22:52:41.445765 913 pg_client_service.cc:2621] Fetching set of database oids
I0821 22:52:41.446851 913 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0821 22:52:41.446864 913 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 01:52:41.446998 910 pg_client_service.cc:2621] Fetching set of database oids
I0822 01:52:41.448098 910 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 01:52:41.448125 910 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 04:52:41.448246 911 pg_client_service.cc:2621] Fetching set of database oids
I0822 04:52:41.450357 911 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 04:52:41.450388 911 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 07:52:41.450677 911 pg_client_service.cc:2621] Fetching set of database oids
I0822 07:52:41.452926 911 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 07:52:41.452975 911 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 10:52:41.453110 912 pg_client_service.cc:2621] Fetching set of database oids
I0822 10:52:41.454126 912 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 10:52:41.454170 912 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 13:52:41.454280 910 pg_client_service.cc:2621] Fetching set of database oids
I0822 13:52:41.455403 910 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 13:52:41.455437 910 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 16:52:41.455554 912 pg_client_service.cc:2621] Fetching set of database oids
I0822 16:52:41.456655 912 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 16:52:41.456688 912 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators
I0822 17:23:08.284735 3292577 tcmalloc_profile.cc:145] Analyzing TCMalloc sampling profile
I0822 17:23:08.285552 3292580 pprof-path-handlers.cc:195] Starting a heap profile: seconds=1 only_growth=0 sample_freq_bytes=10485760
I0822 17:23:08.285580 3292580 tcmalloc_profile.cc:122] Sleeping for 1 seconds while profile is collected.
W0822 17:23:08.570073 3292577 tcmalloc_profile.cc:194] Failed to symbolize 143 symbols
I0822 17:23:09.285717 3292580 tcmalloc_profile.cc:145] Analyzing TCMalloc sampling profile
W0822 17:23:09.337355 3292580 tcmalloc_profile.cc:194] Failed to symbolize 12 symbols
I0822 19:52:41.456775 910 pg_client_service.cc:2621] Fetching set of database oids
I0822 19:52:41.457880 910 pg_client_service.cc:2628] Successfully fetched 7 database oids
I0822 19:52:41.457930 910 pg_client_service.cc:2647] ScheduleCheckObjectIdAllocators