diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..71378c8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,8 @@ +# Run manually to reformat a file: +# clang-format -i --style=file +# find . -iname '*.cc' -o -iname '*.h' -o -iname '*.h.in' | xargs clang-format -i --style=file +BasedOnStyle: Google +DerivePointerAlignment: false + +# FIXME(hds) +IndentWidth: 4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f23d06..bfdc737 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,6 +59,10 @@ jobs: chmod a+x bazel-0.25.2-installer-linux-x86_64.sh ./bazel-0.25.2-installer-linux-x86_64.sh --user export PATH="$PATH:$HOME/bin" + - name: Check lint + run: | + clang-format --dry-run --Werror src/braft/*.h src/braft/*.cpp + - name: Build with cmake if: ${{ matrix.build_tool == 'cmake' }} run: | diff --git a/src/braft/ballot.cpp b/src/braft/ballot.cpp index ec4951a..9f9340c 100644 --- a/src/braft/ballot.cpp +++ b/src/braft/ballot.cpp @@ -16,12 +16,16 @@ #include "braft/ballot.h" +#include #include +#include + #include "braft/configuration.h" namespace braft { -void Ballot::init(const Configuration& conf, std::optional old_conf) { +void Ballot::init(const Configuration& conf, + std::optional old_conf) { _peers.clear(); _old_peers.clear(); _quorum = 0; @@ -29,18 +33,15 @@ void Ballot::init(const Configuration& conf, std::optional CHECK_GT(conf.size(), 0); _peers.reserve(conf.size()); - for (auto iter = conf.begin(); iter != conf.end(); ++iter) { - _peers.push_back(*iter); - } + std::copy(conf.begin(), conf.end(), std::back_inserter(_peers)); _quorum = _peers.size() / 2 + 1; + if (!old_conf.has_value()) { return; } _old_peers.reserve(old_conf->size()); - for (Configuration::const_iterator iter = old_conf->begin(); - iter != old_conf->end(); ++iter) { - _old_peers.push_back(*iter); - } + std::copy(old_conf->begin(), old_conf->end(), + std::back_inserter(_old_peers)); _old_quorum = _old_peers.size() / 2 + 1; } diff --git a/src/braft/ballot_box.cpp b/src/braft/ballot_box.cpp index c28f775..81d5c08 100644 --- a/src/braft/ballot_box.cpp +++ b/src/braft/ballot_box.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,31 +14,30 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) +#include "braft/ballot_box.h" + +#include #include #include -#include + #include #include -#include "braft/ballot_box.h" -#include "braft/util.h" -#include "braft/fsm_caller.h" + #include "braft/closure_queue.h" +#include "braft/fsm_caller.h" +#include "braft/util.h" namespace braft { BallotBox::BallotBox() - : _waiter(NULL) - , _closure_queue(NULL) - , _last_committed_index(0) - , _pending_index(0) -{ -} + : _waiter(NULL), + _closure_queue(NULL), + _last_committed_index(0), + _pending_index(0) {} -BallotBox::~BallotBox() { - clear_pending_tasks(); -} +BallotBox::~BallotBox() { clear_pending_tasks(); } -int BallotBox::init(const BallotBoxOptions &options) { +int BallotBox::init(const BallotBoxOptions& options) { if (options.waiter == NULL || options.closure_queue == NULL) { LOG(ERROR) << "waiter is NULL"; return EINVAL; @@ -48,9 +47,9 @@ int BallotBox::init(const BallotBoxOptions &options) { return 0; } -int BallotBox::commit_at( - int64_t first_log_index, int64_t last_log_index, const PeerId& peer) { - // FIXME(chenzhangyi01): The cricital section is unacceptable because it +int BallotBox::commit_at(int64_t first_log_index, int64_t last_log_index, + const PeerId& peer) { + // FIXME(chenzhangyi01): The cricital section is unacceptable because it // blocks all the other Replicators and LogManagers std::unique_lock lck(_mutex); if (_pending_index == 0) { @@ -59,14 +58,16 @@ int BallotBox::commit_at( if (last_log_index < _pending_index) { return 0; } - if (last_log_index >= _pending_index + (int64_t)_pending_meta_queue.size()) { + if (last_log_index >= + _pending_index + (int64_t)_pending_meta_queue.size()) { return ERANGE; } int64_t last_committed_index = 0; const int64_t start_at = std::max(_pending_index, first_log_index); Ballot::PosHint pos_hint; - for (int64_t log_index = start_at; log_index <= last_log_index; ++log_index) { + for (int64_t log_index = start_at; log_index <= last_log_index; + ++log_index) { Ballot& bl = _pending_meta_queue[log_index - _pending_index]; pos_hint = bl.grant(peer, pos_hint); if (bl.granted()) { @@ -82,15 +83,17 @@ int BallotBox::commit_at( // peers, the quorum would decrease by 1, e.g. 3 of 4 changes to 2 of 3. In // this case, the log after removal may be committed before some previous // logs, since we use the new configuration to deal the quorum of the - // removal request, we think it's safe to commit all the uncommitted + // removal request, we think it's safe to commit all the uncommitted // previous logs, which is not well proved right now // TODO: add vlog when committing previous logs - for (int64_t index = _pending_index; index <= last_committed_index; ++index) { + for (int64_t index = _pending_index; index <= last_committed_index; + ++index) { _pending_meta_queue.pop_front(); } - + _pending_index = last_committed_index + 1; - _last_committed_index.store(last_committed_index, butil::memory_order_relaxed); + _last_committed_index.store(last_committed_index, + butil::memory_order_relaxed); lck.unlock(); // The order doesn't matter _waiter->on_committed(last_committed_index); @@ -111,16 +114,17 @@ int BallotBox::clear_pending_tasks() { int BallotBox::reset_pending_index(int64_t new_pending_index) { BAIDU_SCOPED_LOCK(_mutex); CHECK(_pending_index == 0 && _pending_meta_queue.empty()) - << "pending_index " << _pending_index << " pending_meta_queue " + << "pending_index " << _pending_index << " pending_meta_queue " << _pending_meta_queue.size(); - CHECK_GT(new_pending_index, _last_committed_index.load( - butil::memory_order_relaxed)); + CHECK_GT(new_pending_index, + _last_committed_index.load(butil::memory_order_relaxed)); _pending_index = new_pending_index; _closure_queue->reset_first_index(new_pending_index); return 0; } -int BallotBox::append_pending_task(const Configuration& conf, const Configuration* old_conf, +int BallotBox::append_pending_task(const Configuration& conf, + const Configuration* old_conf, Closure* closure) { Ballot bl; bl.init(conf, @@ -142,12 +146,14 @@ int BallotBox::set_last_committed_index(int64_t last_committed_index) { << ", parameter last_committed_index=" << last_committed_index; return -1; } - if (last_committed_index < - _last_committed_index.load(butil::memory_order_relaxed)) { + if (last_committed_index < + _last_committed_index.load(butil::memory_order_relaxed)) { return EINVAL; } - if (last_committed_index > _last_committed_index.load(butil::memory_order_relaxed)) { - _last_committed_index.store(last_committed_index, butil::memory_order_relaxed); + if (last_committed_index > + _last_committed_index.load(butil::memory_order_relaxed)) { + _last_committed_index.store(last_committed_index, + butil::memory_order_relaxed); lck.unlock(); _waiter->on_committed(last_committed_index); } @@ -164,7 +170,7 @@ void BallotBox::describe(std::ostream& os, bool use_html) { pending_queue_size = _pending_meta_queue.size(); } lck.unlock(); - const char *newline = use_html ? "
" : "\r\n"; + const char* newline = use_html ? "
" : "\r\n"; os << "last_committed_index: " << committed_index << newline; if (pending_queue_size != 0) { os << "pending_index: " << pending_index << newline; diff --git a/src/braft/ballot_box.h b/src/braft/ballot_box.h index 60132e5..3d1a426 100644 --- a/src/braft/ballot_box.h +++ b/src/braft/ballot_box.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,16 +14,18 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_BALLOT_BOX_H -#define BRAFT_BALLOT_BOX_H +#ifndef BRAFT_BALLOT_BOX_H +#define BRAFT_BALLOT_BOX_H + +#include // butil::atomic +#include // int64_t -#include // int64_t -#include // std::set #include -#include // butil::atomic +#include // std::set + +#include "braft/ballot.h" #include "braft/raft.h" #include "braft/util.h" -#include "braft/ballot.h" namespace braft { @@ -31,25 +33,21 @@ class FSMCaller; class ClosureQueue; struct BallotBoxOptions { - BallotBoxOptions() - : waiter(NULL) - , closure_queue(NULL) - {} + BallotBoxOptions() : waiter(NULL), closure_queue(NULL) {} FSMCaller* waiter; ClosureQueue* closure_queue; }; struct BallotBoxStatus { BallotBoxStatus() - : committed_index(0), pending_index(0), pending_queue_size(0) - {} + : committed_index(0), pending_index(0), pending_queue_size(0) {} int64_t committed_index; int64_t pending_index; int64_t pending_queue_size; }; class BallotBox { -public: + public: BallotBox(); ~BallotBox(); @@ -61,46 +59,44 @@ class BallotBox { const PeerId& peer); // Called when the leader steps down, otherwise the behavior is undefined - // When a leader steps down, the uncommitted user applications should + // When a leader steps down, the uncommitted user applications should // fail immediately, which the new leader will deal whether to commit or // truncate. int clear_pending_tasks(); - + // Called when a candidate becomes the new leader, otherwise the behavior is // undefined. - // According to the raft algorithm, the logs from pervious terms can't be - // committed until a log at the new term becomes committed, so + // According to the raft algorithm, the logs from pervious terms can't be + // committed until a log at the new term becomes committed, so // |new_pending_index| should be |last_log_index| + 1. int reset_pending_index(int64_t new_pending_index); // Called by leader, otherwise the behavior is undefined // Store application context before replication. - int append_pending_task(const Configuration& conf, - const Configuration* old_conf, - Closure* closure); + int append_pending_task(const Configuration& conf, + const Configuration* old_conf, Closure* closure); // Called by follower, otherwise the behavior is undefined. // Set committed index received from leader int set_last_committed_index(int64_t last_committed_index); - int64_t last_committed_index() - { return _last_committed_index.load(butil::memory_order_acquire); } + int64_t last_committed_index() { + return _last_committed_index.load(butil::memory_order_acquire); + } void describe(std::ostream& os, bool use_html); void get_status(BallotBoxStatus* ballot_box_status); -private: - - FSMCaller* _waiter; - ClosureQueue* _closure_queue; - raft_mutex_t _mutex; - butil::atomic _last_committed_index; - int64_t _pending_index; - std::deque _pending_meta_queue; - + private: + FSMCaller* _waiter; + ClosureQueue* _closure_queue; + raft_mutex_t _mutex; + butil::atomic _last_committed_index; + int64_t _pending_index; + std::deque _pending_meta_queue; }; } // namespace braft -#endif //BRAFT_BALLOT_BOX_H +#endif // BRAFT_BALLOT_BOX_H diff --git a/src/braft/builtin_service_impl.cpp b/src/braft/builtin_service_impl.cpp index d7764a6..68a9148 100644 --- a/src/braft/builtin_service_impl.cpp +++ b/src/braft/builtin_service_impl.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,13 +17,14 @@ #include "braft/builtin_service_impl.h" -#include +#include #include +#include #include -#include + #include "braft/node.h" -#include "braft/replicator.h" #include "braft/node_manager.h" +#include "braft/replicator.h" namespace braft { @@ -34,9 +35,9 @@ void RaftStatImpl::GetTabInfo(brpc::TabInfoList* info_list) const { } void RaftStatImpl::default_method(::google::protobuf::RpcController* controller, - const ::braft::IndexRequest* /*request*/, - ::braft::IndexResponse* /*response*/, - ::google::protobuf::Closure* done) { + const ::braft::IndexRequest* /*request*/, + ::braft::IndexResponse* /*response*/, + ::google::protobuf::Closure* done) { brpc::ClosureGuard done_guard(done); brpc::Controller* cntl = (brpc::Controller*)controller; std::string group_id = cntl->http_request().unresolved_path(); @@ -55,7 +56,8 @@ void RaftStatImpl::default_method(::google::protobuf::RpcController* controller, butil::IOBufBuilder os; if (html) { os << "\n" - << "\n" + << "\n" << brpc::TabsHead() << ""; cntl->server()->PrintTabsBody(os, "raft"); } @@ -68,7 +70,7 @@ void RaftStatImpl::default_method(::google::protobuf::RpcController* controller, } std::string prev_group_id; - const char *newline = html ? "
" : "\r\n"; + const char* newline = html ? "
" : "\r\n"; for (size_t i = 0; i < nodes.size(); ++i) { const NodeId node_id = nodes[i]->node_id(); group_id = node_id.group_id; diff --git a/src/braft/builtin_service_impl.h b/src/braft/builtin_service_impl.h index 9bafa6c..2fd2d0f 100644 --- a/src/braft/builtin_service_impl.h +++ b/src/braft/builtin_service_impl.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,16 +15,17 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) // Ge,Jun(gejun@baiud.com) -#ifndef BRAFT_BUILTIN_SERVICE_IMPL_H -#define BRAFT_BUILTIN_SERVICE_IMPL_H +#ifndef BRAFT_BUILTIN_SERVICE_IMPL_H +#define BRAFT_BUILTIN_SERVICE_IMPL_H -#include "braft/builtin_service.pb.h" #include +#include "braft/builtin_service.pb.h" + namespace braft { class RaftStatImpl : public raft_stat, public brpc::Tabbed { -public: + public: void default_method(::google::protobuf::RpcController* controller, const ::braft::IndexRequest* request, ::braft::IndexResponse* response, @@ -35,4 +36,4 @@ class RaftStatImpl : public raft_stat, public brpc::Tabbed { } // namespace braft -#endif //BRAFT_BUILTIN_SERVICE_IMPL_H +#endif // BRAFT_BUILTIN_SERVICE_IMPL_H diff --git a/src/braft/cli.cpp b/src/braft/cli.cpp index c044781..0424e94 100644 --- a/src/braft/cli.cpp +++ b/src/braft/cli.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2018 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,28 +16,29 @@ #include "braft/cli.h" -#include // brpc::Channel -#include // brpc::Controller -#include "braft/cli.pb.h" // CliService_Stub +#include // brpc::Channel +#include // brpc::Controller + +#include "braft/cli.pb.h" // CliService_Stub #include "braft/util.h" namespace braft { namespace cli { -static butil::Status get_leader(const GroupId& group_id, const Configuration& conf, - PeerId* leader_id) { +static butil::Status get_leader(const GroupId& group_id, + const Configuration& conf, PeerId* leader_id) { if (conf.empty()) { return butil::Status(EINVAL, "Empty group configuration"); } // Construct a brpc naming service to access all the nodes in this group butil::Status st(-1, "Fail to get leader of group %s", group_id.c_str()); leader_id->reset(); - for (Configuration::const_iterator - iter = conf.begin(); iter != conf.end(); ++iter) { + for (Configuration::const_iterator iter = conf.begin(); iter != conf.end(); + ++iter) { brpc::Channel channel; if (channel.Init(iter->addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - iter->to_string().c_str()); + iter->to_string().c_str()); } CliService_Stub stub(&channel); GetLeaderRequest request; @@ -49,14 +50,14 @@ static butil::Status get_leader(const GroupId& group_id, const Configuration& co if (cntl.Failed()) { if (st.ok()) { st.set_error(cntl.ErrorCode(), "[%s] %s", - butil::endpoint2str(cntl.remote_side()).c_str(), - cntl.ErrorText().c_str()); + butil::endpoint2str(cntl.remote_side()).c_str(), + cntl.ErrorText().c_str()); } else { std::string saved_et = st.error_str(); - st.set_error(cntl.ErrorCode(), "%s, [%s] %s", saved_et.c_str(), - butil::endpoint2str(cntl.remote_side()).c_str(), - cntl.ErrorText().c_str()); - } + st.set_error(cntl.ErrorCode(), "%s, [%s] %s", saved_et.c_str(), + butil::endpoint2str(cntl.remote_side()).c_str(), + cntl.ErrorText().c_str()); + } continue; } leader_id->parse(response.leader_id()); @@ -75,7 +76,7 @@ butil::Status add_peer(const GroupId& group_id, const Configuration& conf, brpc::Channel channel; if (channel.Init(leader_id.addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - leader_id.to_string().c_str()); + leader_id.to_string().c_str()); } AddPeerRequest request; request.set_group_id(group_id); @@ -100,20 +101,19 @@ butil::Status add_peer(const GroupId& group_id, const Configuration& conf, new_conf.add_peer(response.new_peers(i)); } LOG(INFO) << "Configuration of replication group `" << group_id - << "' changed from " << old_conf - << " to " << new_conf; + << "' changed from " << old_conf << " to " << new_conf; return butil::Status::OK(); } butil::Status remove_peer(const GroupId& group_id, const Configuration& conf, - const PeerId& peer_id, const CliOptions& options) { + const PeerId& peer_id, const CliOptions& options) { PeerId leader_id; butil::Status st = get_leader(group_id, conf, &leader_id); BRAFT_RETURN_IF(!st.ok(), st); brpc::Channel channel; if (channel.Init(leader_id.addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - leader_id.to_string().c_str()); + leader_id.to_string().c_str()); } RemovePeerRequest request; request.set_group_id(group_id); @@ -138,8 +138,7 @@ butil::Status remove_peer(const GroupId& group_id, const Configuration& conf, new_conf.add_peer(response.new_peers(i)); } LOG(INFO) << "Configuration of replication group `" << group_id - << "' changed from " << old_conf - << " to " << new_conf; + << "' changed from " << old_conf << " to " << new_conf; return butil::Status::OK(); } @@ -152,7 +151,7 @@ butil::Status reset_peer(const GroupId& group_id, const PeerId& peer_id, brpc::Channel channel; if (channel.Init(peer_id.addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - peer_id.to_string().c_str()); + peer_id.to_string().c_str()); } brpc::Controller cntl; cntl.set_timeout_ms(options.timeout_ms); @@ -160,8 +159,8 @@ butil::Status reset_peer(const GroupId& group_id, const PeerId& peer_id, ResetPeerRequest request; request.set_group_id(group_id); request.set_peer_id(peer_id.to_string()); - for (Configuration::const_iterator - iter = new_conf.begin(); iter != new_conf.end(); ++iter) { + for (Configuration::const_iterator iter = new_conf.begin(); + iter != new_conf.end(); ++iter) { request.add_new_peers(iter->to_string()); } ResetPeerResponse response; @@ -174,11 +173,11 @@ butil::Status reset_peer(const GroupId& group_id, const PeerId& peer_id, } butil::Status snapshot(const GroupId& group_id, const PeerId& peer_id, - const CliOptions& options) { + const CliOptions& options) { brpc::Channel channel; if (channel.Init(peer_id.addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - peer_id.to_string().c_str()); + peer_id.to_string().c_str()); } brpc::Controller cntl; cntl.set_timeout_ms(options.timeout_ms); @@ -206,14 +205,14 @@ butil::Status change_peers(const GroupId& group_id, const Configuration& conf, brpc::Channel channel; if (channel.Init(leader_id.addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - leader_id.to_string().c_str()); + leader_id.to_string().c_str()); } ChangePeersRequest request; request.set_group_id(group_id); request.set_leader_id(leader_id.to_string()); - for (Configuration::const_iterator - iter = new_peers.begin(); iter != new_peers.end(); ++iter) { + for (Configuration::const_iterator iter = new_peers.begin(); + iter != new_peers.end(); ++iter) { request.add_new_peers(iter->to_string()); } ChangePeersResponse response; @@ -235,13 +234,13 @@ butil::Status change_peers(const GroupId& group_id, const Configuration& conf, new_conf.add_peer(response.new_peers(i)); } LOG(INFO) << "Configuration of replication group `" << group_id - << "' changed from " << old_conf - << " to " << new_conf; + << "' changed from " << old_conf << " to " << new_conf; return butil::Status::OK(); } -butil::Status transfer_leader(const GroupId& group_id, const Configuration& conf, - const PeerId& peer, const CliOptions& options) { +butil::Status transfer_leader(const GroupId& group_id, + const Configuration& conf, const PeerId& peer, + const CliOptions& options) { PeerId leader_id; butil::Status st = get_leader(group_id, conf, &leader_id); BRAFT_RETURN_IF(!st.ok(), st); @@ -252,7 +251,7 @@ butil::Status transfer_leader(const GroupId& group_id, const Configuration& conf brpc::Channel channel; if (channel.Init(leader_id.addr, NULL) != 0) { return butil::Status(-1, "Fail to init channel to %s", - leader_id.to_string().c_str()); + leader_id.to_string().c_str()); } TransferLeaderRequest request; request.set_group_id(group_id); diff --git a/src/braft/cli.h b/src/braft/cli.h index 516c063..953398f 100644 --- a/src/braft/cli.h +++ b/src/braft/cli.h @@ -1,11 +1,11 @@ // Copyright (c) 2018 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,8 +14,8 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_CLI_H -#define BRAFT_CLI_H +#ifndef BRAFT_CLI_H +#define BRAFT_CLI_H #include "braft/raft.h" @@ -39,13 +39,14 @@ butil::Status remove_peer(const GroupId& group_id, const Configuration& conf, const PeerId& peer_id, const CliOptions& options); // Gracefully change the peers of the replication group. -butil::Status change_peers(const GroupId& group_id, const Configuration& conf, +butil::Status change_peers(const GroupId& group_id, const Configuration& conf, const Configuration& new_peers, const CliOptions& options); // Transfer the leader of the replication group to the target peer -butil::Status transfer_leader(const GroupId& group_id, const Configuration& conf, - const PeerId& peer, const CliOptions& options); +butil::Status transfer_leader(const GroupId& group_id, + const Configuration& conf, const PeerId& peer, + const CliOptions& options); // Reset the peer set of the target peer butil::Status reset_peer(const GroupId& group_id, const PeerId& peer_id, @@ -59,4 +60,4 @@ butil::Status snapshot(const GroupId& group_id, const PeerId& peer_id, } // namespace cli } // namespace braft -#endif //BRAFT_CLI_H +#endif // BRAFT_CLI_H diff --git a/src/braft/cli_service.cpp b/src/braft/cli_service.cpp index 5f84079..9f1a958 100644 --- a/src/braft/cli_service.cpp +++ b/src/braft/cli_service.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2018 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,19 +16,20 @@ #include "braft/cli_service.h" -#include // brpc::Controller -#include "braft/node_manager.h" // NodeManager -#include "braft/closure_helper.h" // NewCallback +#include // brpc::Controller + +#include "braft/closure_helper.h" // NewCallback +#include "braft/node_manager.h" // NodeManager namespace braft { static void add_peer_returned(brpc::Controller* cntl, - const AddPeerRequest* request, - AddPeerResponse* response, - std::vector old_peers, - scoped_refptr /*node*/, - ::google::protobuf::Closure* done, - const butil::Status& st) { + const AddPeerRequest* request, + AddPeerResponse* response, + std::vector old_peers, + scoped_refptr /*node*/, + ::google::protobuf::Closure* done, + const butil::Status& st) { brpc::ClosureGuard done_guard(done); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); @@ -54,7 +55,8 @@ void CliServiceImpl::add_peer(::google::protobuf::RpcController* controller, brpc::Controller* cntl = (brpc::Controller*)controller; brpc::ClosureGuard done_guard(done); scoped_refptr node; - butil::Status st = get_node(&node, request->group_id(), request->leader_id()); + butil::Status st = + get_node(&node, request->group_id(), request->leader_id()); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); return; @@ -68,25 +70,24 @@ void CliServiceImpl::add_peer(::google::protobuf::RpcController* controller, PeerId adding_peer; if (adding_peer.parse(request->peer_id()) != 0) { cntl->SetFailed(EINVAL, "Fail to parse peer_id %s", - request->peer_id().c_str()); + request->peer_id().c_str()); return; } - LOG(WARNING) << "Receive AddPeerRequest to " << node->node_id() - << " from " << cntl->remote_side() - << ", adding " << request->peer_id(); - Closure* add_peer_done = NewCallback( - add_peer_returned, cntl, request, response, peers, node, - done_guard.release()); + LOG(WARNING) << "Receive AddPeerRequest to " << node->node_id() << " from " + << cntl->remote_side() << ", adding " << request->peer_id(); + Closure* add_peer_done = + NewCallback(add_peer_returned, cntl, request, response, peers, node, + done_guard.release()); return node->add_peer(adding_peer, add_peer_done); } static void remove_peer_returned(brpc::Controller* cntl, - const RemovePeerRequest* request, - RemovePeerResponse* response, - std::vector old_peers, - scoped_refptr /*node*/, - ::google::protobuf::Closure* done, - const butil::Status& st) { + const RemovePeerRequest* request, + RemovePeerResponse* response, + std::vector old_peers, + scoped_refptr /*node*/, + ::google::protobuf::Closure* done, + const butil::Status& st) { brpc::ClosureGuard done_guard(done); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); @@ -107,7 +108,8 @@ void CliServiceImpl::remove_peer(::google::protobuf::RpcController* controller, brpc::Controller* cntl = (brpc::Controller*)controller; brpc::ClosureGuard done_guard(done); scoped_refptr node; - butil::Status st = get_node(&node, request->group_id(), request->leader_id()); + butil::Status st = + get_node(&node, request->group_id(), request->leader_id()); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); return; @@ -121,15 +123,15 @@ void CliServiceImpl::remove_peer(::google::protobuf::RpcController* controller, PeerId removing_peer; if (removing_peer.parse(request->peer_id()) != 0) { cntl->SetFailed(EINVAL, "Fail to parse peer_id %s", - request->peer_id().c_str()); + request->peer_id().c_str()); return; } - LOG(WARNING) << "Receive RemovePeerRequest to " << node->node_id() - << " from " << cntl->remote_side() - << ", removing " << request->peer_id(); - Closure* remove_peer_done = NewCallback( - remove_peer_returned, cntl, request, response, peers, node, - done_guard.release()); + LOG(WARNING) << "Receive RemovePeerRequest to " << node->node_id() + << " from " << cntl->remote_side() << ", removing " + << request->peer_id(); + Closure* remove_peer_done = + NewCallback(remove_peer_returned, cntl, request, response, peers, node, + done_guard.release()); return node->remove_peer(removing_peer, remove_peer_done); } @@ -150,13 +152,13 @@ void CliServiceImpl::reset_peer(::google::protobuf::RpcController* controller, PeerId peer; if (peer.parse(request->new_peers(i)) != 0) { cntl->SetFailed(EINVAL, "Fail to parse %s", - request->new_peers(i).c_str()); + request->new_peers(i).c_str()); return; } new_peers.add_peer(peer); } - LOG(WARNING) << "Receive set_peer to " << node->node_id() - << " from " << cntl->remote_side(); + LOG(WARNING) << "Receive set_peer to " << node->node_id() << " from " + << cntl->remote_side(); st = node->reset_peers(new_peers); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); @@ -177,7 +179,6 @@ void CliServiceImpl::snapshot(::google::protobuf::RpcController* controller, const ::braft::SnapshotRequest* request, ::braft::SnapshotResponse* response, ::google::protobuf::Closure* done) { - brpc::Controller* cntl = (brpc::Controller*)controller; brpc::ClosureGuard done_guard(done); scoped_refptr node; @@ -186,8 +187,8 @@ void CliServiceImpl::snapshot(::google::protobuf::RpcController* controller, cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); return; } - Closure* snapshot_done = NewCallback(snapshot_returned, cntl, node, - done_guard.release()); + Closure* snapshot_done = + NewCallback(snapshot_returned, cntl, node, done_guard.release()); return node->snapshot(snapshot_done); } @@ -202,11 +203,11 @@ void CliServiceImpl::get_leader(::google::protobuf::RpcController* controller, PeerId peer; if (peer.parse(request->peer_id()) != 0) { cntl->SetFailed(EINVAL, "Fail to parse %s", - request->peer_id().c_str()); + request->peer_id().c_str()); return; } - scoped_refptr node = - global_node_manager->get(request->group_id(), peer); + scoped_refptr node = + global_node_manager->get(request->group_id(), peer); if (node) { nodes.push_back(node); } @@ -215,7 +216,7 @@ void CliServiceImpl::get_leader(::google::protobuf::RpcController* controller, } if (nodes.empty()) { cntl->SetFailed(ENOENT, "No nodes in group %s", - request->group_id().c_str()); + request->group_id().c_str()); return; } @@ -230,46 +231,45 @@ void CliServiceImpl::get_leader(::google::protobuf::RpcController* controller, } butil::Status CliServiceImpl::get_node(scoped_refptr* node, - const GroupId& group_id, - const std::string& peer_id) { + const GroupId& group_id, + const std::string& peer_id) { if (!peer_id.empty()) { *node = global_node_manager->get(group_id, peer_id); if (!(*node)) { return butil::Status(ENOENT, "Fail to find node %s in group %s", - peer_id.c_str(), - group_id.c_str()); + peer_id.c_str(), group_id.c_str()); } } else { std::vector > nodes; global_node_manager->get_nodes_by_group_id(group_id, &nodes); if (nodes.empty()) { return butil::Status(ENOENT, "Fail to find node in group %s", - group_id.c_str()); + group_id.c_str()); } if (nodes.size() > 1) { - return butil::Status(EINVAL, "peer must be specified " - "since there're %lu nodes in group %s", - nodes.size(), group_id.c_str()); + return butil::Status(EINVAL, + "peer must be specified " + "since there're %lu nodes in group %s", + nodes.size(), group_id.c_str()); } *node = nodes.front(); } if ((*node)->disable_cli()) { - return butil::Status(EACCES, "CliService is not allowed to access node " - "%s", (*node)->node_id().to_string().c_str()); + return butil::Status(EACCES, + "CliService is not allowed to access node " + "%s", + (*node)->node_id().to_string().c_str()); } return butil::Status::OK(); } -static void change_peers_returned(brpc::Controller* cntl, - const ChangePeersRequest* request, - ChangePeersResponse* response, - std::vector old_peers, - Configuration new_peers, - scoped_refptr /*node*/, - ::google::protobuf::Closure* done, - const butil::Status& st) { +static void change_peers_returned( + brpc::Controller* cntl, const ChangePeersRequest* request, + ChangePeersResponse* response, std::vector old_peers, + Configuration new_peers, scoped_refptr /*node*/, + ::google::protobuf::Closure* done, const butil::Status& st) { brpc::ClosureGuard done_guard(done); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); @@ -278,8 +278,8 @@ static void change_peers_returned(brpc::Controller* cntl, for (size_t i = 0; i < old_peers.size(); ++i) { response->add_old_peers(old_peers[i].to_string()); } - for (Configuration::const_iterator - iter = new_peers.begin(); iter != new_peers.end(); ++iter) { + for (Configuration::const_iterator iter = new_peers.begin(); + iter != new_peers.end(); ++iter) { response->add_new_peers(iter->to_string()); } } @@ -291,7 +291,8 @@ void CliServiceImpl::change_peers(::google::protobuf::RpcController* controller, brpc::Controller* cntl = (brpc::Controller*)controller; brpc::ClosureGuard done_guard(done); scoped_refptr node; - butil::Status st = get_node(&node, request->group_id(), request->leader_id()); + butil::Status st = + get_node(&node, request->group_id(), request->leader_id()); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); return; @@ -307,27 +308,27 @@ void CliServiceImpl::change_peers(::google::protobuf::RpcController* controller, PeerId peer; if (peer.parse(request->new_peers(i)) != 0) { cntl->SetFailed(EINVAL, "Fail to parse %s", - request->new_peers(i).c_str()); + request->new_peers(i).c_str()); return; } conf.add_peer(peer); } - Closure* change_peers_done = NewCallback( - change_peers_returned, - cntl, request, response, old_peers, conf, node, - done_guard.release()); + Closure* change_peers_done = + NewCallback(change_peers_returned, cntl, request, response, old_peers, + conf, node, done_guard.release()); return node->change_peers(conf, change_peers_done); } void CliServiceImpl::transfer_leader( - ::google::protobuf::RpcController* controller, - const ::braft::TransferLeaderRequest* request, - ::braft::TransferLeaderResponse* response, - ::google::protobuf::Closure* done) { + ::google::protobuf::RpcController* controller, + const ::braft::TransferLeaderRequest* request, + ::braft::TransferLeaderResponse* response, + ::google::protobuf::Closure* done) { brpc::Controller* cntl = (brpc::Controller*)controller; brpc::ClosureGuard done_guard(done); scoped_refptr node; - butil::Status st = get_node(&node, request->group_id(), request->leader_id()); + butil::Status st = + get_node(&node, request->group_id(), request->leader_id()); if (!st.ok()) { cntl->SetFailed(st.error_code(), "%s", st.error_cstr()); return; @@ -340,7 +341,7 @@ void CliServiceImpl::transfer_leader( const int rc = node->transfer_leadership_to(peer); if (rc != 0) { cntl->SetFailed(rc, "Fail to invoke transfer_leadership_to : %s", - berror(rc)); + berror(rc)); return; } } diff --git a/src/braft/cli_service.h b/src/braft/cli_service.h index 4121153..41f64bb 100644 --- a/src/braft/cli_service.h +++ b/src/braft/cli_service.h @@ -1,11 +1,11 @@ // Copyright (c) 2018 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,17 +14,18 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_CLI_SERVICE_H -#define BRAFT_CLI_SERVICE_H +#ifndef BRAFT_CLI_SERVICE_H +#define BRAFT_CLI_SERVICE_H #include -#include "braft/cli.pb.h" // CliService -#include "braft/node.h" // NodeImpl + +#include "braft/cli.pb.h" // CliService +#include "braft/node.h" // NodeImpl namespace braft { class CliServiceImpl : public CliService { -public: + public: void add_peer(::google::protobuf::RpcController* controller, const ::braft::AddPeerRequest* request, ::braft::AddPeerResponse* response, @@ -53,12 +54,12 @@ class CliServiceImpl : public CliService { const ::braft::TransferLeaderRequest* request, ::braft::TransferLeaderResponse* response, ::google::protobuf::Closure* done); -private: + + private: butil::Status get_node(scoped_refptr* node, - const GroupId& group_id, - const std::string& peer_id); + const GroupId& group_id, const std::string& peer_id); }; -} +} // namespace braft -#endif //BRAFT_CLI_SERVICE_H +#endif // BRAFT_CLI_SERVICE_H diff --git a/src/braft/closure_helper.h b/src/braft/closure_helper.h index d80daf8..c667821 100644 --- a/src/braft/closure_helper.h +++ b/src/braft/closure_helper.h @@ -43,7 +43,7 @@ #include #endif -#include // braft::Closure +#include // braft::Closure namespace braft { @@ -111,682 +111,873 @@ namespace braft { namespace internal { class LIBPROTOBUF_EXPORT FunctionClosure0 : public Closure { - public: - typedef void (*FunctionType)(const butil::Status &status); - - FunctionClosure0(FunctionType function, bool self_deleting) - : function_(function), self_deleting_(self_deleting) {} - ~FunctionClosure0(); - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; + public: + typedef void (*FunctionType)(const butil::Status& status); + + FunctionClosure0(FunctionType function, bool self_deleting) + : function_(function), self_deleting_(self_deleting) {} + ~FunctionClosure0(); + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; }; template class MethodClosure0 : public Closure { - public: - typedef void (Class::*MethodType)(const butil::Status &status); - - MethodClosure0(Class* object, MethodType method, bool self_deleting) - : object_(object), method_(method), self_deleting_(self_deleting) {} - ~MethodClosure0() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; + public: + typedef void (Class::*MethodType)(const butil::Status& status); + + MethodClosure0(Class* object, MethodType method, bool self_deleting) + : object_(object), method_(method), self_deleting_(self_deleting) {} + ~MethodClosure0() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; }; template class FunctionClosure1 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, const butil::Status &status); - - FunctionClosure1(FunctionType function, bool self_deleting, - Arg1 arg1) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1) {} - ~FunctionClosure1() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; + public: + typedef void (*FunctionType)(Arg1 arg1, const butil::Status& status); + + FunctionClosure1(FunctionType function, bool self_deleting, Arg1 arg1) + : function_(function), self_deleting_(self_deleting), arg1_(arg1) {} + ~FunctionClosure1() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; }; template class MethodClosure1 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, const butil::Status &status); - - MethodClosure1(Class* object, MethodType method, bool self_deleting, - Arg1 arg1) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1) {} - ~MethodClosure1() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, const butil::Status& status); + + MethodClosure1(Class* object, MethodType method, bool self_deleting, + Arg1 arg1) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1) {} + ~MethodClosure1() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; }; template class FunctionClosure2 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, const butil::Status &status); - - FunctionClosure2(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2) {} - ~FunctionClosure2() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, + const butil::Status& status); + + FunctionClosure2(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2) {} + ~FunctionClosure2() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; }; template class MethodClosure2 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, const butil::Status &status); - - MethodClosure2(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2) {} - ~MethodClosure2() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, + const butil::Status& status); + + MethodClosure2(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2) {} + ~MethodClosure2() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; }; template class FunctionClosure3 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, const butil::Status &status); - - FunctionClosure3(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3) {} - ~FunctionClosure3() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + const butil::Status& status); + + FunctionClosure3(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3) {} + ~FunctionClosure3() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; }; template class MethodClosure3 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, const butil::Status &status); - - MethodClosure3(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3) {} - ~MethodClosure3() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + const butil::Status& status); + + MethodClosure3(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3) {} + ~MethodClosure3() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; }; template class FunctionClosure4 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, const butil::Status &status); - - FunctionClosure4(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4) {} - ~FunctionClosure4() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + const butil::Status& status); + + FunctionClosure4(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4) {} + ~FunctionClosure4() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; }; -template +template class MethodClosure4 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, const butil::Status &status); - - MethodClosure4(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4) {} - ~MethodClosure4() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, const butil::Status& status); + + MethodClosure4(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4) {} + ~MethodClosure4() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; }; -template +template class FunctionClosure5 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, const butil::Status &status); - - FunctionClosure5(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4), arg5_(arg5) {} - ~FunctionClosure5() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, arg5_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, const butil::Status& status); + + FunctionClosure5(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5) {} + ~FunctionClosure5() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, arg5_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; }; -template +template class MethodClosure5 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, const butil::Status &status); - - MethodClosure5(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4),arg5_(arg5) {} - ~MethodClosure5() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_,arg5_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, Arg5 arg5, + const butil::Status& status); + + MethodClosure5(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5) {} + ~MethodClosure5() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, arg5_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; }; -template +template class FunctionClosure6 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, const butil::Status &status); - - FunctionClosure6(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4), arg5_(arg5), arg6_(arg6) {} - ~FunctionClosure6() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, + const butil::Status& status); + + FunctionClosure6(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6) {} + ~FunctionClosure6() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; }; -template +template class MethodClosure6 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, const butil::Status &status); - - MethodClosure6(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4),arg5_(arg5),arg6_(arg6) {} - ~MethodClosure6() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_,arg5_,arg6_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, Arg5 arg5, Arg6 arg6, + const butil::Status& status); + + MethodClosure6(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, + Arg6 arg6) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6) {} + ~MethodClosure6() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; }; -template +template class FunctionClosure7 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, const butil::Status &status); - - FunctionClosure7(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4), arg5_(arg5), arg6_(arg6), arg7_(arg7) {} - ~FunctionClosure7() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, + const butil::Status& status); + + FunctionClosure7(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7) {} + ~FunctionClosure7() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; }; -template +template class MethodClosure7 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, const butil::Status &status); - - MethodClosure7(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4),arg5_(arg5),arg6_(arg6),arg7_(arg7) {} - ~MethodClosure7() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_,arg5_,arg6_,arg7_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, const butil::Status& status); + + MethodClosure7(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, + Arg6 arg6, Arg7 arg7) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7) {} + ~MethodClosure7() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, + status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; }; - -template +template class FunctionClosure8 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, const butil::Status &status); - - FunctionClosure8(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4), arg5_(arg5), arg6_(arg6), arg7_(arg7), arg8_(arg8) {} - ~FunctionClosure8() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_,arg8_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; - Arg8 arg8_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + const butil::Status& status); + + FunctionClosure8(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, Arg8 arg8) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7), + arg8_(arg8) {} + ~FunctionClosure8() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, arg8_, + status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; + Arg8 arg8_; }; -template +template class MethodClosure8 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, const butil::Status &status); - - MethodClosure8(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4),arg5_(arg5),arg6_(arg6),arg7_(arg7),arg8_(arg8) {} - ~MethodClosure8() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_,arg5_,arg6_,arg7_,arg8_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; - Arg8 arg8_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, Arg8 arg8, + const butil::Status& status); + + MethodClosure8(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, + Arg6 arg6, Arg7 arg7, Arg8 arg8) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7), + arg8_(arg8) {} + ~MethodClosure8() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, + arg8_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; + Arg8 arg8_; }; - -template +template class FunctionClosure9 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, const butil::Status &status); - - FunctionClosure9(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4), arg5_(arg5), arg6_(arg6), arg7_(arg7), arg8_(arg8), arg9_(arg9) {} - ~FunctionClosure9() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_,arg8_,arg9_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; - Arg8 arg8_; - Arg9 arg9_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + Arg9 arg9, const butil::Status& status); + + FunctionClosure9(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, Arg8 arg8, Arg9 arg9) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7), + arg8_(arg8), + arg9_(arg9) {} + ~FunctionClosure9() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, arg8_, arg9_, + status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; + Arg8 arg8_; + Arg9 arg9_; }; -template +template class MethodClosure9 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, const butil::Status &status); - - MethodClosure9(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4),arg5_(arg5),arg6_(arg6),arg7_(arg7),arg8_(arg8),arg9_(arg9) {} - ~MethodClosure9() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_,arg5_,arg6_,arg7_,arg8_,arg9_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; - Arg8 arg8_; - Arg9 arg9_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, Arg8 arg8, Arg9 arg9, + const butil::Status& status); + + MethodClosure9(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, + Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7), + arg8_(arg8), + arg9_(arg9) {} + ~MethodClosure9() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, + arg8_, arg9_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; + Arg8 arg8_; + Arg9 arg9_; }; -template +template class FunctionClosure10 : public Closure { - public: - typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10, const butil::Status &status); - - FunctionClosure10(FunctionType function, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) - : function_(function), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2), arg3_(arg3), arg4_(arg4), arg5_(arg5), arg6_(arg6), arg7_(arg7), arg8_(arg8), arg9_(arg9), arg10_(arg10) {} - ~FunctionClosure10() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_,arg8_,arg9_,arg10_, status()); - if (needs_delete) delete this; - } - - private: - FunctionType function_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; - Arg8 arg8_; - Arg9 arg9_; - Arg10 arg10_; + public: + typedef void (*FunctionType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + Arg9 arg9, Arg10 arg10, + const butil::Status& status); + + FunctionClosure10(FunctionType function, bool self_deleting, Arg1 arg1, + Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) + : function_(function), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7), + arg8_(arg8), + arg9_(arg9), + arg10_(arg10) {} + ~FunctionClosure10() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + function_(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, arg8_, arg9_, + arg10_, status()); + if (needs_delete) delete this; + } + + private: + FunctionType function_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; + Arg8 arg8_; + Arg9 arg9_; + Arg10 arg10_; }; -template +template class MethodClosure10 : public Closure { - public: - typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10, const butil::Status &status); - - MethodClosure10(Class* object, MethodType method, bool self_deleting, - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) - : object_(object), method_(method), self_deleting_(self_deleting), - arg1_(arg1), arg2_(arg2),arg3_(arg3),arg4_(arg4),arg5_(arg5),arg6_(arg6),arg7_(arg7),arg8_(arg8),arg9_(arg9),arg10_(arg10) {} - ~MethodClosure10() {} - - void Run() { - bool needs_delete = self_deleting_; // read in case callback deletes - (object_->*method_)(arg1_, arg2_,arg3_,arg4_,arg5_,arg6_,arg7_,arg8_,arg9_,arg10_, status()); - if (needs_delete) delete this; - } - - private: - Class* object_; - MethodType method_; - bool self_deleting_; - Arg1 arg1_; - Arg2 arg2_; - Arg3 arg3_; - Arg4 arg4_; - Arg5 arg5_; - Arg6 arg6_; - Arg7 arg7_; - Arg8 arg8_; - Arg9 arg9_; - Arg10 arg10_; + public: + typedef void (Class::*MethodType)(Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7, Arg8 arg8, Arg9 arg9, + Arg10 arg10, const butil::Status& status); + + MethodClosure10(Class* object, MethodType method, bool self_deleting, + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, + Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) + : object_(object), + method_(method), + self_deleting_(self_deleting), + arg1_(arg1), + arg2_(arg2), + arg3_(arg3), + arg4_(arg4), + arg5_(arg5), + arg6_(arg6), + arg7_(arg7), + arg8_(arg8), + arg9_(arg9), + arg10_(arg10) {} + ~MethodClosure10() {} + + void Run() { + bool needs_delete = self_deleting_; // read in case callback deletes + (object_->*method_)(arg1_, arg2_, arg3_, arg4_, arg5_, arg6_, arg7_, + arg8_, arg9_, arg10_, status()); + if (needs_delete) delete this; + } + + private: + Class* object_; + MethodType method_; + bool self_deleting_; + Arg1 arg1_; + Arg2 arg2_; + Arg3 arg3_; + Arg4 arg4_; + Arg5 arg5_; + Arg6 arg6_; + Arg7 arg7_; + Arg8 arg8_; + Arg9 arg9_; + Arg10 arg10_; }; - } // namespace internal // See Closure. inline Closure* NewCallback(void (*function)(const butil::Status&)) { - return new internal::FunctionClosure0(function, true); + return new internal::FunctionClosure0(function, true); } // See Closure. inline Closure* NewPermanentCallback(void (*function)(const butil::Status&)) { - return new internal::FunctionClosure0(function, false); + return new internal::FunctionClosure0(function, false); } // See Closure. template -inline Closure* NewCallback(Class* object, void (Class::*method)(const butil::Status&)) { - return new internal::MethodClosure0(object, method, true); +inline Closure* NewCallback(Class* object, + void (Class::*method)(const butil::Status&)) { + return new internal::MethodClosure0(object, method, true); } // See Closure. template -inline Closure* NewPermanentCallback(Class* object, void (Class::*method)(const butil::Status&)) { - return new internal::MethodClosure0(object, method, false); +inline Closure* NewPermanentCallback( + Class* object, void (Class::*method)(const butil::Status&)) { + return new internal::MethodClosure0(object, method, false); } // See Closure. template inline Closure* NewCallback(void (*function)(Arg1, const butil::Status&), Arg1 arg1) { - return new internal::FunctionClosure1(function, true, arg1); + return new internal::FunctionClosure1(function, true, arg1); } // See Closure. template -inline Closure* NewPermanentCallback(void (*function)(Arg1, const butil::Status&), +inline Closure* NewPermanentCallback(void (*function)(Arg1, + const butil::Status&), Arg1 arg1) { - return new internal::FunctionClosure1(function, false, arg1); + return new internal::FunctionClosure1(function, false, arg1); } // See Closure. template -inline Closure* NewCallback(Class* object, void (Class::*method)(Arg1, const butil::Status&), +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, const butil::Status&), Arg1 arg1) { - return new internal::MethodClosure1(object, method, true, arg1); + return new internal::MethodClosure1(object, method, true, + arg1); } // See Closure. template -inline Closure* NewPermanentCallback(Class* object, void (Class::*method)(Arg1, const butil::Status&), - Arg1 arg1) { - return new internal::MethodClosure1(object, method, false, arg1); +inline Closure* NewPermanentCallback( + Class* object, void (Class::*method)(Arg1, const butil::Status&), + Arg1 arg1) { + return new internal::MethodClosure1(object, method, false, + arg1); } // See Closure. template inline Closure* NewCallback(void (*function)(Arg1, Arg2, const butil::Status&), Arg1 arg1, Arg2 arg2) { - return new internal::FunctionClosure2( - function, true, arg1, arg2); + return new internal::FunctionClosure2(function, true, arg1, + arg2); } // See Closure. template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, const butil::Status&), +inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, + const butil::Status&), Arg1 arg1, Arg2 arg2) { - return new internal::FunctionClosure2( - function, false, arg1, arg2); + return new internal::FunctionClosure2(function, false, arg1, + arg2); } // See Closure. template -inline Closure* NewCallback(Class* object, void (Class::*method)(Arg1, Arg2, const butil::Status&), +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, + const butil::Status&), Arg1 arg1, Arg2 arg2) { - return new internal::MethodClosure2( - object, method, true, arg1, arg2); + return new internal::MethodClosure2(object, method, true, + arg1, arg2); } // See Closure. @@ -794,280 +985,413 @@ template inline Closure* NewPermanentCallback( Class* object, void (Class::*method)(Arg1, Arg2, const butil::Status&), Arg1 arg1, Arg2 arg2) { - return new internal::MethodClosure2( - object, method, false, arg1, arg2); + return new internal::MethodClosure2(object, method, + false, arg1, arg2); } // See Closure. template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, const butil::Status&), +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, + const butil::Status&), Arg1 arg1, Arg2 arg2, Arg3 arg3) { - return new internal::FunctionClosure3( - function, true, arg1, arg2, arg3); + return new internal::FunctionClosure3(function, true, + arg1, arg2, arg3); } // See Closure. template - inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3) { - return new internal::FunctionClosure3( - function, false, arg1, arg2, arg3); +inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3) { + return new internal::FunctionClosure3(function, false, + arg1, arg2, arg3); } // See Closure template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3) { - return new internal::MethodClosure3( - object, method, true, arg1, arg2, arg3); +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3) { + return new internal::MethodClosure3( + object, method, true, arg1, arg2, arg3); } // See Closure template - inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3) { - return new internal::MethodClosure3( - object, method, false, arg1, arg2, arg3); +inline Closure* NewPermanentCallback( + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, const butil::Status&), Arg1 arg1, + Arg2 arg2, Arg3 arg3) { + return new internal::MethodClosure3( + object, method, false, arg1, arg2, arg3); } // See Closure. template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, const butil::Status&), +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, + const butil::Status&), Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) { - return new internal::FunctionClosure4( - function, true, arg1, arg2, arg3, arg4); + return new internal::FunctionClosure4( + function, true, arg1, arg2, arg3, arg4); } // See Closure. template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) { - return new internal::FunctionClosure4( - function, false, arg1, arg2, arg3, arg4); +inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, + Arg4 arg4) { + return new internal::FunctionClosure4( + function, false, arg1, arg2, arg3, arg4); } // See Closure. -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) { - return new internal::MethodClosure4( - object, method, true, arg1, arg2, arg3, arg4); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) { + return new internal::MethodClosure4( + object, method, true, arg1, arg2, arg3, arg4); } // See Closure. -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, const butil::Status&), + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, const butil::Status&), Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4) { - return new internal::MethodClosure4( - object, method, false, arg1, arg2, arg3, arg4); + return new internal::MethodClosure4( + object, method, false, arg1, arg2, arg3, arg4); } // See Closure. -template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) { - return new internal::FunctionClosure5( - function, true, arg1, arg2, arg3, arg4, arg5); +template +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5) { + return new internal::FunctionClosure5( + function, true, arg1, arg2, arg3, arg4, arg5); } // See Closure. -template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) { - return new internal::FunctionClosure5( - function, false, arg1, arg2, arg3, arg4, arg5); +template +inline Closure* NewPermanentCallback( + void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) { + return new internal::FunctionClosure5( + function, false, arg1, arg2, arg3, arg4, arg5); } // See Closure -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) { - return new internal::MethodClosure5( - object, method, true, arg1, arg2, arg3, arg4, arg5); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5) { + return new internal::MethodClosure5( + object, method, true, arg1, arg2, arg3, arg4, arg5); } // See Closure -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, const butil::Status&), + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, const butil::Status&), Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5) { - return new internal::MethodClosure5( - object, method, false, arg1, arg2, arg3, arg4, arg5); + return new internal::MethodClosure5( + object, method, false, arg1, arg2, arg3, arg4, arg5); } // See Closure. -template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) { - return new internal::FunctionClosure6( - function, true, arg1, arg2, arg3, arg4, arg5, arg6); +template +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6) { + return new internal::FunctionClosure6( + function, true, arg1, arg2, arg3, arg4, arg5, arg6); } // See Closure. -template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) { - return new internal::FunctionClosure6( - function, false, arg1, arg2, arg3, arg4, arg5, arg6); +template +inline Closure* NewPermanentCallback( + void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) { + return new internal::FunctionClosure6( + function, false, arg1, arg2, arg3, arg4, arg5, arg6); } // See Closure -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) { - return new internal::MethodClosure6( - object, method, true, arg1, arg2, arg3, arg4, arg5, arg6); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, + Arg6, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6) { + return new internal::MethodClosure6(object, method, true, arg1, arg2, + arg3, arg4, arg5, arg6); } // See Closure -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, const butil::Status&), + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, + const butil::Status&), Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6) { - return new internal::MethodClosure6( - object, method, false, arg1, arg2, arg3, arg4, arg5, arg6); + return new internal::MethodClosure6(object, method, false, arg1, arg2, + arg3, arg4, arg5, arg6); } // See Closure -template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7) { - return new internal::FunctionClosure7( - function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7); +template +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, + Arg7, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7) { + return new internal::FunctionClosure7(function, true, arg1, arg2, + arg3, arg4, arg5, arg6, arg7); } // See Closure. -template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7) { - return new internal::FunctionClosure7( - function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7); +template +inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, + Arg5, Arg6, Arg7, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7) { + return new internal::FunctionClosure7(function, false, arg1, arg2, + arg3, arg4, arg5, arg6, arg7); } // See Closure -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7) { - return new internal::MethodClosure7( - object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, + Arg6, Arg7, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7) { + return new internal::MethodClosure7( + object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7); } // See Closure -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7) { - return new internal::MethodClosure7( - object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7); + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, + Arg7 arg7) { + return new internal::MethodClosure7( + object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7); } // See Closure. -template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) { - return new internal::FunctionClosure8( - function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); +template +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, + Arg7, Arg8, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) { + return new internal::FunctionClosure8( + function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); } // See Closure. -template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) { - return new internal::FunctionClosure8( - function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); +template +inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, + Arg5, Arg6, Arg7, Arg8, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, + Arg8 arg8) { + return new internal::FunctionClosure8( + function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); } // See Closure -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) { - return new internal::MethodClosure8( - object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, + Arg6, Arg7, Arg8, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) { + return new internal::MethodClosure8( + object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); } // See Closure -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8) { - return new internal::MethodClosure8( - object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, + Arg8 arg8) { + return new internal::MethodClosure8( + object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); } // See Closure. -template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) { - return new internal::FunctionClosure9( - function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9); +template +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, + Arg7, Arg8, Arg9, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + Arg9 arg9) { + return new internal::FunctionClosure9( + function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9); } // See Closure. -template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) { - return new internal::FunctionClosure9( - function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9); +template +inline Closure* NewPermanentCallback( + void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, + Arg8 arg8, Arg9 arg9) { + return new internal::FunctionClosure9( + function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9); } // See Closure -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) { - return new internal::MethodClosure9( - object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, + Arg6, Arg7, Arg8, Arg9, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + Arg9 arg9) { + return new internal::MethodClosure9( + object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, + arg9); } // See Closure -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9) { - return new internal::MethodClosure9( - object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9); + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, + Arg8 arg8, Arg9 arg9) { + return new internal::MethodClosure9( + object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, + arg9); } // See Closure. -template -inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, Arg10, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) { - return new internal::FunctionClosure10( - function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10); +template +inline Closure* NewCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, + Arg7, Arg8, Arg9, Arg10, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + Arg9 arg9, Arg10 arg10) { + return new internal::FunctionClosure10( + function, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, + arg10); } // See Closure. -template -inline Closure* NewPermanentCallback(void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, Arg10, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) { - return new internal::FunctionClosure10( - function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10); +template +inline Closure* NewPermanentCallback( + void (*function)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, + Arg10, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, + Arg8 arg8, Arg9 arg9, Arg10 arg10) { + return new internal::FunctionClosure10( + function, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, + arg10); } // See Closure -template -inline Closure* NewCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, Arg10, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) { - return new internal::MethodClosure10( - object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10); +template +inline Closure* NewCallback(Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, + Arg6, Arg7, Arg8, Arg9, Arg10, + const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, + Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, + Arg9 arg9, Arg10 arg10) { + return new internal::MethodClosure10( + object, method, true, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, + arg9, arg10); } // See Closure -template +template inline Closure* NewPermanentCallback( - Class* object, void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, Arg10, const butil::Status&), - Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, Arg8 arg8, Arg9 arg9, Arg10 arg10) { - return new internal::MethodClosure10( - object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10); + Class* object, + void (Class::*method)(Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9, + Arg10, const butil::Status&), + Arg1 arg1, Arg2 arg2, Arg3 arg3, Arg4 arg4, Arg5 arg5, Arg6 arg6, Arg7 arg7, + Arg8 arg8, Arg9 arg9, Arg10 arg10) { + return new internal::MethodClosure10( + object, method, false, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, + arg9, arg10); } } // namespace braft diff --git a/src/braft/closure_queue.cpp b/src/braft/closure_queue.cpp index 0d7d225..9e157a5 100644 --- a/src/braft/closure_queue.cpp +++ b/src/braft/closure_queue.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,20 +14,18 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#include #include "braft/closure_queue.h" + +#include + #include "braft/raft.h" namespace braft { -ClosureQueue::ClosureQueue(bool usercode_in_pthread) - : _first_index(0) - , _usercode_in_pthread(usercode_in_pthread) -{} +ClosureQueue::ClosureQueue(bool usercode_in_pthread) + : _first_index(0), _usercode_in_pthread(usercode_in_pthread) {} -ClosureQueue::~ClosureQueue() { - clear(); -} +ClosureQueue::~ClosureQueue() { clear(); } void ClosureQueue::clear() { std::deque saved_queue; @@ -37,8 +35,8 @@ void ClosureQueue::clear() { _first_index = 0; } bool run_bthread = false; - for (std::deque::iterator - it = saved_queue.begin(); it != saved_queue.end(); ++it) { + for (std::deque::iterator it = saved_queue.begin(); + it != saved_queue.end(); ++it) { if (*it) { (*it)->status().set_error(EPERM, "leader stepped down"); run_closure_in_bthread_nosig(*it, _usercode_in_pthread); @@ -61,8 +59,8 @@ void ClosureQueue::append_pending_closure(Closure* c) { _queue.push_back(c); } -int ClosureQueue::pop_closure_until(int64_t index, - std::vector *out, int64_t *out_first_index) { +int ClosureQueue::pop_closure_until(int64_t index, std::vector* out, + int64_t* out_first_index) { out->clear(); BAIDU_SCOPED_LOCK(_mutex); if (_queue.empty() || index < _first_index) { @@ -84,4 +82,4 @@ int ClosureQueue::pop_closure_until(int64_t index, return 0; } -} // namespace braft +} // namespace braft diff --git a/src/braft/closure_queue.h b/src/braft/closure_queue.h index 761364e..a1a8ac3 100644 --- a/src/braft/closure_queue.h +++ b/src/braft/closure_queue.h @@ -14,8 +14,8 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_CLOSURE_QUEUE_H -#define BRAFT_CLOSURE_QUEUE_H +#ifndef BRAFT_CLOSURE_QUEUE_H +#define BRAFT_CLOSURE_QUEUE_H #include "braft/util.h" @@ -23,7 +23,7 @@ namespace braft { // Holding the closure waiting for the commitment of logs class ClosureQueue { -public: + public: explicit ClosureQueue(bool usercode_in_pthread); ~ClosureQueue(); @@ -40,19 +40,19 @@ class ClosureQueue { void append_pending_closure(Closure* c); // Pop all the closure until |index| (included) into out in the same order - // of their indexes, |out_first_index| would be assigned the index of out[0] if - // out is not empty, index + 1 otherwise. - int pop_closure_until(int64_t index, - std::vector *out, int64_t *out_first_index); -private: - // TODO: a spsc lock-free queue would help - raft_mutex_t _mutex; - int64_t _first_index; - std::deque _queue; - bool _usercode_in_pthread; + // of their indexes, |out_first_index| would be assigned the index of out[0] + // if out is not empty, index + 1 otherwise. + int pop_closure_until(int64_t index, std::vector* out, + int64_t* out_first_index); + private: + // TODO: a spsc lock-free queue would help + raft_mutex_t _mutex; + int64_t _first_index; + std::deque _queue; + bool _usercode_in_pthread; }; -} // namespace braft +} // namespace braft -#endif //BRAFT_CLOSURE_QUEUE_H +#endif // BRAFT_CLOSURE_QUEUE_H diff --git a/src/braft/configuration.cpp b/src/braft/configuration.cpp index 29c8218..95e9d04 100644 --- a/src/braft/configuration.cpp +++ b/src/braft/configuration.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +16,7 @@ // Zhangyi Chen(chenzhangyi01@baidu.com) #include "braft/configuration.h" + #include #include @@ -48,4 +49,4 @@ int Configuration::parse_from(butil::StringPiece conf) { return 0; } -} +} // namespace braft diff --git a/src/braft/configuration.h b/src/braft/configuration.h index 310539a..e673008 100644 --- a/src/braft/configuration.h +++ b/src/braft/configuration.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,14 +19,15 @@ #ifndef BRAFT_RAFT_CONFIGURATION_H #define BRAFT_RAFT_CONFIGURATION_H -#include -#include -#include -#include -#include -#include #include #include +#include + +#include +#include +#include +#include +#include namespace braft { @@ -41,21 +42,25 @@ enum Role { // Represent a participant in a replicating group. struct PeerId { - butil::EndPoint addr; // ip+port. - int idx; // idx in same addr, default 0 + butil::EndPoint addr; // ip+port. + int idx; // idx in same addr, default 0 Role role = REPLICA; PeerId() : idx(0), role(REPLICA) {} - explicit PeerId(butil::EndPoint addr_) : addr(addr_), idx(0), role(REPLICA) {} - PeerId(butil::EndPoint addr_, int idx_) : addr(addr_), idx(idx_), role(REPLICA) {} - PeerId(butil::EndPoint addr_, int idx_, bool witness) : addr(addr_), idx(idx_) { + explicit PeerId(butil::EndPoint addr_) + : addr(addr_), idx(0), role(REPLICA) {} + PeerId(butil::EndPoint addr_, int idx_) + : addr(addr_), idx(idx_), role(REPLICA) {} + PeerId(butil::EndPoint addr_, int idx_, bool witness) + : addr(addr_), idx(idx_) { if (witness) { this->role = WITNESS; - } + } } - /*intended implicit*/PeerId(const std::string& str) - { CHECK_EQ(0, parse(str)); } + /*intended implicit*/ PeerId(const std::string& str) { + CHECK_EQ(0, parse(str)); + } PeerId(const PeerId& id) : addr(id.addr), idx(id.idx), role(id.role) {} void reset() { @@ -68,14 +73,13 @@ struct PeerId { bool is_empty() const { return (addr.ip == butil::IP_ANY && addr.port == 0 && idx == 0); } - bool is_witness() const { - return role == WITNESS; - } + bool is_witness() const { return role == WITNESS; } int parse(const std::string& str) { reset(); char ip_str[64]; int value = REPLICA; - if (2 > sscanf(str.c_str(), "%[^:]%*[:]%d%*[:]%d%*[:]%d", ip_str, &addr.port, &idx, &value)) { + if (2 > sscanf(str.c_str(), "%[^:]%*[:]%d%*[:]%d%*[:]%d", ip_str, + &addr.port, &idx, &value)) { reset(); return -1; } @@ -93,10 +97,11 @@ struct PeerId { std::string to_string() const { char str[128]; - snprintf(str, sizeof(str), "%s:%d:%d", butil::endpoint2str(addr).c_str(), idx, int(role)); + snprintf(str, sizeof(str), "%s:%d:%d", + butil::endpoint2str(addr).c_str(), idx, int(role)); return std::string(str); } - + PeerId& operator=(const PeerId& rhs) = default; }; @@ -116,7 +121,7 @@ inline bool operator!=(const PeerId& id1, const PeerId& id2) { return !(id1 == id2); } -inline std::ostream& operator << (std::ostream& os, const PeerId& id) { +inline std::ostream& operator<<(std::ostream& os, const PeerId& id) { return os << id.addr << ':' << id.idx << ':' << int(id.role); } @@ -125,8 +130,7 @@ struct NodeId { PeerId peer_id; NodeId(const GroupId& group_id_, const PeerId& peer_id_) - : group_id(group_id_), peer_id(peer_id_) { - } + : group_id(group_id_), peer_id(peer_id_) {} std::string to_string() const; }; @@ -147,7 +151,7 @@ inline bool operator!=(const NodeId& id1, const NodeId& id2) { return (id1.group_id != id2.group_id || id1.peer_id != id2.peer_id); } -inline std::ostream& operator << (std::ostream& os, const NodeId& id) { +inline std::ostream& operator<<(std::ostream& os, const NodeId& id) { return os << id.group_id << ':' << id.peer_id; } @@ -159,7 +163,7 @@ inline std::string NodeId::to_string() const { // A set of peers. class Configuration { -public: + public: typedef std::set::const_iterator const_iterator; // Construct an empty configuration. Configuration() {} @@ -183,9 +187,7 @@ class Configuration { } // Assign from peers stored in std::set - void operator=(const std::set& peers) { - _peers = peers; - } + void operator=(const std::set& peers) { _peers = peers; } // Remove all peers. void reset() { _peers.clear(); } @@ -196,7 +198,7 @@ class Configuration { const_iterator begin() const { return _peers.begin(); } const_iterator end() const { return _peers.end(); } - // Clear the container and put peers in. + // Clear the container and put peers in. void list_peers(std::set* peers) const { peers->clear(); *peers = _peers; @@ -216,15 +218,11 @@ class Configuration { // Add a peer. // Returns true if the peer is newly added. - bool add_peer(const PeerId& peer) { - return _peers.insert(peer).second; - } + bool add_peer(const PeerId& peer) { return _peers.insert(peer).second; } // Remove a peer. // Returns true if the peer is removed. - bool remove_peer(const PeerId& peer) { - return _peers.erase(peer); - } + bool remove_peer(const PeerId& peer) { return _peers.erase(peer); } // True if the peer exists. bool contains(const PeerId& peer_id) const { @@ -266,21 +264,20 @@ class Configuration { } return true; } - + // Get the difference between |*this| and |rhs| // |included| would be assigned to |*this| - |rhs| // |excluded| would be assigned to |rhs| - |*this| - void diffs(const Configuration& rhs, - Configuration* included, + void diffs(const Configuration& rhs, Configuration* included, Configuration* excluded) const { *included = *this; *excluded = rhs; - for (std::set::const_iterator - iter = _peers.begin(); iter != _peers.end(); ++iter) { + for (std::set::const_iterator iter = _peers.begin(); + iter != _peers.end(); ++iter) { excluded->_peers.erase(*iter); } - for (std::set::const_iterator - iter = rhs._peers.begin(); iter != rhs._peers.end(); ++iter) { + for (std::set::const_iterator iter = rhs._peers.begin(); + iter != rhs._peers.end(); ++iter) { included->_peers.erase(*iter); } } @@ -288,14 +285,13 @@ class Configuration { // Parse Configuration from a string into |this| // Returns 0 on success, -1 otherwise int parse_from(butil::StringPiece conf); - -private: - std::set _peers; + private: + std::set _peers; }; std::ostream& operator<<(std::ostream& os, const Configuration& a); } // namespace braft -#endif //~BRAFT_RAFT_CONFIGURATION_H +#endif //~BRAFT_RAFT_CONFIGURATION_H diff --git a/src/braft/configuration_manager.cpp b/src/braft/configuration_manager.cpp index f0322df..f92fce3 100644 --- a/src/braft/configuration_manager.cpp +++ b/src/braft/configuration_manager.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,15 +31,15 @@ int ConfigurationManager::add(const ConfigurationEntry& entry) { } void ConfigurationManager::truncate_prefix(const int64_t first_index_kept) { - while (!_configurations.empty() - && _configurations.front().id.index < first_index_kept) { + while (!_configurations.empty() && + _configurations.front().id.index < first_index_kept) { _configurations.pop_front(); } } void ConfigurationManager::truncate_suffix(const int64_t last_index_kept) { - while (!_configurations.empty() - && _configurations.back().id.index > last_index_kept) { + while (!_configurations.empty() && + _configurations.back().id.index > last_index_kept) { _configurations.pop_back(); } } diff --git a/src/braft/configuration_manager.h b/src/braft/configuration_manager.h index 78b9358..f53c5c1 100644 --- a/src/braft/configuration_manager.h +++ b/src/braft/configuration_manager.h @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,11 +14,11 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_CONFIGURATION_MANAGER_H -#define BRAFT_CONFIGURATION_MANAGER_H +#ifndef BRAFT_CONFIGURATION_MANAGER_H +#define BRAFT_CONFIGURATION_MANAGER_H -#include "braft/configuration.h" // Configuration -#include "braft/log_entry.h" // LogId +#include "braft/configuration.h" // Configuration +#include "braft/log_entry.h" // LogId namespace braft { @@ -43,13 +43,14 @@ struct ConfigurationEntry { conf.append_peers(peers); old_conf.append_peers(peers); } - bool contains(const PeerId& peer) const - { return conf.contains(peer) || old_conf.contains(peer); } + bool contains(const PeerId& peer) const { + return conf.contains(peer) || old_conf.contains(peer); + } }; // Manager the history of configuration changing class ConfigurationManager { -public: + public: ConfigurationManager() {} ~ConfigurationManager() {} @@ -68,12 +69,11 @@ class ConfigurationManager { const ConfigurationEntry& last_configuration() const; -private: - + private: std::deque _configurations; ConfigurationEntry _snapshot; }; } // namespace braft -#endif //BRAFT_CONFIGURATION_MANAGER_H +#endif // BRAFT_CONFIGURATION_MANAGER_H diff --git a/src/braft/file_reader.cpp b/src/braft/file_reader.cpp index 367b5e4..cf26dbb 100644 --- a/src/braft/file_reader.cpp +++ b/src/braft/file_reader.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ // Yang,Guodong(yangguodong01@baidu.com) #include "braft/file_reader.h" + #include "braft/util.h" namespace braft { @@ -31,31 +32,25 @@ LocalDirReader::~LocalDirReader() { _fs->close_snapshot(_path); } -bool LocalDirReader::open() { - return _fs->open_snapshot(_path); -} +bool LocalDirReader::open() { return _fs->open_snapshot(_path); } -int LocalDirReader::read_file(butil::IOBuf* out, - const std::string &filename, - off_t offset, - size_t max_count, - bool read_partly, - size_t* read_count, - bool* is_eof) const { - return read_file_with_meta(out, filename, NULL, offset, - max_count, read_count, is_eof); +int LocalDirReader::read_file(butil::IOBuf* out, const std::string& filename, + off_t offset, size_t max_count, bool read_partly, + size_t* read_count, bool* is_eof) const { + return read_file_with_meta(out, filename, NULL, offset, max_count, + read_count, is_eof); } int LocalDirReader::read_file_with_meta(butil::IOBuf* out, - const std::string &filename, + const std::string& filename, google::protobuf::Message* file_meta, - off_t offset, - size_t max_count, + off_t offset, size_t max_count, size_t* read_count, bool* is_eof) const { std::unique_lock lck(_mutex); if (_is_reading) { - // Just let follower retry, if there already a reading request in process. + // Just let follower retry, if there already a reading request in + // process. lck.unlock(); BRAFT_VLOG << "A courrent read file is in process, path: " << _path; return EAGAIN; @@ -65,8 +60,8 @@ int LocalDirReader::read_file_with_meta(butil::IOBuf* out, if (!_eof_reached || offset != 0) { lck.unlock(); BRAFT_VLOG << "Out of order read request, path: " << _path - << " filename: " << filename << " offset: " << offset - << " max_count: " << max_count; + << " filename: " << filename << " offset: " << offset + << " max_count: " << max_count; return EINVAL; } if (_current_file) { @@ -77,7 +72,8 @@ int LocalDirReader::read_file_with_meta(butil::IOBuf* out, } std::string file_path(_path + "/" + filename); butil::File::Error e; - FileAdaptor* file = _fs->open(file_path, O_RDONLY | O_CLOEXEC, file_meta, &e); + FileAdaptor* file = + _fs->open(file_path, O_RDONLY | O_CLOEXEC, file_meta, &e); if (!file) { return file_error_to_os_error(e); } diff --git a/src/braft/file_reader.h b/src/braft/file_reader.h index f333eb2..46846b9 100644 --- a/src/braft/file_reader.h +++ b/src/braft/file_reader.h @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,80 +17,77 @@ // Xiong,Kai(xiongkai@baidu.com) // Yang,Guodong(yangguodong01@baidu.com) -#ifndef BRAFT_FILE_READER_H -#define BRAFT_FILE_READER_H +#ifndef BRAFT_FILE_READER_H +#define BRAFT_FILE_READER_H + +#include // butil::IOBuf +#include // butil::RefCountedThreadsafe + +#include // std::set -#include // std::set -#include // butil::RefCountedThreadsafe -#include // butil::IOBuf -#include "braft/macros.h" #include "braft/file_system_adaptor.h" +#include "braft/macros.h" namespace braft { // Abstract class to read data from a file // All the const method should be thread safe class FileReader : public butil::RefCountedThreadSafe { -friend class butil::RefCountedThreadSafe; -public: + friend class butil::RefCountedThreadSafe; + + public: // Read data from filename at |offset| (from the start of the file) for at - // most |max_count| bytes to |out|. Reading part of a file is allowed if + // most |max_count| bytes to |out|. Reading part of a file is allowed if // |read_partly| is TRUE. If successfully read the file, |read_count| // is the actual read count, it's maybe smaller than |max_count| if the // request is throttled or reach the end of the file. In the case of // reaching the end of the file, |is_eof| is also set to true. // Returns 0 on success, the error otherwise - virtual int read_file(butil::IOBuf* out, - const std::string &filename, - off_t offset, - size_t max_count, - bool read_partly, - size_t* read_count, - bool* is_eof) const = 0; + virtual int read_file(butil::IOBuf* out, const std::string& filename, + off_t offset, size_t max_count, bool read_partly, + size_t* read_count, bool* is_eof) const = 0; // Get the path of this reader virtual const std::string& path() const = 0; -protected: + + protected: FileReader() {} virtual ~FileReader() {} }; // Read files within a local directory class LocalDirReader : public FileReader { -public: - LocalDirReader(FileSystemAdaptor* fs, const std::string& path) - : _path(path), _fs(fs), _current_file(NULL), _is_reading(false), _eof_reached(true) - {} + public: + LocalDirReader(FileSystemAdaptor* fs, const std::string& path) + : _path(path), + _fs(fs), + _current_file(NULL), + _is_reading(false), + _eof_reached(true) {} virtual ~LocalDirReader(); // Open a snapshot for read virtual bool open(); - + // Read data from filename at |offset| (from the start of the file) for at - // most |max_count| bytes to |out|. Reading part of a file is allowed if + // most |max_count| bytes to |out|. Reading part of a file is allowed if // |read_partly| is TRUE. If successfully read the file, |read_count| // is the actual read count, it's maybe smaller than |max_count| if the // request is throttled or reach the end of the file. In the case of // reaching the end of the file, |is_eof| is also set to true. // Returns 0 on success, the error otherwise - virtual int read_file(butil::IOBuf* out, - const std::string &filename, - off_t offset, - size_t max_count, - bool read_partly, - size_t* read_count, - bool* is_eof) const; + virtual int read_file(butil::IOBuf* out, const std::string& filename, + off_t offset, size_t max_count, bool read_partly, + size_t* read_count, bool* is_eof) const; virtual const std::string& path() const { return _path; } -protected: - int read_file_with_meta(butil::IOBuf* out, - const std::string &filename, - google::protobuf::Message* file_meta, - off_t offset, - size_t max_count, - size_t* read_count, + + protected: + int read_file_with_meta(butil::IOBuf* out, const std::string& filename, + google::protobuf::Message* file_meta, off_t offset, + size_t max_count, size_t* read_count, bool* is_eof) const; const scoped_refptr& file_system() const { return _fs; } -private: + private: mutable raft_mutex_t _mutex; std::string _path; scoped_refptr _fs; @@ -102,4 +99,4 @@ class LocalDirReader : public FileReader { } // namespace braft -#endif //BRAFT_FILE_READER_H +#endif // BRAFT_FILE_READER_H diff --git a/src/braft/file_service.cpp b/src/braft/file_service.cpp index 516ff2d..f310507 100644 --- a/src/braft/file_service.cpp +++ b/src/braft/file_service.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,18 +16,21 @@ #include "braft/file_service.h" -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include + +#include + #include "braft/util.h" namespace braft { -DEFINE_bool(raft_file_check_hole, false, "file service check hole switch, default disable"); +DEFINE_bool(raft_file_check_hole, false, + "file service check hole switch, default disable"); void FileServiceImpl::get_file(::google::protobuf::RpcController* controller, const ::braft::GetFileRequest* request, @@ -40,15 +43,18 @@ void FileServiceImpl::get_file(::google::protobuf::RpcController* controller, Map::const_iterator iter = _reader_map.find(request->reader_id()); if (iter == _reader_map.end()) { lck.unlock(); - cntl->SetFailed(ENOENT, "Fail to find reader=%" PRId64, request->reader_id()); + cntl->SetFailed(ENOENT, "Fail to find reader=%" PRId64, + request->reader_id()); return; } // Don't touch iter ever after reader = iter->second; lck.unlock(); - BRAFT_VLOG << "get_file for " << cntl->remote_side() << " path=" << reader->path() + BRAFT_VLOG << "get_file for " << cntl->remote_side() + << " path=" << reader->path() << " filename=" << request->filename() - << " offset=" << request->offset() << " count=" << request->count(); + << " offset=" << request->offset() + << " count=" << request->count(); if (request->count() <= 0 || request->offset() < 0) { cntl->SetFailed(brpc::EREQUEST, "Invalid request=%s", @@ -61,19 +67,17 @@ void FileServiceImpl::get_file(::google::protobuf::RpcController* controller, size_t read_count = 0; const int rc = reader->read_file( - &buf, request->filename(), - request->offset(), request->count(), - request->read_partly(), - &read_count, - &is_eof); + &buf, request->filename(), request->offset(), request->count(), + request->read_partly(), &read_count, &is_eof); if (rc != 0) { cntl->SetFailed(rc, "Fail to read from path=%s filename=%s : %s", - reader->path().c_str(), request->filename().c_str(), berror(rc)); + reader->path().c_str(), request->filename().c_str(), + berror(rc)); return; } response->set_eof(is_eof); - response->set_read_size(read_count); + response->set_read_size(read_count); // skip empty data if (buf.size() == 0) { return; @@ -101,7 +105,8 @@ void FileServiceImpl::get_file(::google::protobuf::RpcController* controller, } FileServiceImpl::FileServiceImpl() { - _next_id = ((int64_t)getpid() << 45) | (butil::gettimeofday_us() << 17 >> 17); + _next_id = + ((int64_t)getpid() << 45) | (butil::gettimeofday_us() << 17 >> 17); } int FileServiceImpl::add_reader(FileReader* reader, int64_t* reader_id) { diff --git a/src/braft/file_service.h b/src/braft/file_service.h index 409e1a0..e8e8849 100644 --- a/src/braft/file_service.h +++ b/src/braft/file_service.h @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,18 +14,19 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_FILE_SERVICE_H -#define BRAFT_FILE_SERVICE_H +#ifndef BRAFT_FILE_SERVICE_H +#define BRAFT_FILE_SERVICE_H #include -#include "braft/file_service.pb.h" + #include "braft/file_reader.h" +#include "braft/file_service.pb.h" #include "braft/util.h" namespace braft { class BAIDU_CACHELINE_ALIGNMENT FileServiceImpl : public FileService { -public: + public: static FileServiceImpl* GetInstance() { return Singleton::get(); } @@ -35,8 +36,9 @@ class BAIDU_CACHELINE_ALIGNMENT FileServiceImpl : public FileService { ::google::protobuf::Closure* done); int add_reader(FileReader* reader, int64_t* reader_id); int remove_reader(int64_t reader_id); -private: -friend struct DefaultSingletonTraits; + + private: + friend struct DefaultSingletonTraits; FileServiceImpl(); ~FileServiceImpl() {} typedef std::map > Map; @@ -45,8 +47,9 @@ friend struct DefaultSingletonTraits; Map _reader_map; }; -inline FileServiceImpl* file_service() -{ return FileServiceImpl::GetInstance(); } +inline FileServiceImpl* file_service() { + return FileServiceImpl::GetInstance(); +} inline int file_service_add(FileReader* reader, int64_t* reader_id) { FileServiceImpl* const fs = file_service(); @@ -60,4 +63,4 @@ inline int file_service_remove(int64_t reader_id) { } // namespace braft -#endif //BRAFT_FILE_SERVICE_H +#endif // BRAFT_FILE_SERVICE_H diff --git a/src/braft/file_system_adaptor.cpp b/src/braft/file_system_adaptor.cpp index ac163d2..3636144 100644 --- a/src/braft/file_system_adaptor.cpp +++ b/src/braft/file_system_adaptor.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,15 +14,14 @@ // Authors: Zheng,PengFei(zhengpengfei@baidu.com) -#include // butil::make_close_on_exec -#include // butil::get_leaky_singleton #include "braft/file_system_adaptor.h" +#include // butil::make_close_on_exec +#include // butil::get_leaky_singleton + namespace braft { -bool PosixDirReader::is_valid() const { - return _dir_reader.IsValid(); -} +bool PosixDirReader::is_valid() const { return _dir_reader.IsValid(); } bool PosixDirReader::next() { bool rc = _dir_reader.Next(); @@ -32,18 +31,16 @@ bool PosixDirReader::next() { return rc; } -const char* PosixDirReader::name() const { - return _dir_reader.name(); -} +const char* PosixDirReader::name() const { return _dir_reader.name(); } -PosixFileAdaptor::~PosixFileAdaptor() { -} +PosixFileAdaptor::~PosixFileAdaptor() {} ssize_t PosixFileAdaptor::write(const butil::IOBuf& data, off_t offset) { return braft::file_pwrite(data, _fd, offset); } -ssize_t PosixFileAdaptor::read(butil::IOPortal* portal, off_t offset, size_t size) { +ssize_t PosixFileAdaptor::read(butil::IOPortal* portal, off_t offset, + size_t size) { return braft::file_pread(portal, _fd, offset, size); } @@ -52,9 +49,7 @@ ssize_t PosixFileAdaptor::size() { return ssize_t(sz); } -bool PosixFileAdaptor::sync() { - return raft_fsync(_fd) == 0; -} +bool PosixFileAdaptor::sync() { return raft_fsync(_fd) == 0; } bool PosixFileAdaptor::close() { if (_fd > 0) { @@ -65,26 +60,29 @@ bool PosixFileAdaptor::close() { return true; } -ssize_t BufferedSequentialReadFileAdaptor::read(butil::IOPortal* portal, off_t offset, size_t size) { +ssize_t BufferedSequentialReadFileAdaptor::read(butil::IOPortal* portal, + off_t offset, size_t size) { if (_error) { errno = _error; return -1; } - + BRAFT_VLOG << "begin read offset " << offset << " count " << size - << ", buffer_offset " << _buffer_offset - << " buffer_size " << _buffer_size; - if (offset < _buffer_offset || offset > off_t(_buffer_offset + _buffer_size)) { - LOG(WARNING) << "Fail to read from buffered file adaptor with invalid range" + << ", buffer_offset " << _buffer_offset << " buffer_size " + << _buffer_size; + if (offset < _buffer_offset || + offset > off_t(_buffer_offset + _buffer_size)) { + LOG(WARNING) + << "Fail to read from buffered file adaptor with invalid range" << ", buffer_offset: " << _buffer_offset - << ", buffer_size: " << _buffer_size - << ", read offset: " << offset + << ", buffer_size: " << _buffer_size << ", read offset: " << offset << ", read size: " << size; errno = EINVAL; return -1; } if (offset > _buffer_offset) { - _buffer.pop_front(std::min(size_t(offset - _buffer_offset), _buffer.size())); + _buffer.pop_front( + std::min(size_t(offset - _buffer_offset), _buffer.size())); _buffer_size -= (offset - _buffer_offset); _buffer_offset = offset; } @@ -113,30 +111,31 @@ ssize_t BufferedSequentialReadFileAdaptor::read(butil::IOPortal* portal, off_t o return nread; } -ssize_t BufferedSequentialWriteFileAdaptor::write(const butil::IOBuf& data, off_t offset) { +ssize_t BufferedSequentialWriteFileAdaptor::write(const butil::IOBuf& data, + off_t offset) { if (_error) { errno = _error; return -1; } - - BRAFT_VLOG << "begin write offset " << offset << ", data_size " << data.size() - << ", buffer_offset " << _buffer_offset - << ", buffer_size " << _buffer_size; + + BRAFT_VLOG << "begin write offset " << offset << ", data_size " + << data.size() << ", buffer_offset " << _buffer_offset + << ", buffer_size " << _buffer_size; if (static_cast(offset) < static_cast(_buffer_offset) + _buffer_size) { - LOG(WARNING) - << "Fail to write into buffered file adaptor with invalid range" - << ", offset: " << offset << ", data_size: " << data.size() - << ", buffer_offset: " << _buffer_offset - << ", buffer_size: " << _buffer_size; - errno = EINVAL; - return -1; + LOG(WARNING) + << "Fail to write into buffered file adaptor with invalid range" + << ", offset: " << offset << ", data_size: " << data.size() + << ", buffer_offset: " << _buffer_offset + << ", buffer_size: " << _buffer_size; + errno = EINVAL; + return -1; } else if (static_cast(offset) > static_cast(_buffer_offset) + _buffer_size) { - // passby hole - CHECK(_buffer_size == 0); - BRAFT_VLOG << "seek to new offset " << offset << " as there is hole"; - seek(offset); + // passby hole + CHECK(_buffer_size == 0); + BRAFT_VLOG << "seek to new offset " << offset << " as there is hole"; + seek(offset); } const size_t saved_size = data.size(); _buffer.append(data); @@ -149,7 +148,7 @@ ssize_t BufferedSequentialWriteFileAdaptor::write(const butil::IOBuf& data, off_ errno = _error; return -1; } - _buffer_offset += write_count; + _buffer_offset += write_count; _buffer_size -= write_count; _buffer.pop_front(write_count); CHECK_EQ(_buffer_size, _buffer.size()); @@ -168,10 +167,10 @@ static void check_cloexec(void) { } } -FileAdaptor* PosixFileSystemAdaptor::open(const std::string& path, int oflag, - const ::google::protobuf::Message* file_meta, - butil::File::Error* e) { - (void) file_meta; +FileAdaptor* PosixFileSystemAdaptor::open( + const std::string& path, int oflag, + const ::google::protobuf::Message* file_meta, butil::File::Error* e) { + (void)file_meta; pthread_once(&s_check_cloexec_once, check_cloexec); bool cloexec = (oflag & O_CLOEXEC); if (cloexec && !s_support_cloexec_on_open) { @@ -179,7 +178,8 @@ FileAdaptor* PosixFileSystemAdaptor::open(const std::string& path, int oflag, } int fd = ::open(path.c_str(), oflag, 0644); if (e) { - *e = (fd == -1) ? butil::File::OSErrorToFileError(errno) : butil::File::FILE_OK; + *e = (fd == -1) ? butil::File::OSErrorToFileError(errno) + : butil::File::FILE_OK; } if (fd == -1) { return NULL; @@ -190,24 +190,28 @@ FileAdaptor* PosixFileSystemAdaptor::open(const std::string& path, int oflag, return new PosixFileAdaptor(fd); } -bool PosixFileSystemAdaptor::delete_file(const std::string& path, bool recursive) { +bool PosixFileSystemAdaptor::delete_file(const std::string& path, + bool recursive) { butil::FilePath file_path(path); return butil::DeleteFile(file_path, recursive); } -bool PosixFileSystemAdaptor::rename(const std::string& old_path, const std::string& new_path) { +bool PosixFileSystemAdaptor::rename(const std::string& old_path, + const std::string& new_path) { return ::rename(old_path.c_str(), new_path.c_str()) == 0; } -bool PosixFileSystemAdaptor::link(const std::string& old_path, const std::string& new_path) { +bool PosixFileSystemAdaptor::link(const std::string& old_path, + const std::string& new_path) { return ::link(old_path.c_str(), new_path.c_str()) == 0; } -bool PosixFileSystemAdaptor::create_directory(const std::string& path, - butil::File::Error* error, - bool create_parent_directories) { +bool PosixFileSystemAdaptor::create_directory(const std::string& path, + butil::File::Error* error, + bool create_parent_directories) { butil::FilePath dir(path); - return butil::CreateDirectoryAndGetError(dir, error, create_parent_directories); + return butil::CreateDirectoryAndGetError(dir, error, + create_parent_directories); } bool PosixFileSystemAdaptor::path_exists(const std::string& path) { @@ -225,14 +229,14 @@ DirReader* PosixFileSystemAdaptor::directory_reader(const std::string& path) { } FileSystemAdaptor* default_file_system() { - static scoped_refptr fs = + static scoped_refptr fs = butil::get_leaky_singleton(); return fs.get(); } int file_error_to_os_error(butil::File::Error e) { switch (e) { - case butil::File::FILE_OK: + case butil::File::FILE_OK: return 0; case butil::File::FILE_ERROR_IN_USE: return EAGAIN; @@ -258,8 +262,7 @@ int file_error_to_os_error(butil::File::Error e) { } bool create_sub_directory(const std::string& parent_path, - const std::string& sub_path, - FileSystemAdaptor* fs, + const std::string& sub_path, FileSystemAdaptor* fs, butil::File::Error* error) { if (!fs) { fs = default_file_system(); @@ -283,13 +286,13 @@ bool create_sub_directory(const std::string& parent_path, butil::FilePath last_path = sub_dir_path; subpaths.push_back(sub_dir_path.BaseName()); for (butil::FilePath path = last_path.DirName(); - path.value() != last_path.value(); path = path.DirName()) { + path.value() != last_path.value(); path = path.DirName()) { subpaths.push_back(path.BaseName()); last_path = path; } butil::FilePath full_path(parent_path); for (std::vector::reverse_iterator i = subpaths.rbegin(); - i != subpaths.rend(); ++i) { + i != subpaths.rend(); ++i) { if (i->value() == "/") { continue; } @@ -305,5 +308,4 @@ bool create_sub_directory(const std::string& parent_path, return true; } - -} // namespace braft +} // namespace braft diff --git a/src/braft/file_system_adaptor.h b/src/braft/file_system_adaptor.h index 9a50b67..a7f66b8 100644 --- a/src/braft/file_system_adaptor.h +++ b/src/braft/file_system_adaptor.h @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,21 +14,24 @@ // Authors: Zheng,PengFei(zhengpengfei@baidu.com) -#ifndef BRAFT_FILE_SYSTEM_ADAPTOR_H -#define BRAFT_FILE_SYSTEM_ADAPTOR_H +#ifndef BRAFT_FILE_SYSTEM_ADAPTOR_H +#define BRAFT_FILE_SYSTEM_ADAPTOR_H -#include #include -#include // butil::File -#include // butil::DirReaderPosix -#include // butil::RefCountedThreadSafe -#include // Singleton -#include // google::protobuf::Message -#include "braft/util.h" +#include // butil::DirReaderPosix +#include // butil::File +#include // butil::RefCountedThreadSafe +#include // Singleton +#include +#include // google::protobuf::Message + #include "braft/fsync.h" +#include "braft/util.h" -#ifndef O_CLOEXEC -#define O_CLOEXEC 02000000 /* define close_on_exec if not defined in fcntl.h*/ +#ifndef O_CLOEXEC +#define O_CLOEXEC \ + 02000000 /*define close_on_exec if not defined in \ + fcntl.h*/ #endif namespace braft { @@ -36,7 +39,7 @@ namespace braft { // DirReader iterates a directory to get sub directories and files, `.' and `..' // should be ignored class DirReader { -public: + public: DirReader() {} virtual ~DirReader() {} @@ -50,29 +53,32 @@ class DirReader { // Get the name of current entry virtual const char* name() const = 0; -private: + private: DISALLOW_COPY_AND_ASSIGN(DirReader); }; template struct DestroyObj { - void operator()(T* const obj) { obj->close(); delete obj; } + void operator()(T* const obj) { + obj->close(); + delete obj; + } }; - class FileAdaptor { -public: + public: virtual ~FileAdaptor() {} - // Write to the file. Different from posix ::pwrite(), write will retry automatically - // when occur EINTR. - // Return |data.size()| if successful, -1 otherwise. + // Write to the file. Different from posix ::pwrite(), write will retry + // automatically when occur EINTR. Return |data.size()| if successful, -1 + // otherwise. virtual ssize_t write(const butil::IOBuf& data, off_t offset) = 0; - // Read from the file. Different from posix ::pread(), read will retry automatically - // when occur EINTR. - // Return a non-negative integer less than or equal to |size| if successful, -1 otherwise. - // In the case of EOF, the return value is a non-negative integer less than |size|. - virtual ssize_t read(butil::IOPortal* portal, off_t offset, size_t size) = 0; + // Read from the file. Different from posix ::pread(), read will retry + // automatically when occur EINTR. Return a non-negative integer less than + // or equal to |size| if successful, -1 otherwise. In the case of EOF, the + // return value is a non-negative integer less than |size|. + virtual ssize_t read(butil::IOPortal* portal, off_t offset, + size_t size) = 0; // Get the size of the file virtual ssize_t size() = 0; @@ -83,72 +89,80 @@ class FileAdaptor { // Close the descriptor of this file adaptor virtual bool close() = 0; -protected: - + protected: FileAdaptor() {} -private: + private: DISALLOW_COPY_AND_ASSIGN(FileAdaptor); }; -class FileSystemAdaptor : public butil::RefCountedThreadSafe { -public: +class FileSystemAdaptor + : public butil::RefCountedThreadSafe { + public: FileSystemAdaptor() {} virtual ~FileSystemAdaptor() {} - // Open a file, oflag can be any valid combination of flags used by posix ::open(), - // file_meta can be used to pass additinal metadata, it won't be modified, and should - // be valid until the file is destroyed. - virtual FileAdaptor* open(const std::string& path, int oflag, + // Open a file, oflag can be any valid combination of flags used by posix + // ::open(), file_meta can be used to pass additinal metadata, it won't be + // modified, and should be valid until the file is destroyed. + virtual FileAdaptor* open(const std::string& path, int oflag, const ::google::protobuf::Message* file_meta, butil::File::Error* e) = 0; - // Deletes the given path, whether it's a file or a directory. If it's a directory, - // it's perfectly happy to delete all of the directory's contents. Passing true to - // recursive deletes subdirectories and their contents as well. - // Returns true if successful, false otherwise. It is considered successful - // to attempt to delete a file that does not exist. + // Deletes the given path, whether it's a file or a directory. If it's a + // directory, it's perfectly happy to delete all of the directory's + // contents. Passing true to recursive deletes subdirectories and their + // contents as well. Returns true if successful, false otherwise. It is + // considered successful to attempt to delete a file that does not exist. virtual bool delete_file(const std::string& path, bool recursive) = 0; // The same as posix ::rename(), will change the name of the old path. - virtual bool rename(const std::string& old_path, const std::string& new_path) = 0; + virtual bool rename(const std::string& old_path, + const std::string& new_path) = 0; // The same as posix ::link(), will link the old path to the new path. - virtual bool link(const std::string& old_path, const std::string& new_path) = 0; - - // Creates a directory. If create_parent_directories is true, parent directories - // will be created if not exist, otherwise, the create operation will fail. - // Returns 'true' on successful creation, or if the directory already exists. - virtual bool create_directory(const std::string& path, + virtual bool link(const std::string& old_path, + const std::string& new_path) = 0; + + // Creates a directory. If create_parent_directories is true, parent + // directories will be created if not exist, otherwise, the create operation + // will fail. Returns 'true' on successful creation, or if the directory + // already exists. + virtual bool create_directory(const std::string& path, butil::File::Error* error, bool create_parent_directories) = 0; // Returns true if the given path exists on the filesystem, false otherwise. virtual bool path_exists(const std::string& path) = 0; - // Returns true if the given path exists and is a directory, false otherwise. + // Returns true if the given path exists and is a directory, false + // otherwise. virtual bool directory_exists(const std::string& path) = 0; - // Get a directory reader to read all sub entries inside a directory. It will - // not recursively search the directory. + // Get a directory reader to read all sub entries inside a directory. It + // will not recursively search the directory. virtual DirReader* directory_reader(const std::string& path) = 0; // This method will be called at the very begin before read snapshot file. // The default implemention is return 'true' directly. - virtual bool open_snapshot(const std::string& /*snapshot_path*/) { return true; } - + virtual bool open_snapshot(const std::string& /*snapshot_path*/) { + return true; + } + // This method will be called after read all snapshot files or failed. // The default implemention is return directly. virtual void close_snapshot(const std::string& /*snapshot_path*/) {} -private: + + private: DISALLOW_COPY_AND_ASSIGN(FileSystemAdaptor); }; // DirReader iterates a directory to get names of all sub directories and files, // except `.' and `..'. class PosixDirReader : public DirReader { -friend class PosixFileSystemAdaptor; -public: + friend class PosixFileSystemAdaptor; + + public: virtual ~PosixDirReader() {} // Check if the dir reader is valid @@ -161,16 +175,17 @@ friend class PosixFileSystemAdaptor; // Get the name of current entry virtual const char* name() const; -protected: + protected: PosixDirReader(const std::string& path) : _dir_reader(path.c_str()) {} -private: + private: butil::DirReaderPosix _dir_reader; }; class PosixFileAdaptor : public FileAdaptor { -friend class PosixFileSystemAdaptor; -public: + friend class PosixFileSystemAdaptor; + + public: virtual ~PosixFileAdaptor(); virtual ssize_t write(const butil::IOBuf& data, off_t offset); @@ -179,15 +194,15 @@ friend class PosixFileSystemAdaptor; virtual bool sync(); virtual bool close(); -protected: + protected: PosixFileAdaptor(int fd) : _fd(fd) {} -private: + private: int _fd; }; class BufferedSequentialReadFileAdaptor : public FileAdaptor { -public: + public: virtual ~BufferedSequentialReadFileAdaptor() {} virtual ssize_t write(const butil::IOBuf& data, off_t offset) { @@ -199,39 +214,41 @@ class BufferedSequentialReadFileAdaptor : public FileAdaptor { CHECK(false); return false; } - virtual bool close() { - return true; - } + virtual bool close() { return true; } virtual ssize_t size() { return _reach_file_eof ? (_buffer_offset + _buffer_size) : SSIZE_MAX; } -protected: + protected: BufferedSequentialReadFileAdaptor() - : _buffer_offset(0), _buffer_size(0), _reach_file_eof(false), _error(0) - {} - - // The |need_count| here is just a suggestion, |nread| can be larger than |need_count| - // actually, if needed. This is useful for some special cases, in which data must be - // read atomically. If |nread| < |need_count|, the wrapper think eof is reached. - virtual int do_read(butil::IOPortal* portal, size_t need_count, size_t* nread) = 0; - -private: + : _buffer_offset(0), + _buffer_size(0), + _reach_file_eof(false), + _error(0) {} + + // The |need_count| here is just a suggestion, |nread| can be larger than + // |need_count| actually, if needed. This is useful for some special cases, + // in which data must be read atomically. If |nread| < |need_count|, the + // wrapper think eof is reached. + virtual int do_read(butil::IOPortal* portal, size_t need_count, + size_t* nread) = 0; + + private: butil::IOBuf _buffer; - off_t _buffer_offset; - size_t _buffer_size; - bool _reach_file_eof; - int _error; + off_t _buffer_offset; + size_t _buffer_size; + bool _reach_file_eof; + int _error; }; class BufferedSequentialWriteFileAdaptor : public FileAdaptor { -public: + public: virtual ~BufferedSequentialWriteFileAdaptor() {} virtual ssize_t write(const butil::IOBuf& data, off_t offset); virtual ssize_t read(butil::IOPortal* portal, off_t offset, size_t size) { CHECK(false); - return -1; + return -1; } virtual bool sync() { CHECK(false); @@ -243,43 +260,41 @@ class BufferedSequentialWriteFileAdaptor : public FileAdaptor { } virtual ssize_t size() { CHECK(false); - return -1; + return -1; } -protected: + protected: BufferedSequentialWriteFileAdaptor() - : _buffer_offset(0), _buffer_size(0), _error(0) - {} - + : _buffer_offset(0), _buffer_size(0), _error(0) {} + // Write |data| into underlayer device with its current offset, |nwrite| // is set to the number of written bytes // Return 0 on success virtual int do_write(const butil::IOBuf& data, size_t* nwrite) = 0; - + // Seek to a given offset when there is hole // NOTICE: Only seek to bigger offset - virtual void seek(off_t offset) { - _buffer_offset = offset; - } + virtual void seek(off_t offset) { _buffer_offset = offset; } butil::IOBuf _buffer; - off_t _buffer_offset; - size_t _buffer_size; - int _error; + off_t _buffer_offset; + size_t _buffer_size; + int _error; }; class PosixFileSystemAdaptor : public FileSystemAdaptor { -public: + public: PosixFileSystemAdaptor() {} virtual ~PosixFileSystemAdaptor() {} - virtual FileAdaptor* open(const std::string& path, int oflag, + virtual FileAdaptor* open(const std::string& path, int oflag, const ::google::protobuf::Message* file_meta, butil::File::Error* e); virtual bool delete_file(const std::string& path, bool recursive); - virtual bool rename(const std::string& old_path, const std::string& new_path); + virtual bool rename(const std::string& old_path, + const std::string& new_path); virtual bool link(const std::string& old_path, const std::string& new_path); - virtual bool create_directory(const std::string& path, + virtual bool create_directory(const std::string& path, butil::File::Error* error, bool create_parent_directories); virtual bool path_exists(const std::string& path); @@ -302,6 +317,6 @@ bool create_sub_directory(const std::string& parent_path, FileSystemAdaptor* fs = NULL, butil::File::Error* error = NULL); -} // namespace braft +} // namespace braft -#endif // BRAFT_FILE_SYSTEM_ADAPTOR_H +#endif // BRAFT_FILE_SYSTEM_ADAPTOR_H diff --git a/src/braft/fsm_caller.cpp b/src/braft/fsm_caller.cpp index a0b6801..7cb34ff 100644 --- a/src/braft/fsm_caller.cpp +++ b/src/braft/fsm_caller.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,45 +16,42 @@ // Wang,Yao(wangyao02@baidu.com) // Xiong,Kai(xiongkai@baidu.com) +#include "braft/fsm_caller.h" + +#include #include -#include "braft/raft.h" + +#include "braft/errno.pb.h" +#include "braft/log_entry.h" #include "braft/log_manager.h" #include "braft/node.h" -#include "braft/util.h" +#include "braft/raft.h" #include "braft/raft.pb.h" -#include "braft/log_entry.h" -#include "braft/errno.pb.h" -#include "braft/node.h" - -#include "braft/fsm_caller.h" -#include +#include "braft/util.h" namespace braft { static bvar::CounterRecorder g_commit_tasks_batch_counter( - "raft_commit_tasks_batch_counter"); + "raft_commit_tasks_batch_counter"); -DEFINE_int32(raft_fsm_caller_commit_batch, 512, - "Max numbers of logs for the state machine to commit in a single batch"); +DEFINE_int32( + raft_fsm_caller_commit_batch, 512, + "Max numbers of logs for the state machine to commit in a single batch"); BRPC_VALIDATE_GFLAG(raft_fsm_caller_commit_batch, brpc::PositiveInteger); FSMCaller::FSMCaller() - : _log_manager(NULL) - , _fsm(NULL) - , _closure_queue(NULL) - , _last_applied_index(0) - , _last_applied_term(0) - , _after_shutdown(NULL) - , _node(NULL) - , _cur_task(IDLE) - , _applying_index(0) - , _queue_started(false) -{ -} - -FSMCaller::~FSMCaller() { - CHECK(_after_shutdown == NULL); -} + : _log_manager(NULL), + _fsm(NULL), + _closure_queue(NULL), + _last_applied_index(0), + _last_applied_term(0), + _after_shutdown(NULL), + _node(NULL), + _cur_task(IDLE), + _applying_index(0), + _queue_started(false) {} + +FSMCaller::~FSMCaller() { CHECK(_after_shutdown == NULL); } int FSMCaller::run(void* meta, bthread::TaskIterator& iter) { FSMCaller* caller = (FSMCaller*)meta; @@ -64,7 +61,7 @@ int FSMCaller::run(void* meta, bthread::TaskIterator& iter) { } int64_t max_committed_index = -1; size_t counter = 0; - size_t batch_size = FLAGS_raft_fsm_caller_commit_batch; + size_t batch_size = FLAGS_raft_fsm_caller_commit_batch; for (; iter; ++iter) { if (iter->type == COMMITTED && counter < batch_size) { if (iter->committed_index > max_committed_index) { @@ -81,52 +78,54 @@ int FSMCaller::run(void* meta, bthread::TaskIterator& iter) { batch_size = FLAGS_raft_fsm_caller_commit_batch; } switch (iter->type) { - case COMMITTED: - if (iter->committed_index > max_committed_index) { - max_committed_index = iter->committed_index; - counter++; - } - break; - case SNAPSHOT_SAVE: - caller->_cur_task = SNAPSHOT_SAVE; - if (caller->pass_by_status(iter->done)) { - caller->do_snapshot_save((SaveSnapshotClosure*)iter->done); - } - break; - case SNAPSHOT_LOAD: - caller->_cur_task = SNAPSHOT_LOAD; - // TODO: do we need to allow the snapshot loading to recover the - // StateMachine if possible? - if (caller->pass_by_status(iter->done)) { - caller->do_snapshot_load((LoadSnapshotClosure*)iter->done); - } - break; - case LEADER_STOP: - caller->_cur_task = LEADER_STOP; - caller->do_leader_stop(*(iter->status)); - delete iter->status; - break; - case LEADER_START: - caller->do_leader_start(*(iter->leader_start_context)); - delete iter->leader_start_context; - break; - case START_FOLLOWING: - caller->_cur_task = START_FOLLOWING; - caller->do_start_following(*(iter->leader_change_context)); - delete iter->leader_change_context; - break; - case STOP_FOLLOWING: - caller->_cur_task = STOP_FOLLOWING; - caller->do_stop_following(*(iter->leader_change_context)); - delete iter->leader_change_context; - break; - case ERROR: - caller->_cur_task = ERROR; - caller->do_on_error((OnErrorClousre*)iter->done); - break; - case IDLE: - CHECK(false) << "Can't reach here"; - break; + case COMMITTED: + if (iter->committed_index > max_committed_index) { + max_committed_index = iter->committed_index; + counter++; + } + break; + case SNAPSHOT_SAVE: + caller->_cur_task = SNAPSHOT_SAVE; + if (caller->pass_by_status(iter->done)) { + caller->do_snapshot_save( + (SaveSnapshotClosure*)iter->done); + } + break; + case SNAPSHOT_LOAD: + caller->_cur_task = SNAPSHOT_LOAD; + // TODO: do we need to allow the snapshot loading to recover + // the StateMachine if possible? + if (caller->pass_by_status(iter->done)) { + caller->do_snapshot_load( + (LoadSnapshotClosure*)iter->done); + } + break; + case LEADER_STOP: + caller->_cur_task = LEADER_STOP; + caller->do_leader_stop(*(iter->status)); + delete iter->status; + break; + case LEADER_START: + caller->do_leader_start(*(iter->leader_start_context)); + delete iter->leader_start_context; + break; + case START_FOLLOWING: + caller->_cur_task = START_FOLLOWING; + caller->do_start_following(*(iter->leader_change_context)); + delete iter->leader_change_context; + break; + case STOP_FOLLOWING: + caller->_cur_task = STOP_FOLLOWING; + caller->do_stop_following(*(iter->leader_change_context)); + delete iter->leader_change_context; + break; + case ERROR: + caller->_cur_task = ERROR; + caller->do_on_error((OnErrorClousre*)iter->done); + break; + case IDLE: + CHECK(false) << "Can't reach here"; + break; }; } } @@ -144,9 +143,8 @@ bool FSMCaller::pass_by_status(Closure* done) { brpc::ClosureGuard done_guard(done); if (!_error.status().ok()) { if (done) { - done->status().set_error( - EINVAL, "FSMCaller is in bad status=`%s'", - _error.status().error_cstr()); + done->status().set_error(EINVAL, "FSMCaller is in bad status=`%s'", + _error.status().error_cstr()); } return false; } @@ -154,9 +152,9 @@ bool FSMCaller::pass_by_status(Closure* done) { return true; } -int FSMCaller::init(const FSMCallerOptions &options) { - if (options.log_manager == NULL || options.fsm == NULL - || options.closure_queue == NULL) { +int FSMCaller::init(const FSMCallerOptions& options) { + if (options.log_manager == NULL || options.fsm == NULL || + options.closure_queue == NULL) { return EINVAL; } _log_manager = options.log_manager; @@ -170,15 +168,12 @@ int FSMCaller::init(const FSMCallerOptions &options) { if (_node) { _node->AddRef(); } - + bthread::ExecutionQueueOptions execq_opt; - execq_opt.bthread_attr = options.usercode_in_pthread - ? BTHREAD_ATTR_PTHREAD - : BTHREAD_ATTR_NORMAL; - if (bthread::execution_queue_start(&_queue_id, - &execq_opt, - FSMCaller::run, - this) != 0) { + execq_opt.bthread_attr = options.usercode_in_pthread ? BTHREAD_ATTR_PTHREAD + : BTHREAD_ATTR_NORMAL; + if (bthread::execution_queue_start(&_queue_id, &execq_opt, FSMCaller::run, + this) != 0) { LOG(ERROR) << "fsm fail to start execution_queue"; return -1; } @@ -216,14 +211,12 @@ int FSMCaller::on_committed(int64_t committed_index) { } class OnErrorClousre : public Closure { -public: - OnErrorClousre(const Error& e) : _e(e) { - } + public: + OnErrorClousre(const Error& e) : _e(e) {} const Error& error() { return _e; } - void Run() { - delete this; - } -private: + void Run() { delete this; } + + private: ~OnErrorClousre() {} Error _e; }; @@ -233,7 +226,7 @@ int FSMCaller::on_error(const Error& e) { ApplyTask t; t.type = ERROR; t.done = c; - if (bthread::execution_queue_execute(_queue_id, t, + if (bthread::execution_queue_execute(_queue_id, t, &bthread::TASK_OPTIONS_URGENT) != 0) { c->Run(); return -1; @@ -264,8 +257,8 @@ void FSMCaller::do_committed(int64_t committed_index) { if (!_error.status().ok()) { return; } - int64_t last_applied_index = _last_applied_index.load( - butil::memory_order_relaxed); + int64_t last_applied_index = + _last_applied_index.load(butil::memory_order_relaxed); // We can tolerate the disorder of committed_index if (last_applied_index >= committed_index) { @@ -277,15 +270,17 @@ void FSMCaller::do_committed(int64_t committed_index) { &first_closure_index)); IteratorImpl iter_impl(_fsm, _log_manager, &closure, first_closure_index, - last_applied_index, committed_index, &_applying_index); + last_applied_index, committed_index, + &_applying_index); for (; iter_impl.is_good();) { if (iter_impl.entry()->type != ENTRY_TYPE_DATA) { if (iter_impl.entry()->type == ENTRY_TYPE_CONFIGURATION) { if (iter_impl.entry()->old_peers == NULL) { - // Joint stage is not supposed to be noticeable by end users. + // Joint stage is not supposed to be noticeable by end + // users. _fsm->on_configuration_committed( - Configuration(*iter_impl.entry()->peers), - iter_impl.entry()->id.index); + Configuration(*iter_impl.entry()->peers), + iter_impl.entry()->id.index); } } // For other entries, we have nothing to do besides flush the @@ -300,9 +295,9 @@ void FSMCaller::do_committed(int64_t committed_index) { Iterator iter(&iter_impl); _fsm->on_apply(iter); LOG_IF(ERROR, iter.valid()) - << "Node " << _node->node_id() - << " Iterator is still valid, did you return before iterator " - " reached the end?"; + << "Node " << _node->node_id() + << " Iterator is still valid, did you return before iterator " + " reached the end?"; // Try move to next in case that we pass the same log twice. iter.next(); } @@ -328,27 +323,27 @@ int FSMCaller::on_snapshot_save(SaveSnapshotClosure* done) { void FSMCaller::do_snapshot_save(SaveSnapshotClosure* done) { CHECK(done); - int64_t last_applied_index = _last_applied_index.load(butil::memory_order_relaxed); + int64_t last_applied_index = + _last_applied_index.load(butil::memory_order_relaxed); SnapshotMeta meta; meta.set_last_included_index(last_applied_index); meta.set_last_included_term(_last_applied_term); ConfigurationEntry conf_entry; _log_manager->get_configuration(last_applied_index, &conf_entry); - for (Configuration::const_iterator - iter = conf_entry.conf.begin(); - iter != conf_entry.conf.end(); ++iter) { + for (Configuration::const_iterator iter = conf_entry.conf.begin(); + iter != conf_entry.conf.end(); ++iter) { *meta.add_peers() = iter->to_string(); } - for (Configuration::const_iterator - iter = conf_entry.old_conf.begin(); - iter != conf_entry.old_conf.end(); ++iter) { + for (Configuration::const_iterator iter = conf_entry.old_conf.begin(); + iter != conf_entry.old_conf.end(); ++iter) { *meta.add_old_peers() = iter->to_string(); } SnapshotWriter* writer = done->start(meta); if (!writer) { - done->status().set_error(EINVAL, "snapshot_storage create SnapshotWriter failed"); + done->status().set_error( + EINVAL, "snapshot_storage create SnapshotWriter failed"); done->Run(); return; } @@ -365,7 +360,7 @@ int FSMCaller::on_snapshot_load(LoadSnapshotClosure* done) { } void FSMCaller::do_snapshot_load(LoadSnapshotClosure* done) { - //TODO done_guard + // TODO done_guard SnapshotReader* reader = done->start(); if (!reader) { done->status().set_error(EINVAL, "open SnapshotReader failed"); @@ -388,17 +383,20 @@ void FSMCaller::do_snapshot_load(LoadSnapshotClosure* done) { } LogId last_applied_id; - last_applied_id.index = _last_applied_index.load(butil::memory_order_relaxed); + last_applied_id.index = + _last_applied_index.load(butil::memory_order_relaxed); last_applied_id.term = _last_applied_term; LogId snapshot_id; snapshot_id.index = meta.last_included_index(); snapshot_id.term = meta.last_included_term(); if (last_applied_id > snapshot_id) { - done->status().set_error(ESTALE,"Loading a stale snapshot" - " last_applied_index=%" PRId64 " last_applied_term=%" PRId64 - " snapshot_index=%" PRId64 " snapshot_term=%" PRId64, - last_applied_id.index, last_applied_id.term, - snapshot_id.index, snapshot_id.term); + done->status().set_error( + ESTALE, + "Loading a stale snapshot" + " last_applied_index=%" PRId64 " last_applied_term=%" PRId64 + " snapshot_index=%" PRId64 " snapshot_term=%" PRId64, + last_applied_id.index, last_applied_id.term, snapshot_id.index, + snapshot_id.term); return done->Run(); } @@ -457,16 +455,19 @@ void FSMCaller::do_leader_stop(const butil::Status& status) { _fsm->on_leader_stop(status); } -void FSMCaller::do_leader_start(const LeaderStartContext& leader_start_context) { +void FSMCaller::do_leader_start( + const LeaderStartContext& leader_start_context) { _node->leader_lease_start(leader_start_context.lease_epoch); _fsm->on_leader_start(leader_start_context.term); } -int FSMCaller::on_start_following(const LeaderChangeContext& start_following_context) { +int FSMCaller::on_start_following( + const LeaderChangeContext& start_following_context) { ApplyTask task; task.type = START_FOLLOWING; - LeaderChangeContext* context = new LeaderChangeContext(start_following_context.leader_id(), - start_following_context.term(), start_following_context.status()); + LeaderChangeContext* context = new LeaderChangeContext( + start_following_context.leader_id(), start_following_context.term(), + start_following_context.status()); task.leader_change_context = context; if (bthread::execution_queue_execute(_queue_id, task) != 0) { delete context; @@ -475,11 +476,13 @@ int FSMCaller::on_start_following(const LeaderChangeContext& start_following_con return 0; } -int FSMCaller::on_stop_following(const LeaderChangeContext& stop_following_context) { +int FSMCaller::on_stop_following( + const LeaderChangeContext& stop_following_context) { ApplyTask task; task.type = STOP_FOLLOWING; - LeaderChangeContext* context = new LeaderChangeContext(stop_following_context.leader_id(), - stop_following_context.term(), stop_following_context.status()); + LeaderChangeContext* context = new LeaderChangeContext( + stop_following_context.leader_id(), stop_following_context.term(), + stop_following_context.status()); task.leader_change_context = context; if (bthread::execution_queue_execute(_queue_id, task) != 0) { delete context; @@ -488,48 +491,50 @@ int FSMCaller::on_stop_following(const LeaderChangeContext& stop_following_conte return 0; } -void FSMCaller::do_start_following(const LeaderChangeContext& start_following_context) { +void FSMCaller::do_start_following( + const LeaderChangeContext& start_following_context) { _fsm->on_start_following(start_following_context); } -void FSMCaller::do_stop_following(const LeaderChangeContext& stop_following_context) { +void FSMCaller::do_stop_following( + const LeaderChangeContext& stop_following_context) { _fsm->on_stop_following(stop_following_context); } -void FSMCaller::describe(std::ostream &os, bool use_html) { +void FSMCaller::describe(std::ostream& os, bool use_html) { const char* newline = (use_html) ? "
" : "\n"; TaskType cur_task = _cur_task; - const int64_t applying_index = _applying_index.load( - butil::memory_order_relaxed); + const int64_t applying_index = + _applying_index.load(butil::memory_order_relaxed); os << "state_machine: "; switch (cur_task) { - case IDLE: - os << "Idle"; - break; - case COMMITTED: - os << "Applying log_index=" << applying_index; - break; - case SNAPSHOT_SAVE: - os << "Saving snapshot"; - break; - case SNAPSHOT_LOAD: - os << "Loading snapshot"; - break; - case ERROR: - os << "Notifying error"; - break; - case LEADER_STOP: - os << "Notifying leader stop"; - break; - case LEADER_START: - os << "Notifying leader start"; - break; - case START_FOLLOWING: - os << "Notifying start following"; - break; - case STOP_FOLLOWING: - os << "Notifying stop following"; - break; + case IDLE: + os << "Idle"; + break; + case COMMITTED: + os << "Applying log_index=" << applying_index; + break; + case SNAPSHOT_SAVE: + os << "Saving snapshot"; + break; + case SNAPSHOT_LOAD: + os << "Loading snapshot"; + break; + case ERROR: + os << "Notifying error"; + break; + case LEADER_STOP: + os << "Notifying leader stop"; + break; + case LEADER_START: + os << "Notifying leader start"; + break; + case START_FOLLOWING: + os << "Notifying start following"; + break; + case STOP_FOLLOWING: + os << "Notifying stop following"; + break; } os << newline; } @@ -551,20 +556,20 @@ void FSMCaller::join() { } IteratorImpl::IteratorImpl(StateMachine* sm, LogManager* lm, - std::vector *closure, - int64_t first_closure_index, - int64_t last_applied_index, - int64_t committed_index, - butil::atomic* applying_index) - : _sm(sm) - , _lm(lm) - , _closure(closure) - , _first_closure_index(first_closure_index) - , _cur_index(last_applied_index) - , _committed_index(committed_index) - , _cur_entry(NULL) - , _applying_index(applying_index) -{ next(); } + std::vector* closure, + int64_t first_closure_index, + int64_t last_applied_index, int64_t committed_index, + butil::atomic* applying_index) + : _sm(sm), + _lm(lm), + _closure(closure), + _first_closure_index(first_closure_index), + _cur_index(last_applied_index), + _committed_index(committed_index), + _cur_entry(NULL), + _applying_index(applying_index) { + next(); +} void IteratorImpl::next() { if (_cur_entry) { @@ -578,9 +583,9 @@ void IteratorImpl::next() { if (_cur_entry == NULL) { _error.set_type(ERROR_TYPE_LOG); _error.status().set_error(-1, - "Fail to get entry at index=%" PRId64 - " while committed_index=%" PRId64, - _cur_index, _committed_index); + "Fail to get entry at index=%" PRId64 + " while committed_index=%" PRId64, + _cur_index, _committed_index); } _applying_index->store(_cur_index, butil::memory_order_relaxed); } @@ -594,8 +599,8 @@ Closure* IteratorImpl::done() const { return (*_closure)[_cur_index - _first_closure_index]; } -void IteratorImpl::set_error_and_rollback( - size_t ntail, const butil::Status* st) { +void IteratorImpl::set_error_and_rollback(size_t ntail, + const butil::Status* st) { if (ntail == 0) { CHECK(false) << "Invalid ntail=" << ntail; return; @@ -610,15 +615,16 @@ void IteratorImpl::set_error_and_rollback( _cur_entry = NULL; } _error.set_type(ERROR_TYPE_STATE_MACHINE); - _error.status().set_error(ESTATEMACHINE, - "StateMachine meet critical error when applying one " - " or more tasks since index=%" PRId64 ", %s", _cur_index, - (st ? st->error_cstr() : "none")); + _error.status().set_error( + ESTATEMACHINE, + "StateMachine meet critical error when applying one " + " or more tasks since index=%" PRId64 ", %s", + _cur_index, (st ? st->error_cstr() : "none")); } void IteratorImpl::run_the_rest_closure_with_error() { for (int64_t i = std::max(_cur_index, _first_closure_index); - i <= _committed_index; ++i) { + i <= _committed_index; ++i) { Closure* done = (*_closure)[i - _first_closure_index]; if (done) { done->status() = _error.status(); diff --git a/src/braft/fsm_caller.h b/src/braft/fsm_caller.h index 897a50f..c5c52f1 100644 --- a/src/braft/fsm_caller.h +++ b/src/braft/fsm_caller.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,17 +16,18 @@ // Wang,Yao(wangyao02@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#ifndef BRAFT_FSM_CALLER_H -#define BRAFT_FSM_CALLER_H +#ifndef BRAFT_FSM_CALLER_H +#define BRAFT_FSM_CALLER_H -#include // BAIDU_CACHELINE_ALIGNMENT #include #include +#include // BAIDU_CACHELINE_ALIGNMENT + #include "braft/ballot_box.h" #include "braft/closure_queue.h" -#include "braft/macros.h" -#include "braft/log_entry.h" #include "braft/lease.h" +#include "braft/log_entry.h" +#include "braft/macros.h" namespace braft { @@ -41,29 +42,31 @@ class LeaderChangeContext; // Backing implementation of Iterator class IteratorImpl { DISALLOW_COPY_AND_ASSIGN(IteratorImpl); -public: + + public: // Move to the next void next(); LogEntry* entry() const { return _cur_entry; } - bool is_good() const { return _cur_index <= _committed_index && !has_error(); } + bool is_good() const { + return _cur_index <= _committed_index && !has_error(); + } Closure* done() const; void set_error_and_rollback(size_t ntail, const butil::Status* st); bool has_error() const { return _error.type() != ERROR_TYPE_NONE; } const Error& error() const { return _error; } int64_t index() const { return _cur_index; } void run_the_rest_closure_with_error(); -private: - IteratorImpl(StateMachine* sm, LogManager* lm, - std::vector *closure, - int64_t first_closure_index, - int64_t last_applied_index, - int64_t committed_index, + + private: + IteratorImpl(StateMachine* sm, LogManager* lm, + std::vector* closure, int64_t first_closure_index, + int64_t last_applied_index, int64_t committed_index, butil::atomic* applying_index); ~IteratorImpl() {} -friend class FSMCaller; + friend class FSMCaller; StateMachine* _sm; LogManager* _lm; - std::vector *_closure; + std::vector* _closure; int64_t _first_closure_index; int64_t _cur_index; int64_t _committed_index; @@ -73,17 +76,16 @@ friend class FSMCaller; }; struct FSMCallerOptions { - FSMCallerOptions() - : log_manager(NULL) - , fsm(NULL) - , after_shutdown(NULL) - , closure_queue(NULL) - , node(NULL) - , usercode_in_pthread(false) - , bootstrap_id() - {} - LogManager *log_manager; - StateMachine *fsm; + FSMCallerOptions() + : log_manager(NULL), + fsm(NULL), + after_shutdown(NULL), + closure_queue(NULL), + node(NULL), + usercode_in_pthread(false), + bootstrap_id() {} + LogManager* log_manager; + StateMachine* fsm; google::protobuf::Closure* after_shutdown; ClosureQueue* closure_queue; NodeImpl* node; @@ -92,19 +94,19 @@ struct FSMCallerOptions { }; class SaveSnapshotClosure : public Closure { -public: + public: // TODO: comments virtual SnapshotWriter* start(const SnapshotMeta& meta) = 0; }; class LoadSnapshotClosure : public Closure { -public: + public: // TODO: comments virtual SnapshotReader* start() = 0; }; class BAIDU_CACHELINE_ALIGNMENT FSMCaller { -public: + public: FSMCaller(); BRAFT_MOCK ~FSMCaller(); int init(const FSMCallerOptions& options); @@ -123,9 +125,9 @@ class BAIDU_CACHELINE_ALIGNMENT FSMCaller { int64_t applying_index() const; void describe(std::ostream& os, bool use_html); void join(); -private: -friend class IteratorImpl; + private: + friend class IteratorImpl; enum TaskType { IDLE, @@ -141,8 +143,7 @@ friend class IteratorImpl; struct LeaderStartContext { LeaderStartContext(int64_t term_, int64_t lease_epoch_) - : term(term_), lease_epoch(lease_epoch_) - {} + : term(term_), lease_epoch(lease_epoch_) {} int64_t term; int64_t lease_epoch; @@ -153,12 +154,12 @@ friend class IteratorImpl; union { // For applying log entry (including configuration change) int64_t committed_index; - + // For on_leader_start LeaderStartContext* leader_start_context; - + // For on_leader_stop - butil::Status* status; + butil::Status* status; // For on_start_following and on_stop_following LeaderChangeContext* leader_change_context; @@ -170,7 +171,7 @@ friend class IteratorImpl; static double get_cumulated_cpu_time(void* arg); static int run(void* meta, bthread::TaskIterator& iter); - void do_shutdown(); //Closure* done); + void do_shutdown(); // Closure* done); void do_committed(int64_t committed_index); void do_cleared(int64_t log_index, Closure* done, int error_code); void do_snapshot_save(SaveSnapshotClosure* done); @@ -184,8 +185,8 @@ friend class IteratorImpl; bool pass_by_status(Closure* done); bthread::ExecutionQueueId _queue_id; - LogManager *_log_manager; - StateMachine *_fsm; + LogManager* _log_manager; + StateMachine* _fsm; ClosureQueue* _closure_queue; butil::atomic _last_applied_index; int64_t _last_applied_term; @@ -197,6 +198,6 @@ friend class IteratorImpl; bool _queue_started; }; -}; +}; // namespace braft -#endif //BRAFT_FSM_CALLER_H +#endif // BRAFT_FSM_CALLER_H diff --git a/src/braft/fsync.cpp b/src/braft/fsync.cpp index dd1a628..2808f27 100644 --- a/src/braft/fsync.cpp +++ b/src/braft/fsync.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,15 +15,14 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) #include "braft/fsync.h" + #include //BRPC_VALIDATE_GFLAG namespace braft { -DEFINE_bool(raft_use_fsync_rather_than_fdatasync, - true, +DEFINE_bool(raft_use_fsync_rather_than_fdatasync, true, "Use fsync rather than fdatasync to flush page cache"); -BRPC_VALIDATE_GFLAG(raft_use_fsync_rather_than_fdatasync, - brpc::PassValidate); +BRPC_VALIDATE_GFLAG(raft_use_fsync_rather_than_fdatasync, brpc::PassValidate); } // namespace braft diff --git a/src/braft/fsync.h b/src/braft/fsync.h index 94fd9b9..f42f46f 100644 --- a/src/braft/fsync.h +++ b/src/braft/fsync.h @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,12 +15,13 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#ifndef BRAFT_FSYNC_H -#define BRAFT_FSYNC_H +#ifndef BRAFT_FSYNC_H +#define BRAFT_FSYNC_H -#include #include #include +#include + #include "braft/storage.h" namespace braft { @@ -39,10 +40,8 @@ inline int raft_fsync(int fd) { } } -inline bool raft_sync_meta() { - return FLAGS_raft_sync || FLAGS_raft_sync_meta; -} +inline bool raft_sync_meta() { return FLAGS_raft_sync || FLAGS_raft_sync_meta; } } // namespace braft -#endif //BRAFT_FSYNC_H +#endif // BRAFT_FSYNC_H diff --git a/src/braft/lease.cpp b/src/braft/lease.cpp index 2e06b95..88d8f66 100644 --- a/src/braft/lease.cpp +++ b/src/braft/lease.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,15 +14,17 @@ // Authors: Pengfei Zheng (zhengpengfei@baidu.com) -#include -#include #include "braft/lease.h" +#include +#include + namespace braft { -DEFINE_bool(raft_enable_leader_lease, false, - "Enable or disable leader lease. only when all peers in a raft group " - "set this configuration to true, leader lease check and vote are safe."); +DEFINE_bool( + raft_enable_leader_lease, false, + "Enable or disable leader lease. only when all peers in a raft group " + "set this configuration to true, leader lease check and vote are safe."); BRPC_VALIDATE_GFLAG(raft_enable_leader_lease, ::brpc::PassValidate); void LeaderLease::init(int64_t election_timeout_ms) { @@ -42,7 +44,8 @@ void LeaderLease::on_leader_stop() { _term = 0; } -void LeaderLease::on_lease_start(int64_t expect_lease_epoch, int64_t last_active_timestamp) { +void LeaderLease::on_lease_start(int64_t expect_lease_epoch, + int64_t last_active_timestamp) { BAIDU_SCOPED_LOCK(_mutex); if (_term == 0 || expect_lease_epoch != _lease_epoch) { return; @@ -72,7 +75,8 @@ void LeaderLease::get_lease_info(LeaseInfo* lease_info) { lease_info->state = LeaderLease::NOT_READY; return; } - if (butil::monotonic_time_ms() < _last_active_timestamp + _election_timeout_ms) { + if (butil::monotonic_time_ms() < + _last_active_timestamp + _election_timeout_ms) { lease_info->term = _term; lease_info->lease_epoch = _lease_epoch; lease_info->state = LeaderLease::VALID; @@ -91,11 +95,12 @@ void LeaderLease::reset_election_timeout_ms(int64_t election_timeout_ms) { _election_timeout_ms = election_timeout_ms; } -void FollowerLease::init(int64_t election_timeout_ms, int64_t max_clock_drift_ms) { +void FollowerLease::init(int64_t election_timeout_ms, + int64_t max_clock_drift_ms) { _election_timeout_ms = election_timeout_ms; _max_clock_drift_ms = max_clock_drift_ms; - // When the node restart, we are not sure when the lease will be expired actually, - // so just be conservative. + // When the node restart, we are not sure when the lease will be expired + // actually, so just be conservative. _last_leader_timestamp = butil::monotonic_time_ms(); } @@ -114,21 +119,19 @@ int64_t FollowerLease::votable_time_from_now() { } int64_t now = butil::monotonic_time_ms(); - int64_t votable_timestamp = _last_leader_timestamp + _election_timeout_ms + - _max_clock_drift_ms; + int64_t votable_timestamp = + _last_leader_timestamp + _election_timeout_ms + _max_clock_drift_ms; if (now >= votable_timestamp) { return 0; } return votable_timestamp - now; } -const PeerId& FollowerLease::last_leader() { - return _last_leader; -} +const PeerId& FollowerLease::last_leader() { return _last_leader; } bool FollowerLease::expired() { - return butil::monotonic_time_ms() - _last_leader_timestamp - >= _election_timeout_ms + _max_clock_drift_ms; + return butil::monotonic_time_ms() - _last_leader_timestamp >= + _election_timeout_ms + _max_clock_drift_ms; } void FollowerLease::reset() { @@ -136,9 +139,7 @@ void FollowerLease::reset() { _last_leader_timestamp = 0; } -void FollowerLease::expire() { - _last_leader_timestamp = 0; -} +void FollowerLease::expire() { _last_leader_timestamp = 0; } void FollowerLease::reset_election_timeout_ms(int64_t election_timeout_ms, int64_t max_clock_drift_ms) { @@ -146,4 +147,4 @@ void FollowerLease::reset_election_timeout_ms(int64_t election_timeout_ms, _max_clock_drift_ms = max_clock_drift_ms; } -} // namespace braft +} // namespace braft diff --git a/src/braft/lease.h b/src/braft/lease.h index 0be0f00..f874a63 100644 --- a/src/braft/lease.h +++ b/src/braft/lease.h @@ -1,11 +1,11 @@ // Copyright (c) 2019 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,7 @@ namespace braft { class LeaderLease { -public: + public: enum InternalState { DISABLED, EXPIRED, @@ -38,22 +38,22 @@ class LeaderLease { }; LeaderLease() - : _election_timeout_ms(0) - , _last_active_timestamp(0) - , _term(0) - , _lease_epoch(0) - {} + : _election_timeout_ms(0), + _last_active_timestamp(0), + _term(0), + _lease_epoch(0) {} void init(int64_t election_timeout_ms); void on_leader_start(int64_t term); void on_leader_stop(); - void on_lease_start(int64_t expect_lease_epoch, int64_t last_active_timestamp); + void on_lease_start(int64_t expect_lease_epoch, + int64_t last_active_timestamp); void get_lease_info(LeaseInfo* lease_info); void renew(int64_t last_active_timestamp); int64_t lease_epoch(); void reset_election_timeout_ms(int64_t election_timeout_ms); -private: + private: raft_mutex_t _mutex; int64_t _election_timeout_ms; int64_t _last_active_timestamp; @@ -62,11 +62,11 @@ class LeaderLease { }; class FollowerLease { -public: + public: FollowerLease() - : _election_timeout_ms(0), _max_clock_drift_ms(0) - , _last_leader_timestamp(0) - {} + : _election_timeout_ms(0), + _max_clock_drift_ms(0), + _last_leader_timestamp(0) {} void init(int64_t election_timeout_ms, int64_t max_clock_drift_ms); void renew(const PeerId& leader_id); @@ -75,16 +75,17 @@ class FollowerLease { bool expired(); void reset(); void expire(); - void reset_election_timeout_ms(int64_t election_timeout_ms, int64_t max_clock_drift_ms); + void reset_election_timeout_ms(int64_t election_timeout_ms, + int64_t max_clock_drift_ms); int64_t last_leader_timestamp(); -private: + private: int64_t _election_timeout_ms; int64_t _max_clock_drift_ms; - PeerId _last_leader; + PeerId _last_leader; int64_t _last_leader_timestamp; }; -} // namespace braft +} // namespace braft -#endif // PUBLIC_RAFT_LEADER_LEASE_H +#endif // PUBLIC_RAFT_LEADER_LEASE_H diff --git a/src/braft/log.cpp b/src/braft/log.cpp index 10a1683..5e67196 100644 --- a/src/braft/log.cpp +++ b/src/braft/log.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,26 +18,26 @@ #include "braft/log.h" -#include -#include // butil::DirReaderPosix -#include // butil::CreateDirectory -#include // butil::string_appendf +#include // +#include // butil::make_close_on_exec +#include // butil::CreateDirectory +#include // butil::DirReaderPosix +#include // butil::RawPacker +#include // butil::string_appendf #include -#include // butil::RawPacker -#include // butil::make_close_on_exec -#include // +#include +#include "braft/fsync.h" #include "braft/local_storage.pb.h" #include "braft/log_entry.h" #include "braft/protobuf_file.h" #include "braft/util.h" -#include "braft/fsync.h" //#define BRAFT_SEGMENT_OPEN_PATTERN "log_inprogress_%020ld" //#define BRAFT_SEGMENT_CLOSED_PATTERN "log_%020ld_%020ld" #define BRAFT_SEGMENT_OPEN_PATTERN "log_inprogress_%020" PRId64 #define BRAFT_SEGMENT_CLOSED_PATTERN "log_%020" PRId64 "_%020" PRId64 -#define BRAFT_SEGMENT_META_FILE "log_meta" +#define BRAFT_SEGMENT_META_FILE "log_meta" namespace braft { @@ -45,18 +45,20 @@ using ::butil::RawPacker; using ::butil::RawUnpacker; DECLARE_bool(raft_trace_append_entry_latency); -DEFINE_int32(raft_max_segment_size, 8 * 1024 * 1024 /*8M*/, +DEFINE_int32(raft_max_segment_size, 8 * 1024 * 1024 /*8M*/, "Max size of one segment file"); BRPC_VALIDATE_GFLAG(raft_max_segment_size, brpc::PositiveInteger); DEFINE_bool(raft_sync_segments, false, "call fsync when a segment is closed"); BRPC_VALIDATE_GFLAG(raft_sync_segments, ::brpc::PassValidate); -DEFINE_bool(raft_recover_log_from_corrupt, false, "recover log by truncating corrupted log"); +DEFINE_bool(raft_recover_log_from_corrupt, false, + "recover log by truncating corrupted log"); BRPC_VALIDATE_GFLAG(raft_recover_log_from_corrupt, ::brpc::PassValidate); static bvar::LatencyRecorder g_open_segment_latency("raft_open_segment"); -static bvar::LatencyRecorder g_segment_append_entry_latency("raft_segment_append_entry"); +static bvar::LatencyRecorder g_segment_append_entry_latency( + "raft_segment_append_entry"); static bvar::LatencyRecorder g_sync_segment_latency("raft_sync_segment"); int ftruncate_uninterrupted(int fd, off_t length) { @@ -69,7 +71,7 @@ int ftruncate_uninterrupted(int fd, off_t length) { enum CheckSumType { CHECKSUM_MURMURHASH32 = 0, - CHECKSUM_CRC32 = 1, + CHECKSUM_CRC32 = 1, }; enum RaftSyncPolicy { @@ -77,7 +79,6 @@ enum RaftSyncPolicy { RAFT_SYNC_BY_BYTES = 1, }; - // Format of Header, all fields are in network order // | -------------------- term (64bits) ------------------------- | // | entry-type (8bits) | checksum_type (8bits) | reserved(16bits) | @@ -95,15 +96,15 @@ struct Segment::EntryHeader { }; std::ostream& operator<<(std::ostream& os, const Segment::EntryHeader& h) { - os << "{term=" << h.term << ", type=" << h.type << ", data_len=" - << h.data_len << ", checksum_type=" << h.checksum_type + os << "{term=" << h.term << ", type=" << h.type + << ", data_len=" << h.data_len << ", checksum_type=" << h.checksum_type << ", data_checksum=" << h.data_checksum << '}'; return os; } int Segment::create() { if (!_is_open) { - CHECK(false) << "Create on a closed segment at first_index=" + CHECK(false) << "Create on a closed segment at first_index=" << _first_index << " in " << _path; return -1; } @@ -114,60 +115,60 @@ int Segment::create() { if (_fd >= 0) { butil::make_close_on_exec(_fd); } - LOG_IF(INFO, _fd >= 0) << "Created new segment `" << path - << "' with fd=" << _fd ; + LOG_IF(INFO, _fd >= 0) << "Created new segment `" << path + << "' with fd=" << _fd; return _fd >= 0 ? 0 : -1; } -inline bool verify_checksum(int checksum_type, - const char* data, size_t len, uint32_t value) { +inline bool verify_checksum(int checksum_type, const char* data, size_t len, + uint32_t value) { switch (checksum_type) { - case CHECKSUM_MURMURHASH32: - return (value == murmurhash32(data, len)); - case CHECKSUM_CRC32: - return (value == crc32(data, len)); - default: - LOG(ERROR) << "Unknown checksum_type=" << checksum_type; - return false; + case CHECKSUM_MURMURHASH32: + return (value == murmurhash32(data, len)); + case CHECKSUM_CRC32: + return (value == crc32(data, len)); + default: + LOG(ERROR) << "Unknown checksum_type=" << checksum_type; + return false; } } -inline bool verify_checksum(int checksum_type, - const butil::IOBuf& data, uint32_t value) { +inline bool verify_checksum(int checksum_type, const butil::IOBuf& data, + uint32_t value) { switch (checksum_type) { - case CHECKSUM_MURMURHASH32: - return (value == murmurhash32(data)); - case CHECKSUM_CRC32: - return (value == crc32(data)); - default: - LOG(ERROR) << "Unknown checksum_type=" << checksum_type; - return false; + case CHECKSUM_MURMURHASH32: + return (value == murmurhash32(data)); + case CHECKSUM_CRC32: + return (value == crc32(data)); + default: + LOG(ERROR) << "Unknown checksum_type=" << checksum_type; + return false; } } inline uint32_t get_checksum(int checksum_type, const char* data, size_t len) { switch (checksum_type) { - case CHECKSUM_MURMURHASH32: - return murmurhash32(data, len); - case CHECKSUM_CRC32: - return crc32(data, len); - default: - CHECK(false) << "Unknown checksum_type=" << checksum_type; - abort(); - return 0; + case CHECKSUM_MURMURHASH32: + return murmurhash32(data, len); + case CHECKSUM_CRC32: + return crc32(data, len); + default: + CHECK(false) << "Unknown checksum_type=" << checksum_type; + abort(); + return 0; } } inline uint32_t get_checksum(int checksum_type, const butil::IOBuf& data) { switch (checksum_type) { - case CHECKSUM_MURMURHASH32: - return murmurhash32(data); - case CHECKSUM_CRC32: - return crc32(data); - default: - CHECK(false) << "Unknown checksum_type=" << checksum_type; - abort(); - return 0; + case CHECKSUM_MURMURHASH32: + return murmurhash32(data); + case CHECKSUM_CRC32: + return crc32(data); + default: + CHECK(false) << "Unknown checksum_type=" << checksum_type; + abort(); + return 0; } } @@ -180,25 +181,26 @@ int Segment::_load_entry(off_t offset, EntryHeader* head, butil::IOBuf* data, return n < 0 ? -1 : 1; } char header_buf[ENTRY_HEADER_SIZE]; - const char *p = (const char *)buf.fetch(header_buf, ENTRY_HEADER_SIZE); + const char* p = (const char*)buf.fetch(header_buf, ENTRY_HEADER_SIZE); int64_t term = 0; uint32_t meta_field; uint32_t data_len = 0; uint32_t data_checksum = 0; uint32_t header_checksum = 0; - RawUnpacker(p).unpack64((uint64_t&)term) - .unpack32(meta_field) - .unpack32(data_len) - .unpack32(data_checksum) - .unpack32(header_checksum); + RawUnpacker(p) + .unpack64((uint64_t&)term) + .unpack32(meta_field) + .unpack32(data_len) + .unpack32(data_checksum) + .unpack32(header_checksum); EntryHeader tmp; tmp.term = term; tmp.type = meta_field >> 24; tmp.checksum_type = (meta_field << 8) >> 24; tmp.data_len = data_len; tmp.data_checksum = data_checksum; - if (!verify_checksum(tmp.checksum_type, - p, ENTRY_HEADER_SIZE - 4, header_checksum)) { + if (!verify_checksum(tmp.checksum_type, p, ENTRY_HEADER_SIZE - 4, + header_checksum)) { LOG(ERROR) << "Found corrupted header at offset=" << offset << ", header=" << tmp << ", path: " << _path; return -1; @@ -209,7 +211,8 @@ int Segment::_load_entry(off_t offset, EntryHeader* head, butil::IOBuf* data, if (data != NULL) { if (buf.length() < ENTRY_HEADER_SIZE + data_len) { const size_t to_read = ENTRY_HEADER_SIZE + data_len - buf.length(); - const ssize_t n = file_pread(&buf, _fd, offset + buf.length(), to_read); + const ssize_t n = + file_pread(&buf, _fd, offset + buf.length(), to_read); if (n != (ssize_t)to_read) { return n < 0 ? -1 : 1; } @@ -219,9 +222,8 @@ int Segment::_load_entry(off_t offset, EntryHeader* head, butil::IOBuf* data, CHECK_EQ(buf.length(), ENTRY_HEADER_SIZE + data_len); buf.pop_front(ENTRY_HEADER_SIZE); if (!verify_checksum(tmp.checksum_type, buf, tmp.data_checksum)) { - LOG(ERROR) << "Found corrupted data at offset=" - << offset + ENTRY_HEADER_SIZE - << " header=" << tmp + LOG(ERROR) << "Found corrupted data at offset=" + << offset + ENTRY_HEADER_SIZE << " header=" << tmp << " path: " << _path; // TODO: abort()? return -1; @@ -233,22 +235,26 @@ int Segment::_load_entry(off_t offset, EntryHeader* head, butil::IOBuf* data, int Segment::_get_meta(int64_t index, LogMeta* meta) const { BAIDU_SCOPED_LOCK(_mutex); - if (index > _last_index.load(butil::memory_order_relaxed) - || index < _first_index) { + if (index > _last_index.load(butil::memory_order_relaxed) || + index < _first_index) { // out of range - BRAFT_VLOG << "_last_index=" << _last_index.load(butil::memory_order_relaxed) - << " _first_index=" << _first_index; + BRAFT_VLOG << "_last_index=" + << _last_index.load(butil::memory_order_relaxed) + << " _first_index=" << _first_index; return -1; } else if (_last_index == _first_index - 1) { - BRAFT_VLOG << "_last_index=" << _last_index.load(butil::memory_order_relaxed) - << " _first_index=" << _first_index; + BRAFT_VLOG << "_last_index=" + << _last_index.load(butil::memory_order_relaxed) + << " _first_index=" << _first_index; // empty return -1; } int64_t meta_index = index - _first_index; int64_t entry_cursor = _offset_and_term[meta_index].first; - int64_t next_cursor = (index < _last_index.load(butil::memory_order_relaxed)) - ? _offset_and_term[meta_index + 1].first : _bytes; + int64_t next_cursor = + (index < _last_index.load(butil::memory_order_relaxed)) + ? _offset_and_term[meta_index + 1].first + : _bytes; DCHECK_LT(entry_cursor, next_cursor); meta->offset = entry_cursor; meta->term = _offset_and_term[meta_index].second; @@ -262,10 +268,11 @@ int Segment::load(ConfigurationManager* configuration_manager) { std::string path(_path); // create fd if (_is_open) { - butil::string_appendf(&path, "/" BRAFT_SEGMENT_OPEN_PATTERN, _first_index); + butil::string_appendf(&path, "/" BRAFT_SEGMENT_OPEN_PATTERN, + _first_index); } else { - butil::string_appendf(&path, "/" BRAFT_SEGMENT_CLOSED_PATTERN, - _first_index, _last_index.load()); + butil::string_appendf(&path, "/" BRAFT_SEGMENT_CLOSED_PATTERN, + _first_index, _last_index.load()); } _fd = ::open(path.c_str(), O_RDWR); if (_fd < 0) { @@ -292,7 +299,8 @@ int Segment::load(ConfigurationManager* configuration_manager) { EntryHeader header; const int rc = _load_entry(entry_off, &header, NULL, ENTRY_HEADER_SIZE); if (rc > 0) { - // The last log was not completely written, which should be truncated + // The last log was not completely written, which should be + // truncated break; } if (rc < 0) { @@ -320,10 +328,10 @@ int Segment::load(ConfigurationManager* configuration_manager) { butil::Status status = parse_configuration_meta(data, entry); if (status.ok()) { ConfigurationEntry conf_entry(*entry); - configuration_manager->add(conf_entry); + configuration_manager->add(conf_entry); } else { - LOG(ERROR) << "fail to parse configuration meta, path: " << _path - << " entry_off " << entry_off; + LOG(ERROR) << "fail to parse configuration meta, path: " + << _path << " entry_off " << entry_off; ret = -1; break; } @@ -337,22 +345,25 @@ int Segment::load(ConfigurationManager* configuration_manager) { if (ret == 0 && !_is_open) { if (actual_last_index < last_index) { LOG(ERROR) << "data lost in a full segment, path: " << _path - << " first_index: " << _first_index << " expect_last_index: " - << last_index << " actual_last_index: " << actual_last_index; + << " first_index: " << _first_index + << " expect_last_index: " << last_index + << " actual_last_index: " << actual_last_index; ret = -1; } else if (actual_last_index > last_index) { // FIXME(zhengpengfei): should we ignore garbage entries silently LOG(ERROR) << "found garbage in a full segment, path: " << _path - << " first_index: " << _first_index << " expect_last_index: " - << last_index << " actual_last_index: " << actual_last_index; + << " first_index: " << _first_index + << " expect_last_index: " << last_index + << " actual_last_index: " << actual_last_index; ret = -1; } } if (ret != 0) { - if (!_is_open || !FLAGS_raft_recover_log_from_corrupt || !is_entry_corrupted) { + if (!_is_open || !FLAGS_raft_recover_log_from_corrupt || + !is_entry_corrupted) { return ret; - // Truncate the log to the last normal index + // Truncate the log to the last normal index } else { ret = 0; } @@ -365,7 +376,8 @@ int Segment::load(ConfigurationManager* configuration_manager) { // truncate last uncompleted entry if (entry_off != file_size) { LOG(INFO) << "truncate last uncompleted write entry, path: " << _path - << " first_index: " << _first_index << " old_size: " << file_size << " new_size: " << entry_off; + << " first_index: " << _first_index + << " old_size: " << file_size << " new_size: " << entry_off; ret = ftruncate_uninterrupted(_fd, entry_off); } @@ -377,65 +389,63 @@ int Segment::load(ConfigurationManager* configuration_manager) { } int Segment::append(const LogEntry* entry) { - if (BAIDU_UNLIKELY(!entry || !_is_open)) { return EINVAL; - } else if (entry->id.index != - _last_index.load(butil::memory_order_consume) + 1) { + } else if (entry->id.index != + _last_index.load(butil::memory_order_consume) + 1) { CHECK(false) << "entry->index=" << entry->id.index - << " _last_index=" << _last_index - << " _first_index=" << _first_index; + << " _last_index=" << _last_index + << " _first_index=" << _first_index; return ERANGE; } butil::IOBuf data; switch (entry->type) { - case ENTRY_TYPE_DATA: - data.append(entry->data); - break; - case ENTRY_TYPE_NO_OP: - break; - case ENTRY_TYPE_CONFIGURATION: - { + case ENTRY_TYPE_DATA: + data.append(entry->data); + break; + case ENTRY_TYPE_NO_OP: + break; + case ENTRY_TYPE_CONFIGURATION: { butil::Status status = serialize_configuration_meta(entry, data); if (!status.ok()) { - LOG(ERROR) << "Fail to serialize ConfigurationPBMeta, path: " + LOG(ERROR) << "Fail to serialize ConfigurationPBMeta, path: " << _path; - return -1; + return -1; } - } - break; - default: - LOG(FATAL) << "unknow entry type: " << entry->type - << ", path: " << _path; - return -1; + } break; + default: + LOG(FATAL) << "unknow entry type: " << entry->type + << ", path: " << _path; + return -1; } CHECK_LE(data.length(), 1ul << 56ul); char header_buf[ENTRY_HEADER_SIZE]; - const uint32_t meta_field = (entry->type << 24 ) | (_checksum_type << 16); + const uint32_t meta_field = (entry->type << 24) | (_checksum_type << 16); RawPacker packer(header_buf); packer.pack64(entry->id.term) - .pack32(meta_field) - .pack32((uint32_t)data.length()) - .pack32(get_checksum(_checksum_type, data)); - packer.pack32(get_checksum( - _checksum_type, header_buf, ENTRY_HEADER_SIZE - 4)); + .pack32(meta_field) + .pack32((uint32_t)data.length()) + .pack32(get_checksum(_checksum_type, data)); + packer.pack32( + get_checksum(_checksum_type, header_buf, ENTRY_HEADER_SIZE - 4)); butil::IOBuf header; header.append(header_buf, ENTRY_HEADER_SIZE); const size_t to_write = header.length() + data.length(); - butil::IOBuf* pieces[2] = { &header, &data }; + butil::IOBuf* pieces[2] = {&header, &data}; size_t start = 0; ssize_t written = 0; while (written < (ssize_t)to_write) { const ssize_t n = butil::IOBuf::cut_multiple_into_file_descriptor( - _fd, pieces + start, ARRAY_SIZE(pieces) - start); + _fd, pieces + start, ARRAY_SIZE(pieces) - start); if (n < 0) { - LOG(ERROR) << "Fail to write to fd=" << _fd - << ", path: " << _path << berror(); + LOG(ERROR) << "Fail to write to fd=" << _fd << ", path: " << _path + << berror(); return -1; } written += n; - for (;start < ARRAY_SIZE(pieces) && pieces[start]->empty(); ++start) {} + for (; start < ARRAY_SIZE(pieces) && pieces[start]->empty(); ++start) { + } } BAIDU_SCOPED_LOCK(_mutex); _offset_and_term.push_back(std::make_pair(_bytes, entry->id.term)); @@ -450,14 +460,13 @@ int Segment::sync(bool will_sync, bool has_conf) { if (_last_index < _first_index) { return 0; } - //CHECK(_is_open); + // CHECK(_is_open); if (will_sync) { if (!FLAGS_raft_sync) { return 0; } - if (FLAGS_raft_sync_policy == RaftSyncPolicy::RAFT_SYNC_BY_BYTES - && FLAGS_raft_sync_per_bytes > _unsynced_bytes - && !has_conf) { + if (FLAGS_raft_sync_policy == RaftSyncPolicy::RAFT_SYNC_BY_BYTES && + FLAGS_raft_sync_per_bytes > _unsynced_bytes && !has_conf) { return 0; } _unsynced_bytes = 0; @@ -467,7 +476,6 @@ int Segment::sync(bool will_sync, bool has_conf) { } LogEntry* Segment::get(const int64_t index) const { - LogMeta meta; if (_get_meta(index, &meta) != 0) { return NULL; @@ -479,8 +487,7 @@ LogEntry* Segment::get(const int64_t index) const { ConfigurationPBMeta configuration_meta; EntryHeader header; butil::IOBuf data; - if (_load_entry(meta.offset, &header, &data, - meta.length) != 0) { + if (_load_entry(meta.offset, &header, &data, meta.length) != 0) { ok = false; break; } @@ -488,29 +495,27 @@ LogEntry* Segment::get(const int64_t index) const { entry = new LogEntry(); entry->AddRef(); switch (header.type) { - case ENTRY_TYPE_DATA: - entry->data.swap(data); - break; - case ENTRY_TYPE_NO_OP: - CHECK(data.empty()) << "Data of NO_OP must be empty"; - break; - case ENTRY_TYPE_CONFIGURATION: - { - butil::Status status = parse_configuration_meta(data, entry); + case ENTRY_TYPE_DATA: + entry->data.swap(data); + break; + case ENTRY_TYPE_NO_OP: + CHECK(data.empty()) << "Data of NO_OP must be empty"; + break; + case ENTRY_TYPE_CONFIGURATION: { + butil::Status status = parse_configuration_meta(data, entry); if (!status.ok()) { - LOG(WARNING) << "Fail to parse ConfigurationPBMeta, path: " - << _path; + LOG(WARNING) + << "Fail to parse ConfigurationPBMeta, path: " << _path; ok = false; break; } - } - break; - default: - CHECK(false) << "Unknown entry type, path: " << _path; - break; + } break; + default: + CHECK(false) << "Unknown entry type, path: " << _path; + break; } - if (!ok) { + if (!ok) { break; } entry->id.index = index; @@ -535,20 +540,19 @@ int64_t Segment::get_term(const int64_t index) const { int Segment::close(bool will_sync) { CHECK(_is_open); - + std::string old_path(_path); butil::string_appendf(&old_path, "/" BRAFT_SEGMENT_OPEN_PATTERN, - _first_index); + _first_index); std::string new_path(_path); - butil::string_appendf(&new_path, "/" BRAFT_SEGMENT_CLOSED_PATTERN, - _first_index, _last_index.load()); + butil::string_appendf(&new_path, "/" BRAFT_SEGMENT_CLOSED_PATTERN, + _first_index, _last_index.load()); // TODO: optimize index memory usage by reconstruct vector - LOG(INFO) << "close a full segment. Current first_index: " << _first_index - << " last_index: " << _last_index - << " raft_sync_segments: " << FLAGS_raft_sync_segments - << " will_sync: " << will_sync - << " path: " << new_path; + LOG(INFO) << "close a full segment. Current first_index: " << _first_index + << " last_index: " << _last_index + << " raft_sync_segments: " << FLAGS_raft_sync_segments + << " will_sync: " << will_sync << " path: " << new_path; int ret = 0; if (_last_index > _first_index) { if (FLAGS_raft_sync_segments && will_sync) { @@ -558,11 +562,10 @@ int Segment::close(bool will_sync) { if (ret == 0) { _is_open = false; const int rc = ::rename(old_path.c_str(), new_path.c_str()); - LOG_IF(INFO, rc == 0) << "Renamed `" << old_path - << "' to `" << new_path <<'\''; - LOG_IF(ERROR, rc != 0) << "Fail to rename `" << old_path - << "' to `" << new_path <<"\', " - << berror(); + LOG_IF(INFO, rc == 0) + << "Renamed `" << old_path << "' to `" << new_path << '\''; + LOG_IF(ERROR, rc != 0) << "Fail to rename `" << old_path << "' to `" + << new_path << "\', " << berror(); return rc; } return ret; @@ -570,19 +573,21 @@ int Segment::close(bool will_sync) { std::string Segment::file_name() { if (!_is_open) { - return butil::string_printf(BRAFT_SEGMENT_CLOSED_PATTERN, _first_index, _last_index.load()); + return butil::string_printf(BRAFT_SEGMENT_CLOSED_PATTERN, _first_index, + _last_index.load()); } else { return butil::string_printf(BRAFT_SEGMENT_OPEN_PATTERN, _first_index); } } static void* run_unlink(void* arg) { - std::string* file_path = (std::string*) arg; + std::string* file_path = (std::string*)arg; butil::Timer timer; timer.start(); int ret = ::unlink(file_path->c_str()); timer.stop(); - BRAFT_VLOG << "unlink " << *file_path << " ret " << ret << " time: " << timer.u_elapsed(); + BRAFT_VLOG << "unlink " << *file_path << " ret " << ret + << " time: " << timer.u_elapsed(); delete file_path; return NULL; @@ -594,10 +599,10 @@ int Segment::unlink() { std::string path(_path); if (_is_open) { butil::string_appendf(&path, "/" BRAFT_SEGMENT_OPEN_PATTERN, - _first_index); + _first_index); } else { butil::string_appendf(&path, "/" BRAFT_SEGMENT_CLOSED_PATTERN, - _first_index, _last_index.load()); + _first_index, _last_index.load()); } std::string tmp_path(path); @@ -612,7 +617,8 @@ int Segment::unlink() { // TODO unlink follow control std::string* file_path = new std::string(tmp_path); bthread_t tid; - if (bthread_start_background(&tid, &BTHREAD_ATTR_NORMAL, run_unlink, file_path) != 0) { + if (bthread_start_background(&tid, &BTHREAD_ATTR_NORMAL, run_unlink, + file_path) != 0) { run_unlink(file_path); } @@ -632,23 +638,23 @@ int Segment::truncate(const int64_t last_index_kept) { first_truncate_in_offset = last_index_kept + 1 - _first_index; truncate_size = _offset_and_term[first_truncate_in_offset].first; BRAFT_VLOG << "Truncating " << _path << " first_index: " << _first_index - << " last_index from " << _last_index << " to " << last_index_kept - << " truncate size to " << truncate_size; + << " last_index from " << _last_index << " to " + << last_index_kept << " truncate size to " << truncate_size; lck.unlock(); - // Truncate on a full segment need to rename back to inprogess segment again, - // because the node may crash before truncate. + // Truncate on a full segment need to rename back to inprogess segment + // again, because the node may crash before truncate. if (!_is_open) { std::string old_path(_path); butil::string_appendf(&old_path, "/" BRAFT_SEGMENT_CLOSED_PATTERN, - _first_index, _last_index.load()); + _first_index, _last_index.load()); std::string new_path(_path); butil::string_appendf(&new_path, "/" BRAFT_SEGMENT_OPEN_PATTERN, - _first_index); + _first_index); int ret = ::rename(old_path.c_str(), new_path.c_str()); - LOG_IF(INFO, ret == 0) << "Renamed `" << old_path << "' to `" - << new_path << '\''; + LOG_IF(INFO, ret == 0) + << "Renamed `" << old_path << "' to `" << new_path << '\''; LOG_IF(ERROR, ret != 0) << "Fail to rename `" << old_path << "' to `" << new_path << "', " << berror(); if (ret != 0) { @@ -666,8 +672,8 @@ int Segment::truncate(const int64_t last_index_kept) { // seek fd off_t ret_off = ::lseek(_fd, truncate_size, SEEK_SET); if (ret_off < 0) { - PLOG(ERROR) << "Fail to lseek fd=" << _fd << " to size=" << truncate_size - << " path: " << _path; + PLOG(ERROR) << "Fail to lseek fd=" << _fd + << " to size=" << truncate_size << " path: " << _path; return -1; } @@ -681,24 +687,27 @@ int Segment::truncate(const int64_t last_index_kept) { int SegmentLogStorage::init(ConfigurationManager* configuration_manager) { if (FLAGS_raft_max_segment_size < 0) { - LOG(FATAL) << "FLAGS_raft_max_segment_size " << FLAGS_raft_max_segment_size + LOG(FATAL) << "FLAGS_raft_max_segment_size " + << FLAGS_raft_max_segment_size << " must be greater than or equal to 0 "; - return -1; + return -1; } butil::FilePath dir_path(_path); butil::File::Error e; if (!butil::CreateDirectoryAndGetError( - dir_path, &e, FLAGS_raft_create_parent_directories)) { + dir_path, &e, FLAGS_raft_create_parent_directories)) { LOG(ERROR) << "Fail to create " << dir_path.value() << " : " << e; return -1; } if (butil::crc32c::IsFastCrc32Supported()) { _checksum_type = CHECKSUM_CRC32; - LOG_ONCE(INFO) << "Use crc32c as the checksum type of appending entries"; + LOG_ONCE(INFO) + << "Use crc32c as the checksum type of appending entries"; } else { _checksum_type = CHECKSUM_MURMURHASH32; - LOG_ONCE(INFO) << "Use murmurhash32 as the checksum type of appending entries"; + LOG_ONCE(INFO) + << "Use murmurhash32 as the checksum type of appending entries"; } int ret = 0; @@ -735,14 +744,16 @@ int64_t SegmentLogStorage::last_log_index() { return _last_log_index.load(butil::memory_order_acquire); } -int SegmentLogStorage::append_entries(const std::vector& entries, IOMetric* metric) { +int SegmentLogStorage::append_entries(const std::vector& entries, + IOMetric* metric) { if (entries.empty()) { return 0; } - if (_last_log_index.load(butil::memory_order_relaxed) + 1 - != entries.front()->id.index) { - LOG(FATAL) << "There's gap between appending entries and _last_log_index" - << " path: " << _path; + if (_last_log_index.load(butil::memory_order_relaxed) + 1 != + entries.front()->id.index) { + LOG(FATAL) + << "There's gap between appending entries and _last_log_index" + << " path: " << _path; return -1; } scoped_refptr last_segment = NULL; @@ -752,7 +763,7 @@ int SegmentLogStorage::append_entries(const std::vector& entries, IOM for (size_t i = 0; i < entries.size(); i++) { now = butil::cpuwide_time_us(); LogEntry* entry = entries[i]; - + scoped_refptr segment = open_segment(); if (FLAGS_raft_trace_append_entry_latency && metric) { delta_time_us = butil::cpuwide_time_us() - now; @@ -782,7 +793,7 @@ int SegmentLogStorage::append_entries(const std::vector& entries, IOM if (FLAGS_raft_trace_append_entry_latency && metric) { delta_time_us = butil::cpuwide_time_us() - now; metric->sync_segment_time_us += delta_time_us; - g_sync_segment_latency << delta_time_us; + g_sync_segment_latency << delta_time_us; } return entries.size(); } @@ -821,8 +832,8 @@ int64_t SegmentLogStorage::get_term(const int64_t index) { } void SegmentLogStorage::pop_segments( - const int64_t first_index_kept, - std::vector >* popped) { + const int64_t first_index_kept, + std::vector >* popped) { popped->clear(); popped->reserve(32); BAIDU_SCOPED_LOCK(_mutex); @@ -853,18 +864,18 @@ void SegmentLogStorage::pop_segments( int SegmentLogStorage::truncate_prefix(const int64_t first_index_kept) { // segment files - if (_first_log_index.load(butil::memory_order_acquire) >= first_index_kept) { - BRAFT_VLOG << "Nothing is going to happen since _first_log_index=" - << _first_log_index.load(butil::memory_order_relaxed) - << " >= first_index_kept=" - << first_index_kept; + if (_first_log_index.load(butil::memory_order_acquire) >= + first_index_kept) { + BRAFT_VLOG << "Nothing is going to happen since _first_log_index=" + << _first_log_index.load(butil::memory_order_relaxed) + << " >= first_index_kept=" << first_index_kept; return 0; } - // NOTE: truncate_prefix is not important, as it has nothing to do with + // NOTE: truncate_prefix is not important, as it has nothing to do with // consensus. We try to save meta on the disk first to make sure even if // the deleting fails or the process crashes (which is unlikely to happen). // The new process would see the latest `first_log_index' - if (save_meta(first_index_kept) != 0) { // NOTE + if (save_meta(first_index_kept) != 0) { // NOTE PLOG(ERROR) << "Fail to save meta, path: " << _path; return -1; } @@ -878,9 +889,8 @@ int SegmentLogStorage::truncate_prefix(const int64_t first_index_kept) { } void SegmentLogStorage::pop_segments_from_back( - const int64_t last_index_kept, - std::vector >* popped, - scoped_refptr* last_segment) { + const int64_t last_index_kept, std::vector >* popped, + scoped_refptr* last_segment) { popped->clear(); popped->reserve(32); *last_segment = NULL; @@ -894,15 +904,15 @@ void SegmentLogStorage::pop_segments_from_back( popped->push_back(_open_segment); _open_segment = NULL; } - for (SegmentMap::reverse_iterator - it = _segments.rbegin(); it != _segments.rend(); ++it) { + for (SegmentMap::reverse_iterator it = _segments.rbegin(); + it != _segments.rend(); ++it) { if (it->second->first_index() <= last_index_kept) { // Not return as we need to maintain _segments at the end of this // routine break; } popped->push_back(it->second); - //XXX: C++03 not support erase reverse_iterator + // XXX: C++03 not support erase reverse_iterator } for (size_t i = 0; i < popped->size(); i++) { _segments.erase((*popped)[i]->first_index()); @@ -912,7 +922,8 @@ void SegmentLogStorage::pop_segments_from_back( } else { // all the logs have been cleared, the we move _first_log_index to the // next index - _first_log_index.store(last_index_kept + 1, butil::memory_order_release); + _first_log_index.store(last_index_kept + 1, + butil::memory_order_release); } } @@ -940,8 +951,8 @@ int SegmentLogStorage::truncate_suffix(const int64_t last_index_kept) { } } - // The truncate suffix order is crucial to satisfy log matching property of raft - // log must be truncated from back to front. + // The truncate suffix order is crucial to satisfy log matching property of + // raft log must be truncated from back to front. for (size_t i = 0; i < popped.size(); ++i) { ret = popped[i]->unlink(); if (ret != 0) { @@ -972,8 +983,8 @@ int SegmentLogStorage::reset(const int64_t next_log_index) { std::vector > popped; std::unique_lock lck(_mutex); popped.reserve(_segments.size()); - for (SegmentMap::const_iterator - it = _segments.begin(); it != _segments.end(); ++it) { + for (SegmentMap::const_iterator it = _segments.begin(); + it != _segments.end(); ++it) { popped.push_back(it->second); } _segments.clear(); @@ -1007,8 +1018,10 @@ int SegmentLogStorage::list_segments(bool is_empty) { // restore segment meta while (dir_reader.Next()) { // unlink unneed segments and unfinished unlinked segments - if ((is_empty && 0 == strncmp(dir_reader.name(), "log_", strlen("log_"))) || - (0 == strncmp(dir_reader.name() + (strlen(dir_reader.name()) - strlen(".tmp")), + if ((is_empty && + 0 == strncmp(dir_reader.name(), "log_", strlen("log_"))) || + (0 == strncmp(dir_reader.name() + + (strlen(dir_reader.name()) - strlen(".tmp")), ".tmp", strlen(".tmp")))) { std::string segment_path(_path); segment_path.append("/"); @@ -1023,28 +1036,29 @@ int SegmentLogStorage::list_segments(bool is_empty) { int match = 0; int64_t first_index = 0; int64_t last_index = 0; - match = sscanf(dir_reader.name(), BRAFT_SEGMENT_CLOSED_PATTERN, + match = sscanf(dir_reader.name(), BRAFT_SEGMENT_CLOSED_PATTERN, &first_index, &last_index); if (match == 2) { LOG(INFO) << "restore closed segment, path: " << _path << " first_index: " << first_index << " last_index: " << last_index; - Segment* segment = new Segment(_path, first_index, last_index, _checksum_type); + Segment* segment = + new Segment(_path, first_index, last_index, _checksum_type); _segments[first_index] = segment; continue; } - match = sscanf(dir_reader.name(), BRAFT_SEGMENT_OPEN_PATTERN, - &first_index); + match = + sscanf(dir_reader.name(), BRAFT_SEGMENT_OPEN_PATTERN, &first_index); if (match == 1) { BRAFT_VLOG << "restore open segment, path: " << _path - << " first_index: " << first_index; + << " first_index: " << first_index; if (!_open_segment) { _open_segment = new Segment(_path, first_index, _checksum_type); continue; } else { LOG(WARNING) << "open segment conflict, path: " << _path - << " first_index: " << first_index; + << " first_index: " << first_index; return -1; } } @@ -1053,33 +1067,35 @@ int SegmentLogStorage::list_segments(bool is_empty) { // check segment int64_t last_log_index = -1; SegmentMap::iterator it; - for (it = _segments.begin(); it != _segments.end(); ) { + for (it = _segments.begin(); it != _segments.end();) { Segment* segment = it->second.get(); if (segment->first_index() > segment->last_index()) { LOG(WARNING) << "closed segment is bad, path: " << _path - << " first_index: " << segment->first_index() - << " last_index: " << segment->last_index(); + << " first_index: " << segment->first_index() + << " last_index: " << segment->last_index(); return -1; } else if (last_log_index != -1 && segment->first_index() != last_log_index + 1) { LOG(WARNING) << "closed segment not in order, path: " << _path - << " first_index: " << segment->first_index() - << " last_log_index: " << last_log_index; + << " first_index: " << segment->first_index() + << " last_log_index: " << last_log_index; return -1; } else if (last_log_index == -1 && - _first_log_index.load(butil::memory_order_acquire) - < segment->first_index()) { + _first_log_index.load(butil::memory_order_acquire) < + segment->first_index()) { LOG(WARNING) << "closed segment has hole, path: " << _path - << " first_log_index: " << _first_log_index.load(butil::memory_order_relaxed) - << " first_index: " << segment->first_index() - << " last_index: " << segment->last_index(); + << " first_log_index: " + << _first_log_index.load(butil::memory_order_relaxed) + << " first_index: " << segment->first_index() + << " last_index: " << segment->last_index(); return -1; } else if (last_log_index == -1 && _first_log_index > segment->last_index()) { LOG(WARNING) << "closed segment need discard, path: " << _path - << " first_log_index: " << _first_log_index.load(butil::memory_order_relaxed) - << " first_index: " << segment->first_index() - << " last_index: " << segment->last_index(); + << " first_log_index: " + << _first_log_index.load(butil::memory_order_relaxed) + << " first_index: " << segment->first_index() + << " last_index: " << segment->last_index(); segment->unlink(); _segments.erase(it++); continue; @@ -1090,14 +1106,18 @@ int SegmentLogStorage::list_segments(bool is_empty) { } if (_open_segment) { if (last_log_index == -1 && - _first_log_index.load(butil::memory_order_relaxed) < _open_segment->first_index()) { - LOG(WARNING) << "open segment has hole, path: " << _path - << " first_log_index: " << _first_log_index.load(butil::memory_order_relaxed) - << " first_index: " << _open_segment->first_index(); - } else if (last_log_index != -1 && _open_segment->first_index() != last_log_index + 1) { + _first_log_index.load(butil::memory_order_relaxed) < + _open_segment->first_index()) { + LOG(WARNING) << "open segment has hole, path: " << _path + << " first_log_index: " + << _first_log_index.load(butil::memory_order_relaxed) + << " first_index: " << _open_segment->first_index(); + } else if (last_log_index != -1 && + _open_segment->first_index() != last_log_index + 1) { LOG(WARNING) << "open segment has hole, path: " << _path - << " first_log_index: " << _first_log_index.load(butil::memory_order_relaxed) - << " first_index: " << _open_segment->first_index(); + << " first_log_index: " + << _first_log_index.load(butil::memory_order_relaxed) + << " first_index: " << _open_segment->first_index(); } CHECK_LE(last_log_index, _open_segment->last_index()); } @@ -1105,7 +1125,8 @@ int SegmentLogStorage::list_segments(bool is_empty) { return 0; } -int SegmentLogStorage::load_segments(ConfigurationManager* configuration_manager) { +int SegmentLogStorage::load_segments( + ConfigurationManager* configuration_manager) { int ret = 0; // closed segments @@ -1113,33 +1134,34 @@ int SegmentLogStorage::load_segments(ConfigurationManager* configuration_manager for (it = _segments.begin(); it != _segments.end(); ++it) { Segment* segment = it->second.get(); LOG(INFO) << "load closed segment, path: " << _path - << " first_index: " << segment->first_index() - << " last_index: " << segment->last_index(); + << " first_index: " << segment->first_index() + << " last_index: " << segment->last_index(); ret = segment->load(configuration_manager); if (ret != 0) { return ret; - } - _last_log_index.store(segment->last_index(), butil::memory_order_release); + } + _last_log_index.store(segment->last_index(), + butil::memory_order_release); } // open segment if (_open_segment) { LOG(INFO) << "load open segment, path: " << _path - << " first_index: " << _open_segment->first_index(); + << " first_index: " << _open_segment->first_index(); ret = _open_segment->load(configuration_manager); if (ret != 0) { return ret; } if (_first_log_index.load() > _open_segment->last_index()) { LOG(WARNING) << "open segment need discard, path: " << _path - << " first_log_index: " << _first_log_index.load() - << " first_index: " << _open_segment->first_index() - << " last_index: " << _open_segment->last_index(); + << " first_log_index: " << _first_log_index.load() + << " first_index: " << _open_segment->first_index() + << " last_index: " << _open_segment->last_index(); _open_segment->unlink(); _open_segment = NULL; } else { - _last_log_index.store(_open_segment->last_index(), - butil::memory_order_release); + _last_log_index.store(_open_segment->last_index(), + butil::memory_order_release); } } if (_last_log_index == 0) { @@ -1162,7 +1184,8 @@ int SegmentLogStorage::save_meta(const int64_t log_index) { timer.stop(); PLOG_IF(ERROR, ret != 0) << "Fail to save meta to " << meta_path; - LOG(INFO) << "log save_meta " << meta_path << " first_log_index: " << log_index + LOG(INFO) << "log save_meta " << meta_path + << " first_log_index: " << log_index << " time: " << timer.u_elapsed(); return ret; } @@ -1177,14 +1200,16 @@ int SegmentLogStorage::load_meta() { ProtoBufFile pb_file(meta_path); LogPBMeta meta; if (0 != pb_file.load(&meta)) { - PLOG_IF(ERROR, errno != ENOENT) << "Fail to load meta from " << meta_path; + PLOG_IF(ERROR, errno != ENOENT) + << "Fail to load meta from " << meta_path; return -1; } _first_log_index.store(meta.first_log_index()); timer.stop(); - LOG(INFO) << "log load_meta " << meta_path << " first_log_index: " << meta.first_log_index() + LOG(INFO) << "log load_meta " << meta_path + << " first_log_index: " << meta.first_log_index() << " time: " << timer.u_elapsed(); return 0; } @@ -1194,7 +1219,8 @@ scoped_refptr SegmentLogStorage::open_segment() { { BAIDU_SCOPED_LOCK(_mutex); if (!_open_segment) { - _open_segment = new Segment(_path, last_log_index() + 1, _checksum_type); + _open_segment = + new Segment(_path, last_log_index() + 1, _checksum_type); if (_open_segment->create() != 0) { _open_segment = NULL; return NULL; @@ -1209,14 +1235,16 @@ scoped_refptr SegmentLogStorage::open_segment() { if (prev_open_segment) { if (prev_open_segment->close(_enable_sync) == 0) { BAIDU_SCOPED_LOCK(_mutex); - _open_segment = new Segment(_path, last_log_index() + 1, _checksum_type); + _open_segment = + new Segment(_path, last_log_index() + 1, _checksum_type); if (_open_segment->create() == 0) { // success break; } } - PLOG(ERROR) << "Fail to close old open_segment or create new open_segment" - << " path: " << _path; + PLOG(ERROR) + << "Fail to close old open_segment or create new open_segment" + << " path: " << _path; // Failed, revert former changes BAIDU_SCOPED_LOCK(_mutex); _segments.erase(prev_open_segment->first_index()); @@ -1235,7 +1263,8 @@ int SegmentLogStorage::get_segment(int64_t index, scoped_refptr* ptr) { return -1; } if (index < first_index || index > last_index + 1) { - LOG_IF(WARNING, index > last_index) << "Attempted to access entry " << index << " outside of log, " + LOG_IF(WARNING, index > last_index) + << "Attempted to access entry " << index << " outside of log, " << " first_log_index: " << first_index << " last_log_index: " << last_index; return -1; @@ -1260,7 +1289,8 @@ int SegmentLogStorage::get_segment(int64_t index, scoped_refptr* ptr) { void SegmentLogStorage::list_files(std::vector* seg_files) { BAIDU_SCOPED_LOCK(_mutex); seg_files->push_back(BRAFT_SEGMENT_META_FILE); - for (SegmentMap::iterator it = _segments.begin(); it != _segments.end(); ++it) { + for (SegmentMap::iterator it = _segments.begin(); it != _segments.end(); + ++it) { scoped_refptr& segment = it->second; seg_files->push_back(segment->file_name()); } @@ -1273,7 +1303,8 @@ void SegmentLogStorage::sync() { std::vector > segments; { BAIDU_SCOPED_LOCK(_mutex); - for (SegmentMap::iterator it = _segments.begin(); it != _segments.end(); ++it) { + for (SegmentMap::iterator it = _segments.begin(); it != _segments.end(); + ++it) { segments.push_back(it->second); } } @@ -1291,7 +1322,7 @@ butil::Status SegmentLogStorage::gc_instance(const std::string& uri) const { butil::Status status; if (gc_dir(uri) != 0) { LOG(WARNING) << "Failed to gc log storage from path " << _path; - status.set_error(EINVAL, "Failed to gc log storage from path %s", + status.set_error(EINVAL, "Failed to gc log storage from path %s", uri.c_str()); return status; } @@ -1299,4 +1330,4 @@ butil::Status SegmentLogStorage::gc_instance(const std::string& uri) const { return status; } -} +} // namespace braft diff --git a/src/braft/log.h b/src/braft/log.h index 145785a..d49f43e 100644 --- a/src/braft/log.h +++ b/src/braft/log.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,34 +19,43 @@ #ifndef BRAFT_LOG_H #define BRAFT_LOG_H -#include -#include -#include #include #include #include +#include + +#include +#include + #include "braft/log_entry.h" #include "braft/storage.h" #include "braft/util.h" namespace braft { -class BAIDU_CACHELINE_ALIGNMENT Segment - : public butil::RefCountedThreadSafe { -public: - Segment(const std::string& path, const int64_t first_index, int checksum_type) - : _path(path), _bytes(0), _unsynced_bytes(0), - _fd(-1), _is_open(true), - _first_index(first_index), _last_index(first_index - 1), - _checksum_type(checksum_type) - {} - Segment(const std::string& path, const int64_t first_index, const int64_t last_index, +class BAIDU_CACHELINE_ALIGNMENT Segment + : public butil::RefCountedThreadSafe { + public: + Segment(const std::string& path, const int64_t first_index, int checksum_type) - : _path(path), _bytes(0), _unsynced_bytes(0), - _fd(-1), _is_open(false), - _first_index(first_index), _last_index(last_index), - _checksum_type(checksum_type) - {} + : _path(path), + _bytes(0), + _unsynced_bytes(0), + _fd(-1), + _is_open(true), + _first_index(first_index), + _last_index(first_index - 1), + _checksum_type(checksum_type) {} + Segment(const std::string& path, const int64_t first_index, + const int64_t last_index, int checksum_type) + : _path(path), + _bytes(0), + _unsynced_bytes(0), + _fd(-1), + _is_open(false), + _first_index(first_index), + _last_index(last_index), + _checksum_type(checksum_type) {} struct EntryHeader; @@ -78,25 +87,20 @@ class BAIDU_CACHELINE_ALIGNMENT Segment // truncate segment to last_index_kept int truncate(const int64_t last_index_kept); - bool is_open() const { - return _is_open; - } + bool is_open() const { return _is_open; } - int64_t bytes() const { - return _bytes; - } + int64_t bytes() const { return _bytes; } - int64_t first_index() const { - return _first_index; - } + int64_t first_index() const { return _first_index; } int64_t last_index() const { return _last_index.load(butil::memory_order_consume); } std::string file_name(); -private: -friend class butil::RefCountedThreadSafe; + + private: + friend class butil::RefCountedThreadSafe; ~Segment() { if (_fd >= 0) { ::close(_fd); @@ -110,7 +114,7 @@ friend class butil::RefCountedThreadSafe; int64_t term; }; - int _load_entry(off_t offset, EntryHeader *head, butil::IOBuf *body, + int _load_entry(off_t offset, EntryHeader* head, butil::IOBuf* body, size_t size_hint) const; int _get_meta(int64_t index, LogMeta* meta) const; @@ -126,34 +130,34 @@ friend class butil::RefCountedThreadSafe; const int64_t _first_index; butil::atomic _last_index; int _checksum_type; - std::vector > _offset_and_term; + std::vector > + _offset_and_term; }; -// LogStorage use segmented append-only file, all data in disk, all index in memory. -// append one log entry, only cause one disk write, every disk write will call fsync(). +// LogStorage use segmented append-only file, all data in disk, all index in +// memory. append one log entry, only cause one disk write, every disk write +// will call fsync(). // // SegmentLog layout: // log_meta: record start_log // log_000001-0001000: closed segment // log_inprogress_0001001: open segment class SegmentLogStorage : public LogStorage { -public: + public: typedef std::map > SegmentMap; explicit SegmentLogStorage(const std::string& path, bool enable_sync = true) - : _path(path) - , _first_log_index(1) - , _last_log_index(0) - , _checksum_type(0) - , _enable_sync(enable_sync) - {} + : _path(path), + _first_log_index(1), + _last_log_index(0), + _checksum_type(0), + _enable_sync(enable_sync) {} SegmentLogStorage() - : _first_log_index(1) - , _last_log_index(0) - , _checksum_type(0) - , _enable_sync(true) - {} + : _first_log_index(1), + _last_log_index(0), + _checksum_type(0), + _enable_sync(true) {} virtual ~SegmentLogStorage() {} @@ -178,18 +182,20 @@ class SegmentLogStorage : public LogStorage { int append_entry(const LogEntry* entry); // append entries to log and update IOMetric, return success append number - virtual int append_entries(const std::vector& entries, IOMetric* metric); + virtual int append_entries(const std::vector& entries, + IOMetric* metric); // delete logs from storage's head, [1, first_index_kept) will be discarded virtual int truncate_prefix(const int64_t first_index_kept); - // delete uncommitted logs from storage's tail, (last_index_kept, infinity) will be discarded + // delete uncommitted logs from storage's tail, (last_index_kept, infinity) + // will be discarded virtual int truncate_suffix(const int64_t last_index_kept); virtual int reset(const int64_t next_log_index); LogStorage* new_instance(const std::string& uri) const; - + butil::Status gc_instance(const std::string& uri) const; SegmentMap segments() { @@ -200,21 +206,19 @@ class SegmentLogStorage : public LogStorage { void list_files(std::vector* seg_files); void sync(); -private: + + private: scoped_refptr open_segment(); int save_meta(const int64_t log_index); int load_meta(); int list_segments(bool is_empty); int load_segments(ConfigurationManager* configuration_manager); int get_segment(int64_t log_index, scoped_refptr* ptr); - void pop_segments( - int64_t first_index_kept, - std::vector >* poped); - void pop_segments_from_back( - const int64_t last_index_kept, - std::vector >* popped, - scoped_refptr* last_segment); - + void pop_segments(int64_t first_index_kept, + std::vector >* poped); + void pop_segments_from_back(const int64_t last_index_kept, + std::vector >* popped, + scoped_refptr* last_segment); std::string _path; butil::atomic _first_log_index; @@ -228,4 +232,4 @@ class SegmentLogStorage : public LogStorage { } // namespace braft -#endif //~BRAFT_LOG_H +#endif //~BRAFT_LOG_H diff --git a/src/braft/log_entry.cpp b/src/braft/log_entry.cpp index 540ac17..a7eb162 100644 --- a/src/braft/log_entry.cpp +++ b/src/braft/log_entry.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,13 +15,14 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) #include "braft/log_entry.h" + #include "braft/local_storage.pb.h" namespace braft { bvar::Adder g_nentries("raft_num_log_entries"); -LogEntry::LogEntry(): type(ENTRY_TYPE_UNKNOWN), peers(NULL), old_peers(NULL) { +LogEntry::LogEntry() : type(ENTRY_TYPE_UNKNOWN), peers(NULL), old_peers(NULL) { g_nentries << 1; } @@ -31,7 +32,8 @@ LogEntry::~LogEntry() { delete old_peers; } -butil::Status parse_configuration_meta(const butil::IOBuf& data, LogEntry* entry) { +butil::Status parse_configuration_meta(const butil::IOBuf& data, + LogEntry* entry) { butil::Status status; ConfigurationPBMeta meta; butil::IOBufAsZeroCopyInputStream wrapper(data); @@ -49,10 +51,11 @@ butil::Status parse_configuration_meta(const butil::IOBuf& data, LogEntry* entry entry->old_peers->push_back(PeerId(meta.old_peers(i))); } } - return status; + return status; } -butil::Status serialize_configuration_meta(const LogEntry* entry, butil::IOBuf& data) { +butil::Status serialize_configuration_meta(const LogEntry* entry, + butil::IOBuf& data) { butil::Status status; ConfigurationPBMeta meta; for (size_t i = 0; i < entry->peers->size(); ++i) { @@ -70,4 +73,4 @@ butil::Status serialize_configuration_meta(const LogEntry* entry, butil::IOBuf& return status; } -} +} // namespace braft diff --git a/src/braft/log_entry.h b/src/braft/log_entry.h index 8aa18c6..4633103 100644 --- a/src/braft/log_entry.h +++ b/src/braft/log_entry.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,12 +14,13 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_LOG_ENTRY_H -#define BRAFT_LOG_ENTRY_H +#ifndef BRAFT_LOG_ENTRY_H +#define BRAFT_LOG_ENTRY_H -#include // butil::IOBuf -#include // butil::RefCountedThreadSafe +#include // butil::IOBuf +#include // butil::RefCountedThreadSafe #include // fmix64 + #include "braft/configuration.h" #include "braft/raft.pb.h" #include "braft/util.h" @@ -36,16 +37,16 @@ struct LogId { // term start from 1, log index start from 1 struct LogEntry : public butil::RefCountedThreadSafe { -public: - EntryType type; // log type + public: + EntryType type; // log type LogId id; - std::vector* peers; // peers - std::vector* old_peers; // peers + std::vector* peers; // peers + std::vector* old_peers; // peers butil::IOBuf data; LogEntry(); -private: + private: DISALLOW_COPY_AND_ASSIGN(LogEntry); friend class butil::RefCountedThreadSafe; virtual ~LogEntry(); @@ -94,10 +95,12 @@ inline std::ostream& operator<<(std::ostream& os, const LogId& id) { return os; } -butil::Status parse_configuration_meta(const butil::IOBuf& data, LogEntry* entry); +butil::Status parse_configuration_meta(const butil::IOBuf& data, + LogEntry* entry); -butil::Status serialize_configuration_meta(const LogEntry* entry, butil::IOBuf& data); +butil::Status serialize_configuration_meta(const LogEntry* entry, + butil::IOBuf& data); } // namespace braft -#endif //BRAFT_LOG_ENTRY_H +#endif // BRAFT_LOG_ENTRY_H diff --git a/src/braft/log_manager.cpp b/src/braft/log_manager.cpp index d36c8ad..5adb136 100644 --- a/src/braft/log_manager.cpp +++ b/src/braft/log_manager.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,39 +17,39 @@ #include "braft/log_manager.h" -#include // LOG -#include // butil::get_object -#include // bthread_flush -#include // bthread::CountdownEvent -#include // BRPC_VALIDATE_GFLAG -#include "braft/storage.h" // LogStorage -#include "braft/fsm_caller.h" // FSMCaller +#include // BRPC_VALIDATE_GFLAG +#include // bthread::CountdownEvent +#include // bthread_flush +#include // LOG +#include // butil::get_object + +#include "braft/fsm_caller.h" // FSMCaller +#include "braft/storage.h" // LogStorage namespace braft { DEFINE_int32(raft_leader_batch, 256, "max leader io batch"); BRPC_VALIDATE_GFLAG(raft_leader_batch, ::brpc::PositiveInteger); -static bvar::Adder g_read_entry_from_storage - ("raft_read_entry_from_storage_count"); -static bvar::PerSecond > g_read_entry_from_storage_second - ("raft_read_entry_from_storage_second", &g_read_entry_from_storage); +static bvar::Adder g_read_entry_from_storage( + "raft_read_entry_from_storage_count"); +static bvar::PerSecond > g_read_entry_from_storage_second( + "raft_read_entry_from_storage_second", &g_read_entry_from_storage); -static bvar::Adder g_read_term_from_storage - ("raft_read_term_from_storage_count"); -static bvar::PerSecond > g_read_term_from_storage_second - ("raft_read_term_from_storage_second", &g_read_term_from_storage); +static bvar::Adder g_read_term_from_storage( + "raft_read_term_from_storage_count"); +static bvar::PerSecond > g_read_term_from_storage_second( + "raft_read_term_from_storage_second", &g_read_term_from_storage); static bvar::LatencyRecorder g_storage_append_entries_latency( - "raft_storage_append_entries"); + "raft_storage_append_entries"); static bvar::LatencyRecorder g_nomralized_append_entries_latency( - "raft_storage_append_entries_normalized"); + "raft_storage_append_entries_normalized"); static bvar::Adder g_storage_append_entries_concurrency( - "raft_storage_append_entries_concurrency"); + "raft_storage_append_entries_concurrency"); static bvar::CounterRecorder g_storage_flush_batch_counter( - "raft_storage_flush_batch_counter"); - + "raft_storage_flush_batch_counter"); void LogManager::StableClosure::update_metric(IOMetric* m) { metric.open_segment_time_us = m->open_segment_time_us; @@ -58,24 +58,20 @@ void LogManager::StableClosure::update_metric(IOMetric* m) { } LogManagerOptions::LogManagerOptions() - : log_storage(NULL) - , configuration_manager(NULL) - , fsm_caller(NULL) -{} + : log_storage(NULL), configuration_manager(NULL), fsm_caller(NULL) {} LogManager::LogManager() - : _log_storage(NULL) - , _config_manager(NULL) - , _stopped(false) - , _has_error(false) - , _next_wait_id(0) - , _first_log_index(0) - , _last_log_index(0) -{ + : _log_storage(NULL), + _config_manager(NULL), + _stopped(false), + _has_error(false), + _next_wait_id(0), + _first_log_index(0), + _last_log_index(0) { CHECK_EQ(0, start_disk_thread()); } -int LogManager::init(const LogManagerOptions &options) { +int LogManager::init(const LogManagerOptions& options) { BAIDU_SCOPED_LOCK(_mutex); if (options.log_storage == NULL) { return EINVAL; @@ -111,10 +107,8 @@ LogManager::~LogManager() { int LogManager::start_disk_thread() { bthread::ExecutionQueueOptions queue_options; queue_options.bthread_attr = BTHREAD_ATTR_NORMAL; - return bthread::execution_queue_start(&_disk_queue, - &queue_options, - disk_thread, - this); + return bthread::execution_queue_start(&_disk_queue, &queue_options, + disk_thread, this); } int LogManager::stop_disk_thread() { @@ -129,8 +123,8 @@ void LogManager::clear_memory_logs(const LogId& id) { nentries = 0; { BAIDU_SCOPED_LOCK(_mutex); - while (!_logs_in_memory.empty() - && nentries < ARRAY_SIZE(entries_to_clear)) { + while (!_logs_in_memory.empty() && + nentries < ARRAY_SIZE(entries_to_clear)) { LogEntry* entry = _logs_in_memory.front(); if (entry->id >= id) { break; @@ -151,22 +145,19 @@ int64_t LogManager::first_log_index() { } class LastLogIdClosure : public LogManager::StableClosure { -public: - LastLogIdClosure() { - } - void Run() { - _event.signal(); - } + public: + LastLogIdClosure() {} + void Run() { _event.signal(); } void set_last_log_id(const LogId& log_id) { - CHECK(log_id.index == 0 || log_id.term != 0) << "Invalid log_id=" << log_id; + CHECK(log_id.index == 0 || log_id.term != 0) + << "Invalid log_id=" << log_id; _last_log_id = log_id; } LogId last_log_id() const { return _last_log_id; } - void wait() { - _event.wait(); - } -private: + void wait() { _event.wait(); } + + private: bthread::CountdownEvent _event; LogId _last_log_id; }; @@ -207,44 +198,37 @@ LogId LogManager::last_log_id(bool is_flush) { } class TruncatePrefixClosure : public LogManager::StableClosure { -public: + public: explicit TruncatePrefixClosure(const int64_t first_index_kept) - : _first_index_kept(first_index_kept) - {} - void Run() { - delete this; - } + : _first_index_kept(first_index_kept) {} + void Run() { delete this; } int64_t first_index_kept() const { return _first_index_kept; } -private: + + private: int64_t _first_index_kept; }; class TruncateSuffixClosure : public LogManager::StableClosure { -public: + public: TruncateSuffixClosure(int64_t last_index_kept, int64_t last_term_kept) - : _last_index_kept(last_index_kept) - , _last_term_kept(last_term_kept) - {} - void Run() { - delete this; - } + : _last_index_kept(last_index_kept), _last_term_kept(last_term_kept) {} + void Run() { delete this; } int64_t last_index_kept() const { return _last_index_kept; } int64_t last_term_kept() const { return _last_term_kept; } -private: + + private: int64_t _last_index_kept; int64_t _last_term_kept; }; class ResetClosure : public LogManager::StableClosure { -public: + public: explicit ResetClosure(int64_t next_log_index) - : _next_log_index(next_log_index) - {} - void Run() { - delete this; - } + : _next_log_index(next_log_index) {} + void Run() { delete this; } int64_t next_log_index() const { return _next_log_index; } -private: + + private: int64_t _next_log_index; }; @@ -297,7 +281,8 @@ int LogManager::reset(const int64_t next_log_index, ResetClosure* c = new ResetClosure(next_log_index); const int ret = bthread::execution_queue_execute(_disk_queue, c); lck.unlock(); - CHECK_EQ(0, ret) << "execq execute failed, ret: " << ret << " err: " << berror(); + CHECK_EQ(0, ret) << "execq execute failed, ret: " << ret + << " err: " << berror(); for (size_t i = 0; i < saved_logs_in_memory.size(); ++i) { saved_logs_in_memory[i]->Release(); } @@ -305,9 +290,9 @@ int LogManager::reset(const int64_t next_log_index, } void LogManager::unsafe_truncate_suffix(const int64_t last_index_kept) { - if (last_index_kept < _applied_id.index) { - LOG(FATAL) << "Can't truncate logs before _applied_id=" <<_applied_id.index + LOG(FATAL) << "Can't truncate logs before _applied_id=" + << _applied_id.index << ", last_log_kept=" << last_index_kept; return; } @@ -326,16 +311,16 @@ void LogManager::unsafe_truncate_suffix(const int64_t last_index_kept) { CHECK(last_index_kept == 0 || last_term_kept != 0) << "last_index_kept=" << last_index_kept; _config_manager->truncate_suffix(last_index_kept); - TruncateSuffixClosure* tsc = new - TruncateSuffixClosure(last_index_kept, last_term_kept); + TruncateSuffixClosure* tsc = + new TruncateSuffixClosure(last_index_kept, last_term_kept); CHECK_EQ(0, bthread::execution_queue_execute(_disk_queue, tsc)); } -int LogManager::check_and_resolve_conflict( - std::vector *entries, StableClosure* done) { - AsyncClosureGuard done_guard(done); +int LogManager::check_and_resolve_conflict(std::vector* entries, + StableClosure* done) { + AsyncClosureGuard done_guard(done); if (entries->front()->id.index == 0) { - // Node is currently the leader and |entries| are from the user who + // Node is currently the leader and |entries| are from the user who // don't know the correct indexes the logs should assign to. So we have // to assign indexes to the appending entries for (size_t i = 0; i < entries->size(); ++i) { @@ -344,20 +329,23 @@ int LogManager::check_and_resolve_conflict( done_guard.release(); return 0; } else { - // Node is currently a follower and |entries| are from the leader. We + // Node is currently a follower and |entries| are from the leader. We // should check and resolve the confliction between the local logs and // |entries| if (entries->front()->id.index > _last_log_index + 1) { - done->status().set_error(EINVAL, "There's gap between first_index=%" PRId64 + done->status().set_error(EINVAL, + "There's gap between first_index=%" PRId64 " and last_log_index=%" PRId64, - entries->front()->id.index, _last_log_index); + entries->front()->id.index, + _last_log_index); return -1; } const int64_t applied_index = _applied_id.index; if (entries->back()->id.index <= applied_index) { LOG(WARNING) << "Received entries of which the last_log=" << entries->back()->id.index - << " is not greater than _applied_index=" << applied_index + << " is not greater than _applied_index=" + << applied_index << ", return immediately with nothing changed"; return 1; } @@ -371,29 +359,30 @@ int LogManager::check_and_resolve_conflict( // ones. size_t conflicting_index = 0; for (; conflicting_index < entries->size(); ++conflicting_index) { - if (unsafe_get_term((*entries)[conflicting_index]->id.index) - != (*entries)[conflicting_index]->id.term) { + if (unsafe_get_term((*entries)[conflicting_index]->id.index) != + (*entries)[conflicting_index]->id.term) { break; } } if (conflicting_index != entries->size()) { - if ((*entries)[conflicting_index]->id.index <= _last_log_index) { + if ((*entries)[conflicting_index]->id.index <= + _last_log_index) { // Truncate all the conflicting entries to make local logs // consensus with the leader. unsafe_truncate_suffix( - (*entries)[conflicting_index]->id.index - 1); + (*entries)[conflicting_index]->id.index - 1); } _last_log_index = entries->back()->id.index; - } // else this is a duplicated AppendEntriesRequest, we have + } // else this is a duplicated AppendEntriesRequest, we have // nothing to do besides releasing all the entries - + // Release all the entries before the conflicting_index and the rest // would be append to _logs_in_memory and _log_storage after this // function returns for (size_t i = 0; i < conflicting_index; ++i) { (*entries)[i]->Release(); } - entries->erase(entries->begin(), + entries->erase(entries->begin(), entries->begin() + conflicting_index); } done_guard.release(); @@ -404,8 +393,8 @@ int LogManager::check_and_resolve_conflict( return -1; } -void LogManager::append_entries( - std::vector *entries, StableClosure* done) { +void LogManager::append_entries(std::vector* entries, + StableClosure* done) { CHECK(done); if (_has_error.load(butil::memory_order_relaxed)) { for (size_t i = 0; i < entries->size(); ++i) { @@ -437,16 +426,18 @@ void LogManager::append_entries( if (!entries->empty()) { done->_first_log_index = entries->front()->id.index; - _logs_in_memory.insert(_logs_in_memory.end(), entries->begin(), entries->end()); + _logs_in_memory.insert(_logs_in_memory.end(), entries->begin(), + entries->end()); } done->_entries.swap(*entries); int ret = bthread::execution_queue_execute(_disk_queue, done); - CHECK_EQ(0, ret) << "execq execute failed, ret: " << ret << " err: " << berror(); + CHECK_EQ(0, ret) << "execq execute failed, ret: " << ret + << " err: " << berror(); wakeup_all_waiter(lck); } -void LogManager::append_to_storage(std::vector* to_append, +void LogManager::append_to_storage(std::vector* to_append, LogId* last_id, IOMetric* metric) { if (!_has_error.load(butil::memory_order_relaxed)) { size_t written_size = 0; @@ -462,16 +453,17 @@ void LogManager::append_to_storage(std::vector* to_append, if (nappent != (int)to_append->size()) { // FIXME LOG(ERROR) << "Fail to append_entries, " - << "nappent=" << nappent + << "nappent=" << nappent << ", to_append=" << to_append->size(); report_error(EIO, "Fail to append entries"); } - if (nappent > 0) { + if (nappent > 0) { *last_id = (*to_append)[nappent - 1]->id; } g_storage_append_entries_latency << timer.u_elapsed(); if (written_size) { - g_nomralized_append_entries_latency << timer.u_elapsed() * 1024 / written_size; + g_nomralized_append_entries_latency + << timer.u_elapsed() * 1024 / written_size; } } for (size_t j = 0; j < to_append->size(); ++j) { @@ -480,20 +472,19 @@ void LogManager::append_to_storage(std::vector* to_append, to_append->clear(); } -DEFINE_int32(raft_max_append_buffer_size, 256 * 1024, +DEFINE_int32(raft_max_append_buffer_size, 256 * 1024, "Flush buffer to LogStorage if the buffer size reaches the limit"); class AppendBatcher { -public: - AppendBatcher(LogManager::StableClosure* storage[], size_t cap, LogId* last_id, - LogManager* lm) - : _storage(storage) - , _cap(cap) - , _size(0) - , _buffer_size(0) - , _last_id(last_id) - , _lm(lm) - { + public: + AppendBatcher(LogManager::StableClosure* storage[], size_t cap, + LogId* last_id, LogManager* lm) + : _storage(storage), + _cap(cap), + _size(0), + _buffer_size(0), + _last_id(last_id), + _lm(lm) { _to_append.reserve(1024); } ~AppendBatcher() { flush(); } @@ -506,8 +497,8 @@ class AppendBatcher { for (size_t i = 0; i < _size; ++i) { _storage[i]->_entries.clear(); if (_lm->_has_error.load(butil::memory_order_relaxed)) { - _storage[i]->status().set_error( - EIO, "Corrupted LogStorage"); + _storage[i]->status().set_error(EIO, + "Corrupted LogStorage"); } _storage[i]->update_metric(&metric); _storage[i]->Run(); @@ -518,25 +509,25 @@ class AppendBatcher { _buffer_size = 0; } void append(LogManager::StableClosure* done) { - if (_size == _cap || - _buffer_size >= (size_t)FLAGS_raft_max_append_buffer_size) { + if (_size == _cap || + _buffer_size >= (size_t)FLAGS_raft_max_append_buffer_size) { flush(); } _storage[_size++] = done; - _to_append.insert(_to_append.end(), - done->_entries.begin(), done->_entries.end()); + _to_append.insert(_to_append.end(), done->_entries.begin(), + done->_entries.end()); for (size_t i = 0; i < done->_entries.size(); ++i) { _buffer_size += done->_entries[i]->data.length(); } } -private: + private: LogManager::StableClosure** _storage; size_t _cap; size_t _size; size_t _buffer_size; std::vector _to_append; - LogId *_last_id; + LogId* _last_id; LogManager* _lm; }; @@ -551,21 +542,20 @@ int LogManager::disk_thread(void* meta, LogId last_id = log_manager->_disk_id; StableClosure* storage[256]; AppendBatcher ab(storage, ARRAY_SIZE(storage), &last_id, log_manager); - + for (; iter; ++iter) { - // ^^^ Must iterate to the end to release to corresponding - // even if some error has occurred + // ^^^ Must iterate to the end to release to corresponding + // even if some error has occurred StableClosure* done = *iter; - done->metric.bthread_queue_time_us = butil::cpuwide_time_us() - - done->metric.start_time_us; + done->metric.bthread_queue_time_us = + butil::cpuwide_time_us() - done->metric.start_time_us; if (!done->_entries.empty()) { ab.append(done); } else { ab.flush(); int ret = 0; do { - LastLogIdClosure* llic = - dynamic_cast(done); + LastLogIdClosure* llic = dynamic_cast(done); if (llic) { // Not used log_manager->get_disk_id() as it might be out of // date @@ -573,28 +563,28 @@ int LogManager::disk_thread(void* meta, llic->set_last_log_id(last_id); break; } - TruncatePrefixClosure* tpc = - dynamic_cast(done); + TruncatePrefixClosure* tpc = + dynamic_cast(done); if (tpc) { BRAFT_VLOG << "Truncating storage to first_index_kept=" - << tpc->first_index_kept(); + << tpc->first_index_kept(); ret = log_manager->_log_storage->truncate_prefix( - tpc->first_index_kept()); + tpc->first_index_kept()); break; } - TruncateSuffixClosure* tsc = - dynamic_cast(done); + TruncateSuffixClosure* tsc = + dynamic_cast(done); if (tsc) { LOG(WARNING) << "Truncating storage to last_index_kept=" << tsc->last_index_kept(); ret = log_manager->_log_storage->truncate_suffix( - tsc->last_index_kept()); + tsc->last_index_kept()); if (ret == 0) { // update last_id after truncate_suffix last_id.index = tsc->last_index_kept(); last_id.term = tsc->last_term_kept(); CHECK(last_id.index == 0 || last_id.term != 0) - << "last_id=" << last_id; + << "last_id=" << last_id; } break; } @@ -602,13 +592,15 @@ int LogManager::disk_thread(void* meta, if (rc) { LOG(INFO) << "Reseting storage to next_log_index=" << rc->next_log_index(); - ret = log_manager->_log_storage->reset(rc->next_log_index()); + ret = + log_manager->_log_storage->reset(rc->next_log_index()); break; } } while (0); if (ret != 0) { - log_manager->report_error(ret, "Failed operation on LogStorage"); + log_manager->report_error(ret, + "Failed operation on LogStorage"); } done->Run(); } @@ -621,8 +613,8 @@ int LogManager::disk_thread(void* meta, void LogManager::set_snapshot(const SnapshotMeta* meta) { BRAFT_VLOG << "Set snapshot last_included_index=" - << meta->last_included_index() - << " last_included_term=" << meta->last_included_term(); + << meta->last_included_index() + << " last_included_term=" << meta->last_included_term(); std::unique_lock lck(_mutex); if (meta->last_included_index() <= _last_snapshot_id.index) { return; @@ -649,10 +641,10 @@ void LogManager::set_snapshot(const SnapshotMeta* meta) { _applied_id = _last_snapshot_id; } // NOTICE: not to update disk_id here as we are not sure if this node really - // has these logs on disk storage. Just leave disk_id as it was, which can keep - // these logs in memory all the time until they are flushed to disk. By this - // way we can avoid some corner cases which failed to get logs. - + // has these logs on disk storage. Just leave disk_id as it was, which can + // keep these logs in memory all the time until they are flushed to disk. By + // this way we can avoid some corner cases which failed to get logs. + if (term == 0) { // last_included_index is larger than last_index // FIXME: what if last_included_index is less than first_index? @@ -692,7 +684,8 @@ LogEntry* LogManager::get_entry_from_memory(const int64_t index) { if (!_logs_in_memory.empty()) { int64_t first_index = _logs_in_memory.front()->id.index; int64_t last_index = _logs_in_memory.back()->id.index; - CHECK_EQ(last_index - first_index + 1, static_cast(_logs_in_memory.size())); + CHECK_EQ(last_index - first_index + 1, + static_cast(_logs_in_memory.size())); if (index >= first_index && index <= last_index) { entry = _logs_in_memory[index - first_index]; } @@ -778,7 +771,8 @@ LogEntry* LogManager::get_entry(const int64_t index) { return entry; } -void LogManager::get_configuration(const int64_t index, ConfigurationEntry* conf) { +void LogManager::get_configuration(const int64_t index, + ConfigurationEntry* conf) { BAIDU_SCOPED_LOCK(_mutex); return _config_manager->get(index, conf); } @@ -826,16 +820,17 @@ void LogManager::shutdown() { wakeup_all_waiter(lck); } -void* LogManager::run_on_new_log(void *arg) { +void* LogManager::run_on_new_log(void* arg) { WaitMeta* wm = (WaitMeta*)arg; wm->on_new_log(wm->arg, wm->error_code); butil::return_object(wm); return NULL; } -LogManager::WaitId LogManager::wait( - int64_t expected_last_log_index, - int (*on_new_log)(void *arg, int error_code), void *arg) { +LogManager::WaitId LogManager::wait(int64_t expected_last_log_index, + int (*on_new_log)(void* arg, + int error_code), + void* arg) { WaitMeta* wm = butil::get_object(); if (BAIDU_UNLIKELY(wm == NULL)) { PLOG(FATAL) << "Fail to new WaitMeta"; @@ -849,7 +844,7 @@ LogManager::WaitId LogManager::wait( } LogManager::WaitId LogManager::notify_on_new_log( - int64_t expected_last_log_index, WaitMeta* wm) { + int64_t expected_last_log_index, WaitMeta* wm) { std::unique_lock lck(_mutex); if (expected_last_log_index != _last_log_index || _stopped) { wm->error_code = _stopped ? ESTOP : 0; @@ -891,8 +886,9 @@ void LogManager::wakeup_all_waiter(std::unique_lock& lck) { } WaitMeta* wm[_wait_map.size()]; size_t nwm = 0; - for (butil::FlatMap::const_iterator - iter = _wait_map.begin(); iter != _wait_map.end(); ++iter) { + for (butil::FlatMap::const_iterator iter = + _wait_map.begin(); + iter != _wait_map.end(); ++iter) { wm[nwm++] = iter->second; } _wait_map.clear(); @@ -902,9 +898,7 @@ void LogManager::wakeup_all_waiter(std::unique_lock& lck) { wm[i]->error_code = error_code; bthread_t tid; bthread_attr_t attr = BTHREAD_ATTR_NORMAL | BTHREAD_NOSIGNAL; - if (bthread_start_background( - &tid, &attr, - run_on_new_log, wm[i]) != 0) { + if (bthread_start_background(&tid, &attr, run_on_new_log, wm[i]) != 0) { PLOG(ERROR) << "Fail to start bthread"; run_on_new_log(wm[i]); } @@ -952,16 +946,20 @@ butil::Status LogManager::check_consistency() { if (_first_log_index == 1) { return butil::Status::OK(); } - return butil::Status(EIO, "Missing logs in (0, %" PRId64 ")", _first_log_index); + return butil::Status(EIO, "Missing logs in (0, %" PRId64 ")", + _first_log_index); } else { - if (_last_snapshot_id.index >= _first_log_index - 1 - && _last_snapshot_id.index <= _last_log_index) { + if (_last_snapshot_id.index >= _first_log_index - 1 && + _last_snapshot_id.index <= _last_log_index) { return butil::Status::OK(); } - return butil::Status(EIO, "There's a gap between snapshot={%" PRId64 ", %" PRId64 "}" - " and log=[%" PRId64 ", %" PRId64 "] ", - _last_snapshot_id.index, _last_snapshot_id.term, - _first_log_index, _last_log_index); + return butil::Status(EIO, + "There's a gap between snapshot={%" PRId64 + ", %" PRId64 + "}" + " and log=[%" PRId64 ", %" PRId64 "] ", + _last_snapshot_id.index, _last_snapshot_id.term, + _first_log_index, _last_log_index); } CHECK(false) << "Can't reach here"; return butil::Status(-1, "Impossible condition"); diff --git a/src/braft/log_manager.h b/src/braft/log_manager.h index 3e83b3c..2362076 100644 --- a/src/braft/log_manager.h +++ b/src/braft/log_manager.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,19 +15,20 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) // Wang,Yao(wangyao02@baidu.com) -#ifndef BRAFT_LOG_MANAGER_H -#define BRAFT_LOG_MANAGER_H +#ifndef BRAFT_LOG_MANAGER_H +#define BRAFT_LOG_MANAGER_H -#include // BAIDU_CACHELINE_ALIGNMENT -#include // butil::FlatMap -#include // std::deque -#include // bthread::ExecutionQueueId +#include // bthread::ExecutionQueueId +#include // butil::FlatMap +#include // BAIDU_CACHELINE_ALIGNMENT -#include "braft/raft.h" // Closure -#include "braft/util.h" // raft_mutex_t -#include "braft/log_entry.h" // LogEntry -#include "braft/configuration_manager.h" // ConfigurationManager -#include "braft/storage.h" // Storage +#include // std::deque + +#include "braft/configuration_manager.h" // ConfigurationManager +#include "braft/log_entry.h" // LogEntry +#include "braft/raft.h" // Closure +#include "braft/storage.h" // Storage +#include "braft/util.h" // raft_mutex_t namespace braft { @@ -43,8 +44,10 @@ struct LogManagerOptions { struct LogManagerStatus { LogManagerStatus() - : first_index(1), last_index(0), disk_index(0), known_applied_index(0) - {} + : first_index(1), + last_index(0), + disk_index(0), + known_applied_index(0) {} int64_t first_index; int64_t last_index; int64_t disk_index; @@ -54,19 +57,21 @@ struct LogManagerStatus { class SnapshotMeta; class BAIDU_CACHELINE_ALIGNMENT LogManager { -public: + public: typedef int64_t WaitId; class StableClosure : public Closure { - public: + public: StableClosure() : _first_log_index(0) {} void update_metric(IOMetric* metric); - protected: + + protected: int64_t _first_log_index; IOMetric metric; - private: - friend class LogManager; - friend class AppendBatcher; + + private: + friend class LogManager; + friend class AppendBatcher; std::vector _entries; }; @@ -78,7 +83,7 @@ class BAIDU_CACHELINE_ALIGNMENT LogManager { // Append log entry vector and wait until it's stable (NOT COMMITTED!) // success return 0, fail return errno - void append_entries(std::vector *entries, StableClosure* done); + void append_entries(std::vector* entries, StableClosure* done); // Notify the log manager about the latest snapshot, which indicates the // logs which can be safely truncated. @@ -108,29 +113,28 @@ class BAIDU_CACHELINE_ALIGNMENT LogManager { // Returns: // success return last memory and logstorage index, empty return 0 int64_t last_log_index(bool is_flush = false); - + // Return the id the last log. LogId last_log_id(bool is_flush = false); void get_configuration(int64_t index, ConfigurationEntry* conf); // Check if |current| should be updated to the latest configuration - // Returns true and |current| is assigned to the lastest configuration, returns - // false otherweise + // Returns true and |current| is assigned to the lastest configuration, + // returns false otherweise bool check_and_set_configuration(ConfigurationEntry* current); - // Wait until there are more logs since |last_log_index| and |on_new_log| + // Wait until there are more logs since |last_log_index| and |on_new_log| // would be called after there are new logs or error occurs WaitId wait(int64_t expected_last_log_index, - int (*on_new_log)(void *arg, int error_code), void *arg); + int (*on_new_log)(void* arg, int error_code), void* arg); // Remove a waiter // Returns: // - 0: success // - -1: id is Invalid int remove_waiter(WaitId id); - - + // Set the applied id, indicating that the log before applied_id (inclded) // can be droped from memory logs void set_applied_id(const LogId& applied_id); @@ -139,7 +143,7 @@ class BAIDU_CACHELINE_ALIGNMENT LogManager { // one of the following condition // - Log starts from 1. OR // - Log starts from a positive position and there must be a snapshot - // of which the last_included_id is in the range + // of which the last_included_id is in the range // [first_log_index-1, last_log_index] // Returns butil::Status::OK if valid, a specific error otherwise butil::Status check_consistency(); @@ -149,25 +153,26 @@ class BAIDU_CACHELINE_ALIGNMENT LogManager { // Get the internal status of LogManager. void get_status(LogManagerStatus* status); -private: -friend class AppendBatcher; + private: + friend class AppendBatcher; struct WaitMeta { - int (*on_new_log)(void *arg, int error_code); + int (*on_new_log)(void* arg, int error_code); void* arg; int error_code; }; - void append_to_storage(std::vector* to_append, LogId* last_id, IOMetric* metric); + void append_to_storage(std::vector* to_append, LogId* last_id, + IOMetric* metric); static int disk_thread(void* meta, bthread::TaskIterator& iter); - + // delete logs from storage's head, [1, first_index_kept) will be discarded // Returns: // success return 0, failed return -1 int truncate_prefix(const int64_t first_index_kept, std::unique_lock& lck); - + int reset(const int64_t next_log_index, std::unique_lock& lck); @@ -179,7 +184,7 @@ friend class AppendBatcher; WaitId notify_on_new_log(int64_t expected_last_log_index, WaitMeta* wm); - int check_and_resolve_conflict(std::vector* entries, + int check_and_resolve_conflict(std::vector* entries, StableClosure* done); void unsafe_truncate_suffix(const int64_t last_index_kept); @@ -194,7 +199,7 @@ friend class AppendBatcher; int stop_disk_thread(); void wakeup_all_waiter(std::unique_lock& lck); - static void *run_on_new_log(void* arg); + static void* run_on_new_log(void* arg); void report_error(int error_code, const char* fmt, ...); @@ -218,11 +223,11 @@ friend class AppendBatcher; int64_t _last_log_index; // the last snapshot's log_id LogId _last_snapshot_id; - // the virtual first log, for finding next_index of replicator, which + // the virtual first log, for finding next_index of replicator, which // can avoid install_snapshot too often in extreme case where a follower's // install_snapshot is slower than leader's save_snapshot - // [NOTICE] there should not be hole between this log_id and _last_snapshot_id, - // or may cause some unexpect cases + // [NOTICE] there should not be hole between this log_id and + // _last_snapshot_id, or may cause some unexpect cases LogId _virtual_first_log_id; bthread::ExecutionQueueId _disk_queue; @@ -230,4 +235,4 @@ friend class AppendBatcher; } // namespace braft -#endif //BRAFT_LOG_MANAGER_H +#endif // BRAFT_LOG_MANAGER_H diff --git a/src/braft/macros.h b/src/braft/macros.h index bbbf928..4f2c47a 100644 --- a/src/braft/macros.h +++ b/src/braft/macros.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,17 +14,17 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_MACROS_H -#define BRAFT_MACROS_H +#ifndef BRAFT_MACROS_H +#define BRAFT_MACROS_H -#include #include +#include #include -#define BRAFT_VLOG_IS_ON VLOG_IS_ON(89) -#define BRAFT_VLOG VLOG(89) -#define BRAFT_VPLOG VPLOG(89) -#define BRAFT_VLOG_IF(cond) VLOG_IF(89, (cond)) +#define BRAFT_VLOG_IS_ON VLOG_IS_ON(89) +#define BRAFT_VLOG VLOG(89) +#define BRAFT_VPLOG VPLOG(89) +#define BRAFT_VLOG_IF(cond) VLOG_IF(89, (cond)) #define BRAFT_VPLOG_IF(cond) VPLOG_IF(89, (cond)) //#define USE_BTHREAD_MUTEX @@ -37,7 +37,7 @@ namespace braft { typedef ::bthread::Mutex raft_mutex_t; } // namespace braft -#else // USE_BTHREAD_MUTEX +#else // USE_BTHREAD_MUTEX #include namespace braft { @@ -52,23 +52,23 @@ typedef ::butil::Mutex raft_mutex_t; #define BRAFT_MOCK #endif -#define BRAFT_GET_ARG3(arg1, arg2, arg3, ...) arg3 +#define BRAFT_GET_ARG3(arg1, arg2, arg3, ...) arg3 -#define BRAFT_RETURN_IF1(expr, rc) \ - do { \ - if ((expr)) { \ - return (rc); \ - } \ +#define BRAFT_RETURN_IF1(expr, rc) \ + do { \ + if ((expr)) { \ + return (rc); \ + } \ } while (0) -#define BRAFT_RETURN_IF0(expr) \ - do { \ - if ((expr)) { \ - return; \ - } \ +#define BRAFT_RETURN_IF0(expr) \ + do { \ + if ((expr)) { \ + return; \ + } \ } while (0) -#define BRAFT_RETURN_IF(expr, args...) \ - BRAFT_GET_ARG3(1, ##args, BRAFT_RETURN_IF1, BRAFT_RETURN_IF0)(expr, ##args) +#define BRAFT_RETURN_IF(expr, args...) \ + BRAFT_GET_ARG3(1, ##args, BRAFT_RETURN_IF1, BRAFT_RETURN_IF0)(expr, ##args) -#endif //BRAFT_MACROS_H +#endif // BRAFT_MACROS_H diff --git a/src/braft/memory_log.cpp b/src/braft/memory_log.cpp index 40e2051..1c2f7fe 100644 --- a/src/braft/memory_log.cpp +++ b/src/braft/memory_log.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,10 +14,12 @@ // Authors: Qin,Duohao(qinduohao@baidu.com) +#include "braft/memory_log.h" + #include + #include "braft/log_entry.h" #include "braft/memory_log.h" -#include "braft/memory_log.h" namespace braft { @@ -29,27 +31,31 @@ int MemoryLogStorage::init(ConfigurationManager* configuration_manager) { LogEntry* MemoryLogStorage::get_entry(const int64_t index) { std::unique_lock lck(_mutex); - if (index < _first_log_index.load(butil::memory_order_relaxed) - || index > _last_log_index.load(butil::memory_order_relaxed)) { + if (index < _first_log_index.load(butil::memory_order_relaxed) || + index > _last_log_index.load(butil::memory_order_relaxed)) { return NULL; } - LogEntry* temp = _log_entry_data[index - _first_log_index.load(butil::memory_order_relaxed)]; + LogEntry* temp = _log_entry_data[index - _first_log_index.load( + butil::memory_order_relaxed)]; temp->AddRef(); - CHECK(temp->id.index == index) << "get_entry entry index not equal. logentry index:" - << temp->id.index << " required_index:" << index; + CHECK(temp->id.index == index) + << "get_entry entry index not equal. logentry index:" << temp->id.index + << " required_index:" << index; lck.unlock(); return temp; } int64_t MemoryLogStorage::get_term(const int64_t index) { std::unique_lock lck(_mutex); - if (index < _first_log_index.load(butil::memory_order_relaxed) - || index > _last_log_index.load(butil::memory_order_relaxed)) { + if (index < _first_log_index.load(butil::memory_order_relaxed) || + index > _last_log_index.load(butil::memory_order_relaxed)) { return 0; } - LogEntry* temp = _log_entry_data.at(index - _first_log_index.load(butil::memory_order_relaxed)); - CHECK(temp->id.index == index) << "get_term entry index not equal. logentry index:" - << temp->id.index << " required_index:" << index; + LogEntry* temp = _log_entry_data.at( + index - _first_log_index.load(butil::memory_order_relaxed)); + CHECK(temp->id.index == index) + << "get_term entry index not equal. logentry index:" << temp->id.index + << " required_index:" << index; int64_t ret = temp->id.term; lck.unlock(); return ret; @@ -58,10 +64,10 @@ int64_t MemoryLogStorage::get_term(const int64_t index) { int MemoryLogStorage::append_entry(const LogEntry* input_entry) { std::unique_lock lck(_mutex); if (input_entry->id.index != - _last_log_index.load(butil::memory_order_relaxed) + 1) { + _last_log_index.load(butil::memory_order_relaxed) + 1) { CHECK(false) << "input_entry index=" << input_entry->id.index - << " _last_log_index=" << _last_log_index - << " _first_log_index=" << _first_log_index; + << " _last_log_index=" << _last_log_index + << " _first_log_index=" << _first_log_index; return ERANGE; } input_entry->AddRef(); @@ -71,7 +77,7 @@ int MemoryLogStorage::append_entry(const LogEntry* input_entry) { return 0; } -int MemoryLogStorage::append_entries(const std::vector& entries, +int MemoryLogStorage::append_entries(const std::vector& entries, IOMetric* metric) { if (entries.empty()) { return 0; @@ -96,9 +102,10 @@ int MemoryLogStorage::truncate_prefix(const int64_t first_index_kept) { } } _first_log_index.store(first_index_kept, butil::memory_order_release); - if (_first_log_index.load(butil::memory_order_relaxed) - > _last_log_index.load(butil::memory_order_relaxed)) { - _last_log_index.store(first_index_kept - 1, butil::memory_order_release); + if (_first_log_index.load(butil::memory_order_relaxed) > + _last_log_index.load(butil::memory_order_relaxed)) { + _last_log_index.store(first_index_kept - 1, + butil::memory_order_release); } lck.unlock(); @@ -121,9 +128,10 @@ int MemoryLogStorage::truncate_suffix(const int64_t last_index_kept) { } } _last_log_index.store(last_index_kept, butil::memory_order_release); - if (_first_log_index.load(butil::memory_order_relaxed) - > _last_log_index.load(butil::memory_order_relaxed)) { - _first_log_index.store(last_index_kept + 1, butil::memory_order_release); + if (_first_log_index.load(butil::memory_order_relaxed) > + _last_log_index.load(butil::memory_order_relaxed)) { + _first_log_index.store(last_index_kept + 1, + butil::memory_order_release); } lck.unlock(); @@ -163,4 +171,4 @@ butil::Status MemoryLogStorage::gc_instance(const std::string& uri) const { return butil::Status::OK(); } -} // namespace braft +} // namespace braft diff --git a/src/braft/memory_log.h b/src/braft/memory_log.h index 75779a5..4bebb27 100644 --- a/src/braft/memory_log.h +++ b/src/braft/memory_log.h @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,11 +17,13 @@ #ifndef BRAFT_MEMORY_LOG_H #define BRAFT_MEMORY_LOG_H -#include -#include #include #include #include + +#include +#include + #include "braft/log_entry.h" #include "braft/storage.h" #include "braft/util.h" @@ -29,16 +31,12 @@ namespace braft { class BAIDU_CACHELINE_ALIGNMENT MemoryLogStorage : public LogStorage { -public: + public: typedef std::deque MemoryData; MemoryLogStorage(const std::string& path) - : _path(path), - _first_log_index(1), - _last_log_index(0) {} - MemoryLogStorage() - : _first_log_index(1), - _last_log_index(0) {} + : _path(path), _first_log_index(1), _last_log_index(0) {} + MemoryLogStorage() : _first_log_index(1), _last_log_index(0) {} virtual ~MemoryLogStorage() { reset(1); } @@ -63,13 +61,16 @@ class BAIDU_CACHELINE_ALIGNMENT MemoryLogStorage : public LogStorage { // append entries to log virtual int append_entry(const LogEntry* entry); - // append entries to log and update IOMetric, return append success number - virtual int append_entries(const std::vector& entries, IOMetric* metric); + // append entries to log and update IOMetric, return append success number + virtual int append_entries(const std::vector& entries, + IOMetric* metric); - // delete logs from storage's head, [first_log_index, first_index_kept) will be discarded + // delete logs from storage's head, [first_log_index, first_index_kept) will + // be discarded virtual int truncate_prefix(const int64_t first_index_kept); - // delete uncommitted logs from storage's tail, (last_index_kept, last_log_index] will be discarded + // delete uncommitted logs from storage's tail, (last_index_kept, + // last_log_index] will be discarded virtual int truncate_suffix(const int64_t last_index_kept); // Drop all the existing logs and reset next log index to |next_log_index|. @@ -85,7 +86,7 @@ class BAIDU_CACHELINE_ALIGNMENT MemoryLogStorage : public LogStorage { // in |uri| virtual butil::Status gc_instance(const std::string& uri) const; -private: + private: std::string _path; butil::atomic _first_log_index; butil::atomic _last_log_index; @@ -93,6 +94,6 @@ class BAIDU_CACHELINE_ALIGNMENT MemoryLogStorage : public LogStorage { raft_mutex_t _mutex; }; -} // namespace braft +} // namespace braft -#endif //~BRAFT_MEMORY_LOG_H +#endif //~BRAFT_MEMORY_LOG_H diff --git a/src/braft/node.cpp b/src/braft/node.cpp index 84b23c1..8635dfc 100644 --- a/src/braft/node.cpp +++ b/src/braft/node.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,47 +16,50 @@ // Zhangyi Chen(chenzhangyi01@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#include -#include -#include +#include "braft/node.h" + #include +#include +#include +#include + #include +#include "braft/builtin_service_impl.h" #include "braft/configuration.h" #include "braft/errno.pb.h" -#include "braft/util.h" -#include "braft/raft.h" -#include "braft/node.h" +#include "braft/file_service.h" #include "braft/log.h" +#include "braft/node_manager.h" +#include "braft/raft.h" #include "braft/raft_meta.h" #include "braft/snapshot.h" -#include "braft/file_service.h" -#include "braft/builtin_service_impl.h" -#include "braft/node_manager.h" #include "braft/snapshot_executor.h" -#include "braft/errno.pb.h" #include "braft/sync_point.h" +#include "braft/util.h" #include "butil/logging.h" - namespace braft { -DEFINE_int32(raft_max_election_delay_ms, 1000, +DEFINE_int32(raft_max_election_delay_ms, 1000, "Max election delay time allowed by user"); BRPC_VALIDATE_GFLAG(raft_max_election_delay_ms, brpc::PositiveInteger); -DEFINE_bool(raft_step_down_when_vote_timedout, true, +DEFINE_bool(raft_step_down_when_vote_timedout, true, "candidate steps down when reaching timeout"); BRPC_VALIDATE_GFLAG(raft_step_down_when_vote_timedout, brpc::PassValidate); -DEFINE_bool(raft_enable_append_entries_cache, false, - "enable cache for out-of-order append entries requests, should used when " - "pipeline replication is enabled (raft_max_parallel_append_entries_rpc_num > 1)."); +DEFINE_bool( + raft_enable_append_entries_cache, false, + "enable cache for out-of-order append entries requests, should used when " + "pipeline replication is enabled (raft_max_parallel_append_entries_rpc_num " + "> 1)."); BRPC_VALIDATE_GFLAG(raft_enable_append_entries_cache, ::brpc::PassValidate); DEFINE_int32(raft_max_append_entries_cache_size, 8, "the max size of out-of-order append entries cache"); -BRPC_VALIDATE_GFLAG(raft_max_append_entries_cache_size, ::brpc::PositiveInteger); +BRPC_VALIDATE_GFLAG(raft_max_append_entries_cache_size, + ::brpc::PositiveInteger); DEFINE_int64(raft_append_entry_high_lat_us, 1000 * 1000, "append entry high latency us"); @@ -72,8 +75,9 @@ BRPC_VALIDATE_GFLAG(raft_rpc_channel_connect_timeout_ms, brpc::PositiveInteger); DECLARE_bool(raft_enable_leader_lease); -DEFINE_bool(raft_enable_witness_to_leader, false, - "enable witness temporarily to become leader when leader down accidently"); +DEFINE_bool( + raft_enable_witness_to_leader, false, + "enable witness temporarily to become leader when leader down accidently"); #ifndef UNIT_TEST static bvar::Adder g_num_nodes("raft_node_count"); @@ -83,7 +87,7 @@ bvar::Adder g_num_nodes("raft_node_count"); #endif static bvar::CounterRecorder g_apply_tasks_batch_counter( - "raft_apply_tasks_batch_counter"); + "raft_apply_tasks_batch_counter"); int SnapshotTimer::adjust_timeout_ms(int timeout_ms) { if (!_first_schedule) { @@ -97,7 +101,7 @@ int SnapshotTimer::adjust_timeout_ms(int timeout_ms) { } class ConfigurationChangeDone : public Closure { -public: + public: void Run() { if (status().ok()) { _node->on_configuration_change_done(_term); @@ -108,21 +112,21 @@ class ConfigurationChangeDone : public Closure { } delete this; } -private: - ConfigurationChangeDone( - NodeImpl* node, int64_t term, bool leader_start, int64_t lease_epoch) - : _node(node) - , _term(term) - , _leader_start(leader_start) - , _lease_epoch(lease_epoch) - { + + private: + ConfigurationChangeDone(NodeImpl* node, int64_t term, bool leader_start, + int64_t lease_epoch) + : _node(node), + _term(term), + _leader_start(leader_start), + _lease_epoch(lease_epoch) { _node->AddRef(); } ~ConfigurationChangeDone() { _node->Release(); _node = NULL; } -friend class NodeImpl; + friend class NodeImpl; NodeImpl* _node; int64_t _term; bool _leader_start; @@ -134,65 +138,70 @@ inline int random_timeout(int timeout_ms) { return butil::fast_rand_in(timeout_ms, timeout_ms + delta); } -DEFINE_int32(raft_election_heartbeat_factor, 10, "raft election:heartbeat timeout factor"); +DEFINE_int32(raft_election_heartbeat_factor, 10, + "raft election:heartbeat timeout factor"); static inline int heartbeat_timeout(int election_timeout) { - if (FLAGS_raft_election_heartbeat_factor <= 0){ - LOG(WARNING) << "raft_election_heartbeat_factor flag must be greater than 1" - << ", but get "<< FLAGS_raft_election_heartbeat_factor - << ", it will be set to default value 10."; + if (FLAGS_raft_election_heartbeat_factor <= 0) { + LOG(WARNING) + << "raft_election_heartbeat_factor flag must be greater than 1" + << ", but get " << FLAGS_raft_election_heartbeat_factor + << ", it will be set to default value 10."; FLAGS_raft_election_heartbeat_factor = 10; } - return std::max(election_timeout / FLAGS_raft_election_heartbeat_factor, 10); + return std::max(election_timeout / FLAGS_raft_election_heartbeat_factor, + 10); } NodeImpl::NodeImpl(const GroupId& group_id, const PeerId& peer_id) - : _state(STATE_UNINITIALIZED) - , _current_term(0) - , _group_id(group_id) - , _server_id(peer_id) - , _conf_ctx(this) - , _log_storage(NULL) - , _meta_storage(NULL) - , _closure_queue(NULL) - , _config_manager(NULL) - , _log_manager(NULL) - , _fsm_caller(NULL) - , _ballot_box(NULL) - , _snapshot_executor(NULL) - , _stop_transfer_arg(NULL) - , _vote_triggered(false) - , _waking_candidate(0) - , _append_entries_cache(NULL) - , _append_entries_cache_version(0) - , _node_readonly(false) - , _majority_nodes_readonly(false) { - butil::string_printf(&_v_group_id, "%s_%d", _group_id.c_str(), _server_id.idx); + : _state(STATE_UNINITIALIZED), + _current_term(0), + _group_id(group_id), + _server_id(peer_id), + _conf_ctx(this), + _log_storage(NULL), + _meta_storage(NULL), + _closure_queue(NULL), + _config_manager(NULL), + _log_manager(NULL), + _fsm_caller(NULL), + _ballot_box(NULL), + _snapshot_executor(NULL), + _stop_transfer_arg(NULL), + _vote_triggered(false), + _waking_candidate(0), + _append_entries_cache(NULL), + _append_entries_cache_version(0), + _node_readonly(false), + _majority_nodes_readonly(false) { + butil::string_printf(&_v_group_id, "%s_%d", _group_id.c_str(), + _server_id.idx); AddRef(); g_num_nodes << 1; } NodeImpl::NodeImpl() - : _state(STATE_UNINITIALIZED) - , _current_term(0) - , _group_id() - , _server_id() - , _conf_ctx(this) - , _log_storage(NULL) - , _meta_storage(NULL) - , _closure_queue(NULL) - , _config_manager(NULL) - , _log_manager(NULL) - , _fsm_caller(NULL) - , _ballot_box(NULL) - , _snapshot_executor(NULL) - , _stop_transfer_arg(NULL) - , _vote_triggered(false) - , _waking_candidate(0) - , _append_entries_cache(NULL) - , _append_entries_cache_version(0) - , _node_readonly(false) - , _majority_nodes_readonly(false) { - butil::string_printf(&_v_group_id, "%s_%d", _group_id.c_str(), _server_id.idx); + : _state(STATE_UNINITIALIZED), + _current_term(0), + _group_id(), + _server_id(), + _conf_ctx(this), + _log_storage(NULL), + _meta_storage(NULL), + _closure_queue(NULL), + _config_manager(NULL), + _log_manager(NULL), + _fsm_caller(NULL), + _ballot_box(NULL), + _snapshot_executor(NULL), + _stop_transfer_arg(NULL), + _vote_triggered(false), + _waking_candidate(0), + _append_entries_cache(NULL), + _append_entries_cache_version(0), + _node_readonly(false), + _majority_nodes_readonly(false) { + butil::string_printf(&_v_group_id, "%s_%d", _group_id.c_str(), + _server_id.idx); AddRef(); g_num_nodes << 1; } @@ -300,8 +309,8 @@ int NodeImpl::init_meta_storage() { _meta_storage = RaftMetaStorage::create(_options.raft_meta_uri); if (!_meta_storage) { LOG(ERROR) << "node " << _group_id << ":" << _server_id - << " failed to create meta storage, uri " - << _options.raft_meta_uri; + << " failed to create meta storage, uri " + << _options.raft_meta_uri; return ENOENT; } @@ -309,20 +318,18 @@ int NodeImpl::init_meta_storage() { butil::Status status = _meta_storage->init(); if (!status.ok()) { LOG(ERROR) << "node " << _group_id << ":" << _server_id - << " failed to init meta storage, uri " - << _options.raft_meta_uri - << ", error " << status; + << " failed to init meta storage, uri " + << _options.raft_meta_uri << ", error " << status; return status.error_code(); } - + // get term and votedfor - status = _meta_storage-> - get_term_and_votedfor(&_current_term, &_voted_id, _v_group_id); + status = _meta_storage->get_term_and_votedfor(&_current_term, &_voted_id, + _v_group_id); if (!status.ok()) { LOG(ERROR) << "node " << _group_id << ":" << _server_id << " failed to get term and voted_id when init meta storage," - << " uri " << _options.raft_meta_uri - << ", error " << status; + << " uri " << _options.raft_meta_uri << ", error " << status; return status.error_code(); } @@ -331,19 +338,22 @@ int NodeImpl::init_meta_storage() { if (_current_term < last_log_id.term) { // Please check when this bad case really happens! LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " init with invalid term: current_term is " << _current_term - << " but last log term is " << last_log_id.term + << " init with invalid term: current_term is " + << _current_term << " but last log term is " + << last_log_id.term << ". Raft will use a newer term to avoid corner cases"; - // Increase term by 1 is more safety although it may cause a working + // Increase term by 1 is more safety although it may cause a working // Leader to stepdown _current_term = last_log_id.term + 1; _voted_id.reset(); - butil::Status status = _meta_storage-> - set_term_and_votedfor(_current_term, PeerId(), _v_group_id); + butil::Status status = _meta_storage->set_term_and_votedfor( + _current_term, PeerId(), _v_group_id); if (!status.ok()) { - LOG(ERROR) << "node " << _group_id << ":" << _server_id - << " fail to set_term_and_votedfor when correct its term," - " error: " << status; + LOG(ERROR) + << "node " << _group_id << ":" << _server_id + << " fail to set_term_and_votedfor when correct its term," + " error: " + << status; return status.error_code(); } } @@ -362,7 +372,6 @@ void NodeImpl::handle_snapshot_timeout() { lck.unlock(); // TODO: do_snapshot in another thread to avoid blocking the timer thread. do_snapshot(NULL); - } int NodeImpl::init_fsm_caller(const LogId& bootstrap_id) { @@ -388,22 +397,21 @@ int NodeImpl::init_fsm_caller(const LogId& bootstrap_id) { } class BootstrapStableClosure : public LogManager::StableClosure { -public: + public: void Run() { _done.Run(); } void wait() { _done.wait(); } -private: + + private: SynchronizedClosure _done; }; int NodeImpl::bootstrap(const BootstrapOptions& options) { - if (options.last_log_index > 0) { if (options.group_conf.empty() || options.fsm == NULL) { LOG(ERROR) << "Invalid arguments for " << __FUNCTION__ << "group_conf=" << options.group_conf << " fsm=" << options.fsm - << " while last_log_index=" - << options.last_log_index; + << " while last_log_index=" << options.last_log_index; return -1; } } @@ -440,12 +448,13 @@ int NodeImpl::bootstrap(const BootstrapOptions& options) { } if (_current_term == 0) { _current_term = 1; - butil::Status status = _meta_storage-> - set_term_and_votedfor(1, PeerId(), _v_group_id); + butil::Status status = + _meta_storage->set_term_and_votedfor(1, PeerId(), _v_group_id); if (!status.ok()) { - // TODO add group_id + // TODO add group_id LOG(ERROR) << "Fail to set term and votedfor when bootstrap," - " error: " << status; + " error: " + << status; return -1; } return -1; @@ -466,8 +475,8 @@ int NodeImpl::bootstrap(const BootstrapOptions& options) { _snapshot_executor->do_snapshot(&done); done.wait(); if (!done.status().ok()) { - LOG(ERROR) << "Fail to save snapshot " << done.status() - << " from " << _options.snapshot_uri; + LOG(ERROR) << "Fail to save snapshot " << done.status() << " from " + << _options.snapshot_uri; return -1; } } @@ -499,28 +508,31 @@ int NodeImpl::init(const NodeOptions& options) { // check _server_id if (butil::IP_ANY == _server_id.addr.ip) { - LOG(ERROR) << "Group " << _group_id + LOG(ERROR) << "Group " << _group_id << " Node can't started from IP_ANY"; return -1; } if (!global_node_manager->server_exists(_server_id.addr)) { - LOG(ERROR) << "Group " << _group_id - << " No RPC Server attached to " << _server_id.addr + LOG(ERROR) << "Group " << _group_id << " No RPC Server attached to " + << _server_id.addr << ", did you forget to call braft::add_service()?"; return -1; } if (options.witness) { - // When this node is a witness, set the election_timeout to be twice + // When this node is a witness, set the election_timeout to be twice // of the normal replica to ensure that the normal replica has a higher // priority and is selected as the master if (FLAGS_raft_enable_witness_to_leader) { - CHECK_EQ(0, _election_timer.init(this, options.election_timeout_ms * 2)); - CHECK_EQ(0, _vote_timer.init(this, options.election_timeout_ms * 2 + options.max_clock_drift_ms)); + CHECK_EQ( + 0, _election_timer.init(this, options.election_timeout_ms * 2)); + CHECK_EQ(0, _vote_timer.init(this, options.election_timeout_ms * 2 + + options.max_clock_drift_ms)); } } else { CHECK_EQ(0, _election_timer.init(this, options.election_timeout_ms)); - CHECK_EQ(0, _vote_timer.init(this, options.election_timeout_ms + options.max_clock_drift_ms)); + CHECK_EQ(0, _vote_timer.init(this, options.election_timeout_ms + + options.max_clock_drift_ms)); } CHECK_EQ(0, _stepdown_timer.init(this, options.election_timeout_ms)); CHECK_EQ(0, _snapshot_timer.init(this, options.snapshot_interval_s * 1000)); @@ -529,7 +541,7 @@ int NodeImpl::init(const NodeOptions& options) { if (bthread::execution_queue_start(&_apply_queue_id, NULL, execute_applying_tasks, this) != 0) { - LOG(ERROR) << "node " << _group_id << ":" << _server_id + LOG(ERROR) << "node " << _group_id << ":" << _server_id << " fail to start execution_queue"; return -1; } @@ -546,9 +558,11 @@ int NodeImpl::init(const NodeOptions& options) { _leader_lease.init(options.election_timeout_ms); if (options.witness) { - _follower_lease.init(options.election_timeout_ms * 2, options.max_clock_drift_ms); + _follower_lease.init(options.election_timeout_ms * 2, + options.max_clock_drift_ms); } else { - _follower_lease.init(options.election_timeout_ms, options.max_clock_drift_ms); + _follower_lease.init(options.election_timeout_ms, + options.max_clock_drift_ms); } // log storage and log manager init @@ -576,8 +590,10 @@ int NodeImpl::init(const NodeOptions& options) { } // snapshot storage init and load - // NOTE: snapshot maybe discard entries when snapshot saved but not discard entries. - // init log storage before snapshot storage, snapshot storage will update configration + // NOTE: snapshot maybe discard entries when snapshot saved but not discard + // entries. + // init log storage before snapshot storage, snapshot storage will + // update configration if (init_snapshot_storage() != 0) { LOG(ERROR) << "node " << _group_id << ":" << _server_id << " init_snapshot_storage failed"; @@ -587,8 +603,7 @@ int NodeImpl::init(const NodeOptions& options) { butil::Status st = _log_manager->check_consistency(); if (!st.ok()) { LOG(ERROR) << "node " << _group_id << ":" << _server_id - << " is initialized with inconsitency log: " - << st; + << " is initialized with inconsitency log: " << st; return -1; } @@ -599,7 +614,7 @@ int NodeImpl::init(const NodeOptions& options) { } else { _conf.conf = _options.initial_conf; } - + // init meta and check term if (init_meta_storage() != 0) { LOG(ERROR) << "node " << _group_id << ":" << _server_id @@ -614,17 +629,16 @@ int NodeImpl::init(const NodeOptions& options) { // init replicator ReplicatorGroupOptions rg_options; - rg_options.heartbeat_timeout_ms = heartbeat_timeout(_options.election_timeout_ms); + rg_options.heartbeat_timeout_ms = + heartbeat_timeout(_options.election_timeout_ms); rg_options.election_timeout_ms = _options.election_timeout_ms; rg_options.log_manager = _log_manager; rg_options.ballot_box = _ballot_box; rg_options.node = this; - rg_options.snapshot_throttle = _options.snapshot_throttle - ? _options.snapshot_throttle->get() - : NULL; - rg_options.snapshot_storage = _snapshot_executor - ? _snapshot_executor->snapshot_storage() - : NULL; + rg_options.snapshot_throttle = + _options.snapshot_throttle ? _options.snapshot_throttle->get() : NULL; + rg_options.snapshot_storage = + _snapshot_executor ? _snapshot_executor->snapshot_storage() : NULL; _replicator_group.init(NodeId(_group_id, _server_id), rg_options); // set state to follower @@ -633,13 +647,12 @@ int NodeImpl::init(const NodeOptions& options) { LOG(INFO) << "node " << _group_id << ":" << _server_id << " init," << " term: " << _current_term << " last_log_id: " << _log_manager->last_log_id() - << " conf: " << _conf.conf - << " old_conf: " << _conf.old_conf; + << " conf: " << _conf.conf << " old_conf: " << _conf.old_conf; // start snapshot timer if (_snapshot_executor && _options.snapshot_interval_s > 0) { - BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " start snapshot_timer"; + BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " start snapshot_timer"; _snapshot_timer.start(); } @@ -649,16 +662,16 @@ int NodeImpl::init(const NodeOptions& options) { // add node to NodeManager if (!global_node_manager->add(this)) { - LOG(ERROR) << "NodeManager add " << _group_id - << ":" << _server_id << " failed"; + LOG(ERROR) << "NodeManager add " << _group_id << ":" << _server_id + << " failed"; return -1; } // Now the raft node is started , have to acquire the lock to avoid race // conditions std::unique_lock lck(_mutex); - if (_conf.stable() && _conf.conf.size() == 1u - && _conf.conf.contains(_server_id)) { + if (_conf.stable() && _conf.conf.size() == 1u && + _conf.conf.contains(_server_id)) { // The group contains only this server which must be the LEADER, trigger // the timer immediately. elect_self(&lck); @@ -667,12 +680,13 @@ int NodeImpl::init(const NodeOptions& options) { return 0; } -DEFINE_int32(raft_apply_batch, 32, "Max number of tasks that can be applied " - " in a single batch"); +DEFINE_int32(raft_apply_batch, 32, + "Max number of tasks that can be applied " + " in a single batch"); BRPC_VALIDATE_GFLAG(raft_apply_batch, ::brpc::PositiveInteger); int NodeImpl::execute_applying_tasks( - void* meta, bthread::TaskIterator& iter) { + void* meta, bthread::TaskIterator& iter) { if (iter.is_queue_stopped()) { return 0; } @@ -725,13 +739,10 @@ void NodeImpl::on_configuration_change_done(int64_t term) { } class OnCaughtUp : public CatchupClosure { -public: - OnCaughtUp(NodeImpl* node, int64_t term, const PeerId& peer, int64_t version) - : _node(node) - , _term(term) - , _peer(peer) - , _version(version) - { + public: + OnCaughtUp(NodeImpl* node, int64_t term, const PeerId& peer, + int64_t version) + : _node(node), _term(term), _peer(peer), _version(version) { _node->AddRef(); } ~OnCaughtUp() { @@ -744,22 +755,24 @@ class OnCaughtUp : public CatchupClosure { _node->on_caughtup(_peer, _term, _version, status()); delete this; }; -private: + + private: NodeImpl* _node; int64_t _term; PeerId _peer; int64_t _version; }; -void NodeImpl::on_caughtup(const PeerId& peer, int64_t term, - int64_t version, const butil::Status& st) { +void NodeImpl::on_caughtup(const PeerId& peer, int64_t term, int64_t version, + const butil::Status& st) { BAIDU_SCOPED_LOCK(_mutex); // CHECK _state and _current_term to avoid ABA problem if (_state != STATE_LEADER || term != _current_term) { - // if leader stepped down, reset() has already been called in step_down(), - // so nothing needs to be done here - LOG(WARNING) << "node " << node_id() << " stepped down when waiting peer " - << peer << " to catch up, current state is " << state2str(_state) + // if leader stepped down, reset() has already been called in + // step_down(), so nothing needs to be done here + LOG(WARNING) << "node " << node_id() + << " stepped down when waiting peer " << peer + << " to catch up, current state is " << state2str(_state) << ", current term is " << _current_term << ", expect term is " << term; return; @@ -771,24 +784,24 @@ void NodeImpl::on_caughtup(const PeerId& peer, int64_t term, } // Retry if this peer is still alive - if (st.error_code() == ETIMEDOUT - && (butil::monotonic_time_ms() - - _replicator_group.last_rpc_send_timestamp(peer)) - <= _options.election_timeout_ms) { - - LOG(INFO) << "node " << _group_id << ":" << _server_id - << " waits peer " << peer << " to catch up"; - - OnCaughtUp* caught_up = new OnCaughtUp(this, _current_term, peer, version); - timespec due_time = butil::milliseconds_from_now( - _options.get_catchup_timeout_ms()); - - if (0 == _replicator_group.wait_caughtup( - peer, _options.catchup_margin, &due_time, caught_up)) { + if (st.error_code() == ETIMEDOUT && + (butil::monotonic_time_ms() - + _replicator_group.last_rpc_send_timestamp(peer)) <= + _options.election_timeout_ms) { + LOG(INFO) << "node " << _group_id << ":" << _server_id << " waits peer " + << peer << " to catch up"; + + OnCaughtUp* caught_up = + new OnCaughtUp(this, _current_term, peer, version); + timespec due_time = + butil::milliseconds_from_now(_options.get_catchup_timeout_ms()); + + if (0 == _replicator_group.wait_caughtup(peer, _options.catchup_margin, + &due_time, caught_up)) { return; } else { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " wait_caughtup failed, peer " << peer; + << " wait_caughtup failed, peer " << peer; delete caught_up; } } @@ -807,8 +820,8 @@ void NodeImpl::check_dead_nodes(const Configuration& conf, int64_t now_ms) { continue; } - if (now_ms - _replicator_group.last_rpc_send_timestamp(peers[i]) - <= _options.election_timeout_ms) { + if (now_ms - _replicator_group.last_rpc_send_timestamp(peers[i]) <= + _options.election_timeout_ms) { ++alive_count; continue; } @@ -817,11 +830,10 @@ void NodeImpl::check_dead_nodes(const Configuration& conf, int64_t now_ms) { if (alive_count >= peers.size() / 2 + 1) { return; } - LOG(WARNING) << "node " << node_id() - << " term " << _current_term + LOG(WARNING) << "node " << node_id() << " term " << _current_term << " steps down when alive nodes don't satisfy quorum" - " dead_nodes: " << dead_nodes - << " conf: " << conf; + " dead_nodes: " + << dead_nodes << " conf: " << conf; butil::Status status; status.set_error(ERAFTTIMEDOUT, "Majority of the group dies"); step_down(_current_term, false, status); @@ -832,9 +844,9 @@ void NodeImpl::handle_stepdown_timeout() { // check state if (_state > STATE_TRANSFERRING) { - BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " stop stepdown_timer" - << " state is " << state2str(_state); + BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " stop stepdown_timer" + << " state is " << state2str(_state); return; } check_witness(_conf.conf); @@ -847,12 +859,13 @@ void NodeImpl::handle_stepdown_timeout() { void NodeImpl::check_witness(const Configuration& conf) { if (is_witness()) { - LOG(WARNING) << "node " << node_id() - << " term " << _current_term - << " steps down as it's a witness but become leader temporarily" - << " conf: " << conf; + LOG(WARNING) + << "node " << node_id() << " term " << _current_term + << " steps down as it's a witness but become leader temporarily" + << " conf: " << conf; butil::Status status; - status.set_error(ETRANSFERLEADERSHIP, "Witness becomes leader temporarily"); + status.set_error(ETRANSFERLEADERSHIP, + "Witness becomes leader temporarily"); step_down(_current_term, true, status); } } @@ -861,9 +874,10 @@ void NodeImpl::unsafe_register_conf_change(const Configuration& old_conf, const Configuration& new_conf, Closure* done) { if (_state != STATE_LEADER) { - LOG(WARNING) << "[" << node_id() - << "] Refusing configuration changing because the state is " - << state2str(_state) ; + LOG(WARNING) + << "[" << node_id() + << "] Refusing configuration changing because the state is " + << state2str(_state); if (done) { if (_state == STATE_TRANSFERRING) { done->status().set_error(EBUSY, "Is transferring leadership"); @@ -880,7 +894,8 @@ void NodeImpl::unsafe_register_conf_change(const Configuration& old_conf, LOG(WARNING) << "[" << node_id() << " ] Refusing concurrent configuration changing"; if (done) { - done->status().set_error(EBUSY, "Doing another configuration change"); + done->status().set_error(EBUSY, + "Doing another configuration change"); run_closure_in_bthread(done); } return; @@ -927,18 +942,20 @@ butil::Status NodeImpl::reset_peers(const Configuration& new_peers) { BAIDU_SCOPED_LOCK(_mutex); if (new_peers.empty()) { - LOG(WARNING) << "node " << _group_id << ":" << _server_id << " set empty peers"; + LOG(WARNING) << "node " << _group_id << ":" << _server_id + << " set empty peers"; return butil::Status(EINVAL, "new_peers is empty"); } // check state if (!is_active_state(_state)) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " is in state " << state2str(_state) << ", can't reset_peer"; + << " is in state " << state2str(_state) + << ", can't reset_peer"; return butil::Status(EPERM, "Bad state %s", state2str(_state)); } // check bootstrap if (_conf.conf.empty()) { - LOG(INFO) << "node " << _group_id << ":" << _server_id + LOG(INFO) << "node " << _group_id << ":" << _server_id << " reset_peers to " << new_peers << " from empty"; _conf.conf = new_peers; butil::Status status; @@ -960,9 +977,8 @@ butil::Status NodeImpl::reset_peers(const Configuration& new_peers) { } Configuration new_conf(new_peers); - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " set_peer from " - << _conf.conf << " to " << new_conf; + LOG(WARNING) << "node " << _group_id << ":" << _server_id + << " set_peer from " << _conf.conf << " to " << new_conf; // change conf and step_down _conf.conf = new_conf; _conf.old_conf.reset(); @@ -972,12 +988,10 @@ butil::Status NodeImpl::reset_peers(const Configuration& new_peers) { return butil::Status::OK(); } -void NodeImpl::snapshot(Closure* done) { - do_snapshot(done); -} +void NodeImpl::snapshot(Closure* done) { do_snapshot(done); } void NodeImpl::do_snapshot(Closure* done) { - LOG(INFO) << "node " << _group_id << ":" << _server_id + LOG(INFO) << "node " << _group_id << ":" << _server_id << " starts to do snapshot"; if (_snapshot_executor) { _snapshot_executor->do_snapshot(done); @@ -995,8 +1009,10 @@ void NodeImpl::shutdown(Closure* done) { { BAIDU_SCOPED_LOCK(_mutex); - LOG(INFO) << "node " << _group_id << ":" << _server_id << " shutdown," - " current_term " << _current_term << " state " << state2str(_state); + LOG(INFO) << "node " << _group_id << ":" << _server_id + << " shutdown," + " current_term " + << _current_term << " state " << state2str(_state); if (_state < STATE_SHUTTING) { // Got the right to shut // Remove node from NodeManager and |this| would not be accessed by @@ -1028,9 +1044,10 @@ void NodeImpl::shutdown(Closure* done) { _snapshot_executor->shutdown(); } - // step_down will call _commitment_manager->clear_pending_applications(), - // this can avoid send LogEntry with closure to fsm_caller. - // fsm_caller shutdown will not leak user's closure. + // step_down will call + // _commitment_manager->clear_pending_applications(), this can avoid + // send LogEntry with closure to fsm_caller. fsm_caller shutdown + // will not leak user's closure. if (_fsm_caller) { _fsm_caller->shutdown(); } @@ -1077,7 +1094,6 @@ void NodeImpl::handle_election_timeout() { // Trigger vote manually, or wait until follower lease expire. if (!_vote_triggered && !_follower_lease.expired()) { - return; } bool triggered = _vote_triggered; @@ -1087,7 +1103,7 @@ void NodeImpl::handle_election_timeout() { PeerId empty_id; butil::Status status; status.set_error(ERAFTTIMEDOUT, "Lost connection from leader %s", - _leader_id.to_string().c_str()); + _leader_id.to_string().c_str()); reset_leader_id(empty_id, status); return pre_vote(&lck, triggered); @@ -1104,7 +1120,8 @@ void NodeImpl::handle_timeout_now_request(brpc::Controller* controller, const int64_t saved_current_term = _current_term; if (request->term() > _current_term) { butil::Status status; - status.set_error(EHIGHERTERMREQUEST, "Raft node receives higher term request."); + status.set_error(EHIGHERTERMREQUEST, + "Raft node receives higher term request."); step_down(request->term(), false, status); } response->set_term(_current_term); @@ -1112,8 +1129,8 @@ void NodeImpl::handle_timeout_now_request(brpc::Controller* controller, lck.unlock(); LOG(INFO) << "node " << _group_id << ":" << _server_id << " received handle_timeout_now_request while _current_term=" - << saved_current_term << " didn't match request_term=" - << request->term(); + << saved_current_term + << " didn't match request_term=" << request->term(); return; } if (_state != STATE_FOLLOWER) { @@ -1123,7 +1140,7 @@ void NodeImpl::handle_timeout_now_request(brpc::Controller* controller, response->set_success(false); lck.unlock(); LOG(INFO) << "node " << _group_id << ":" << _server_id - << " received handle_timeout_now_request while state is " + << " received handle_timeout_now_request while state is " << state2str(saved_state) << " at term=" << saved_term; return; } @@ -1149,21 +1166,19 @@ void NodeImpl::handle_timeout_now_request(brpc::Controller* controller, // Note: don't touch controller, request, response, done anymore since they // were dereferenced at this point LOG(INFO) << "node " << _group_id << ":" << _server_id - << " received handle_timeout_now_request from " - << remote_side << " at term=" << saved_term; - + << " received handle_timeout_now_request from " << remote_side + << " at term=" << saved_term; } class StopTransferArg { -DISALLOW_COPY_AND_ASSIGN(StopTransferArg); -public: + DISALLOW_COPY_AND_ASSIGN(StopTransferArg); + + public: StopTransferArg(NodeImpl* n, int64_t t, const PeerId& p) : node(n), term(t), peer(p) { node->AddRef(); } - ~StopTransferArg() { - node->Release(); - } + ~StopTransferArg() { node->Release(); } NodeImpl* node; int64_t term; PeerId peer; @@ -1176,8 +1191,9 @@ void on_transfer_timeout(void* arg) { } void NodeImpl::handle_transfer_timeout(int64_t term, const PeerId& peer) { - LOG(INFO) << "node " << node_id() << " failed to transfer leadership to peer=" - << peer << " : reached timeout"; + LOG(INFO) << "node " << node_id() + << " failed to transfer leadership to peer=" << peer + << " : reached timeout"; BAIDU_SCOPED_LOCK(_mutex); if (term == _current_term) { _replicator_group.stop_transfer_leadership(peer); @@ -1202,14 +1218,13 @@ int NodeImpl::transfer_leadership_to(const PeerId& peer) { // TimeoutNowRequest and increase the term while somehow another leader // which was not replicated with the newest configuration has been // elected. If no add_peer with this very |peer| is to be invoked ever - // after nor this peer is to be killed, this peer will spin in the voting - // procedure and make the each new leader stepped down when the peer - // reached vote timedout and it starts to vote (because it will increase - // the term of the group) - // To make things simple, refuse the operation and force users to - // invoke transfer_leadership_to after configuration changing is - // completed so that the peer's configuration is up-to-date when it - // receives the TimeOutNowRequest. + // after nor this peer is to be killed, this peer will spin in the + // voting procedure and make the each new leader stepped down when the + // peer reached vote timedout and it starts to vote (because it will + // increase the term of the group) To make things simple, refuse the + // operation and force users to invoke transfer_leadership_to after + // configuration changing is completed so that the peer's configuration + // is up-to-date when it receives the TimeOutNowRequest. LOG(WARNING) << "node " << _group_id << ":" << _server_id << " refused to transfer leadership to peer " << peer << " when the leader is changing the configuration"; @@ -1218,17 +1233,18 @@ int NodeImpl::transfer_leadership_to(const PeerId& peer) { PeerId peer_id = peer; // if peer_id is ANY_PEER(0.0.0.0:0:0), the peer with the largest - // last_log_id will be selected. + // last_log_id will be selected. if (peer_id == ANY_PEER) { LOG(INFO) << "node " << _group_id << ":" << _server_id << " starts to transfer leadership to any peer."; - // find the next candidate which is the most possible to become new leader + // find the next candidate which is the most possible to become new + // leader if (_replicator_group.find_the_next_candidate(&peer_id, _conf) != 0) { - return -1; + return -1; } } if (peer_id == _server_id) { - LOG(INFO) << "node " << _group_id << ":" << _server_id + LOG(INFO) << "node " << _group_id << ":" << _server_id << " transfering leadership to self"; return 0; } @@ -1239,34 +1255,39 @@ int NodeImpl::transfer_leadership_to(const PeerId& peer) { return EINVAL; } const int64_t last_log_index = _log_manager->last_log_index(); - const int rc = _replicator_group.transfer_leadership_to(peer_id, last_log_index); + const int rc = + _replicator_group.transfer_leadership_to(peer_id, last_log_index); if (rc != 0) { if (rc == EINVAL) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " fail to transfer leadership, no such peer=" << peer_id; + << " fail to transfer leadership, no such peer=" + << peer_id; } else if (rc == EHOSTUNREACH) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " fail to transfer leadership, peer=" << peer_id - << " whose consecutive_error_times not 0."; + << " fail to transfer leadership, peer=" << peer_id + << " whose consecutive_error_times not 0."; } else { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " fail to transfer leadership, peer=" << peer_id - << " err: " << berror(rc); + << " fail to transfer leadership, peer=" << peer_id + << " err: " << berror(rc); } return rc; } _state = STATE_TRANSFERRING; butil::Status status; - status.set_error(ETRANSFERLEADERSHIP, "Raft leader is transferring " - "leadership to %s", peer_id.to_string().c_str()); + status.set_error(ETRANSFERLEADERSHIP, + "Raft leader is transferring " + "leadership to %s", + peer_id.to_string().c_str()); _leader_lease.on_leader_stop(); _fsm_caller->on_leader_stop(status); LOG(INFO) << "node " << _group_id << ":" << _server_id << " starts to transfer leadership to " << peer_id; _stop_transfer_arg = new StopTransferArg(this, _current_term, peer_id); - if (bthread_timer_add(&_transfer_timer, - butil::milliseconds_from_now(_options.election_timeout_ms), - on_transfer_timeout, _stop_transfer_arg) != 0) { + if (bthread_timer_add( + &_transfer_timer, + butil::milliseconds_from_now(_options.election_timeout_ms), + on_transfer_timeout, _stop_transfer_arg) != 0) { lck.unlock(); LOG(ERROR) << "Fail to add timer"; on_transfer_timeout(_stop_transfer_arg); @@ -1280,11 +1301,14 @@ butil::Status NodeImpl::vote(int election_timeout_ms) { if (_state != STATE_FOLLOWER) { return butil::Status(EPERM, "is not follower"); } - int max_election_timeout_ms = _options.max_clock_drift_ms + _options.election_timeout_ms; + int max_election_timeout_ms = + _options.max_clock_drift_ms + _options.election_timeout_ms; if (election_timeout_ms > max_election_timeout_ms) { - return butil::Status(EINVAL, "election_timeout_ms larger than safety threshold"); + return butil::Status( + EINVAL, "election_timeout_ms larger than safety threshold"); } - election_timeout_ms = std::min(election_timeout_ms, max_election_timeout_ms); + election_timeout_ms = + std::min(election_timeout_ms, max_election_timeout_ms); int max_clock_drift_ms = max_election_timeout_ms - election_timeout_ms; unsafe_reset_election_timeout_ms(election_timeout_ms, max_clock_drift_ms); _vote_triggered = true; @@ -1292,29 +1316,36 @@ butil::Status NodeImpl::vote(int election_timeout_ms) { const State saved_state = _state; lck.unlock(); - LOG(INFO) << "node " << _group_id << ":" << _server_id << " trigger-vote," - " current_term " << saved_current_term << " state " << state2str(saved_state) << - " election_timeout " << election_timeout_ms; + LOG(INFO) << "node " << _group_id << ":" << _server_id + << " trigger-vote," + " current_term " + << saved_current_term << " state " << state2str(saved_state) + << " election_timeout " << election_timeout_ms; return butil::Status(); } butil::Status NodeImpl::reset_election_timeout_ms(int election_timeout_ms) { std::unique_lock lck(_mutex); - int max_election_timeout_ms = _options.max_clock_drift_ms + _options.election_timeout_ms; + int max_election_timeout_ms = + _options.max_clock_drift_ms + _options.election_timeout_ms; if (election_timeout_ms > max_election_timeout_ms) { - return butil::Status(EINVAL, "election_timeout_ms larger than safety threshold"); + return butil::Status( + EINVAL, "election_timeout_ms larger than safety threshold"); } - election_timeout_ms = std::min(election_timeout_ms, max_election_timeout_ms); + election_timeout_ms = + std::min(election_timeout_ms, max_election_timeout_ms); int max_clock_drift_ms = max_election_timeout_ms - election_timeout_ms; unsafe_reset_election_timeout_ms(election_timeout_ms, max_clock_drift_ms); const int64_t saved_current_term = _current_term; const State saved_state = _state; lck.unlock(); - LOG(INFO) << "node " << _group_id << ":" << _server_id << " reset_election_timeout," - " current_term " << saved_current_term << " state " << state2str(saved_state) << - " new election_timeout " << election_timeout_ms << " new clock_drift_ms " << - max_clock_drift_ms; + LOG(INFO) << "node " << _group_id << ":" << _server_id + << " reset_election_timeout," + " current_term " + << saved_current_term << " state " << state2str(saved_state) + << " new election_timeout " << election_timeout_ms + << " new clock_drift_ms " << max_clock_drift_ms; return butil::Status(); } @@ -1326,10 +1357,12 @@ void NodeImpl::reset_election_timeout_ms(int election_timeout_ms, const State saved_state = _state; lck.unlock(); - LOG(INFO) << "node " << _group_id << ":" << _server_id << " reset_election_timeout," - " current_term " << saved_current_term << " state " << state2str(saved_state) << - " new election_timeout " << election_timeout_ms << " new clock_drift_ms " << - max_clock_drift_ms; + LOG(INFO) << "node " << _group_id << ":" << _server_id + << " reset_election_timeout," + " current_term " + << saved_current_term << " state " << state2str(saved_state) + << " new election_timeout " << election_timeout_ms + << " new clock_drift_ms " << max_clock_drift_ms; } void NodeImpl::unsafe_reset_election_timeout_ms(int election_timeout_ms, @@ -1337,15 +1370,18 @@ void NodeImpl::unsafe_reset_election_timeout_ms(int election_timeout_ms, _options.election_timeout_ms = election_timeout_ms; _options.max_clock_drift_ms = max_clock_drift_ms; _replicator_group.reset_heartbeat_interval( - heartbeat_timeout(_options.election_timeout_ms)); - _replicator_group.reset_election_timeout_interval(_options.election_timeout_ms); + heartbeat_timeout(_options.election_timeout_ms)); + _replicator_group.reset_election_timeout_interval( + _options.election_timeout_ms); if (_options.witness && FLAGS_raft_enable_witness_to_leader) { _election_timer.reset(election_timeout_ms * 2); - _follower_lease.reset_election_timeout_ms(election_timeout_ms * 2, _options.max_clock_drift_ms); + _follower_lease.reset_election_timeout_ms(election_timeout_ms * 2, + _options.max_clock_drift_ms); } else { _election_timer.reset(election_timeout_ms); _leader_lease.reset_election_timeout_ms(election_timeout_ms); - _follower_lease.reset_election_timeout_ms(election_timeout_ms, _options.max_clock_drift_ms); + _follower_lease.reset_election_timeout_ms(election_timeout_ms, + _options.max_clock_drift_ms); } } @@ -1361,7 +1397,8 @@ void NodeImpl::on_error(const Error& e) { // if it is follower, also step down to call on_stop_following if (_state <= STATE_FOLLOWER) { butil::Status status; - status.set_error(EBADNODE, "Raft node(leader or candidate) is in error."); + status.set_error(EBADNODE, + "Raft node(leader or candidate) is in error."); step_down(_current_term, _state == STATE_LEADER, status); } if (_state < STATE_ERROR) { @@ -1375,12 +1412,11 @@ void NodeImpl::handle_vote_timeout() { // check state if (_state != STATE_CANDIDATE) { - return; + return; } if (FLAGS_raft_step_down_when_vote_timedout) { // step down to follower - LOG(WARNING) << "node " << node_id() - << " term " << _current_term + LOG(WARNING) << "node " << node_id() << " term " << _current_term << " steps down when reaching vote timeout:" " fail to get quorum vote-granted"; butil::Status status; @@ -1389,15 +1425,15 @@ void NodeImpl::handle_vote_timeout() { pre_vote(&lck, false); } else { // retry vote - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " retry elect"; + LOG(WARNING) << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " retry elect"; elect_self(&lck); } } -void NodeImpl::handle_request_vote_response(const PeerId& peer_id, const int64_t term, - const int64_t ctx_version, - const RequestVoteResponse& response) { +void NodeImpl::handle_request_vote_response( + const PeerId& peer_id, const int64_t term, const int64_t ctx_version, + const RequestVoteResponse& response) { BAIDU_SCOPED_LOCK(_mutex); if (ctx_version != _vote_ctx.version()) { @@ -1426,17 +1462,19 @@ void NodeImpl::handle_request_vote_response(const PeerId& peer_id, const int64_t if (response.term() > _current_term) { LOG(WARNING) << "node " << _group_id << ":" << _server_id << " received invalid RequestVoteResponse from " << peer_id - << " term " << response.term() << " expect " << _current_term; + << " term " << response.term() << " expect " + << _current_term; butil::Status status; - status.set_error(EHIGHERTERMRESPONSE, "Raft node receives higher term " - "request_vote_response."); + status.set_error(EHIGHERTERMRESPONSE, + "Raft node receives higher term " + "request_vote_response."); step_down(response.term(), false, status); return; } LOG(INFO) << "node " << _group_id << ":" << _server_id - << " received RequestVoteResponse from " << peer_id - << " term " << response.term() << " granted " << response.granted() + << " received RequestVoteResponse from " << peer_id << " term " + << response.term() << " granted " << response.granted() << " rejected_by_lease " << response.rejected_by_lease() << " disrupted " << response.disrupted(); if (!response.granted() && !response.rejected_by_lease()) { @@ -1444,7 +1482,8 @@ void NodeImpl::handle_request_vote_response(const PeerId& peer_id, const int64_t } if (response.disrupted()) { - _vote_ctx.set_disrupted_leader(DisruptedLeader(peer_id, response.previous_term())); + _vote_ctx.set_disrupted_leader( + DisruptedLeader(peer_id, response.previous_term())); } if (response.granted()) { _vote_ctx.grant(peer_id); @@ -1478,19 +1517,18 @@ struct OnRequestVoteRPCDone : public google::protobuf::Closure { : peer(peer_id_), term(term_), ctx_version(ctx_version_), node(node_) { node->AddRef(); } - virtual ~OnRequestVoteRPCDone() { - node->Release(); - } + virtual ~OnRequestVoteRPCDone() { node->Release(); } void Run() { do { if (cntl.ErrorCode() != 0) { LOG(WARNING) << "node " << node->node_id() - << " received RequestVoteResponse from " << peer - << " error: " << cntl.ErrorText(); + << " received RequestVoteResponse from " << peer + << " error: " << cntl.ErrorText(); break; } - node->handle_request_vote_response(peer, term, ctx_version, response); + node->handle_request_vote_response(peer, term, ctx_version, + response); } while (0); delete this; } @@ -1504,7 +1542,8 @@ struct OnRequestVoteRPCDone : public google::protobuf::Closure { NodeImpl* node; }; -void NodeImpl::handle_pre_vote_response(const PeerId& peer_id, const int64_t term, +void NodeImpl::handle_pre_vote_response(const PeerId& peer_id, + const int64_t term, const int64_t ctx_version, const RequestVoteResponse& response) { std::unique_lock lck(_mutex); @@ -1521,7 +1560,8 @@ void NodeImpl::handle_pre_vote_response(const PeerId& peer_id, const int64_t ter if (_state != STATE_FOLLOWER) { LOG(WARNING) << "node " << _group_id << ":" << _server_id << " received invalid PreVoteResponse from " << peer_id - << " state not in STATE_FOLLOWER but " << state2str(_state); + << " state not in STATE_FOLLOWER but " + << state2str(_state); return; } // check stale response @@ -1535,17 +1575,19 @@ void NodeImpl::handle_pre_vote_response(const PeerId& peer_id, const int64_t ter if (response.term() > _current_term) { LOG(WARNING) << "node " << _group_id << ":" << _server_id << " received invalid PreVoteResponse from " << peer_id - << " term " << response.term() << " expect " << _current_term; + << " term " << response.term() << " expect " + << _current_term; butil::Status status; - status.set_error(EHIGHERTERMRESPONSE, "Raft node receives higher term " - "pre_vote_response."); + status.set_error(EHIGHERTERMRESPONSE, + "Raft node receives higher term " + "pre_vote_response."); step_down(response.term(), false, status); return; } LOG(INFO) << "node " << _group_id << ":" << _server_id - << " received PreVoteResponse from " << peer_id - << " term " << response.term() << " granted " << response.granted() + << " received PreVoteResponse from " << peer_id << " term " + << response.term() << " granted " << response.granted() << " rejected_by_lease " << response.rejected_by_lease() << " disrupted " << response.disrupted(); @@ -1559,7 +1601,8 @@ void NodeImpl::handle_pre_vote_response(const PeerId& peer_id, const int64_t ter } if (response.disrupted()) { - _pre_vote_ctx.set_disrupted_leader(DisruptedLeader(peer_id, response.previous_term())); + _pre_vote_ctx.set_disrupted_leader( + DisruptedLeader(peer_id, response.previous_term())); } std::set peers; if (response.rejected_by_lease()) { @@ -1572,8 +1615,8 @@ void NodeImpl::handle_pre_vote_response(const PeerId& peer_id, const int64_t ter _pre_vote_ctx.pop_grantable_peers(&peers); peers.insert(peer_id); } - for (std::set::const_iterator it = peers.begin(); - it != peers.end(); ++it) { + for (std::set::const_iterator it = peers.begin(); it != peers.end(); + ++it) { _pre_vote_ctx.grant(*it); if (*it == _follower_lease.last_leader()) { _pre_vote_ctx.grant(_server_id); @@ -1591,16 +1634,14 @@ struct OnPreVoteRPCDone : public google::protobuf::Closure { : peer(peer_id_), term(term_), ctx_version(ctx_version_), node(node_) { node->AddRef(); } - virtual ~OnPreVoteRPCDone() { - node->Release(); - } + virtual ~OnPreVoteRPCDone() { node->Release(); } void Run() { do { if (cntl.ErrorCode() != 0) { - LOG(WARNING) << "node " << node->node_id() - << " request PreVote from " << peer - << " error: " << cntl.ErrorText(); + LOG(WARNING) + << "node " << node->node_id() << " request PreVote from " + << peer << " error: " << cntl.ErrorText(); break; } node->handle_pre_vote_response(peer, term, ctx_version, response); @@ -1618,11 +1659,11 @@ struct OnPreVoteRPCDone : public google::protobuf::Closure { }; void NodeImpl::pre_vote(std::unique_lock* lck, bool triggered) { - LOG(INFO) << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " start pre_vote"; + LOG(INFO) << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " start pre_vote"; if (_snapshot_executor && _snapshot_executor->is_installing_snapshot()) { - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " term " << _current_term + LOG(WARNING) << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " doesn't do pre_vote when installing snapshot as the " " configuration is possibly out of date"; return; @@ -1641,7 +1682,8 @@ void NodeImpl::pre_vote(std::unique_lock* lck, bool triggered) { // pre_vote need defense ABA after unlock&lock if (old_term != _current_term) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " raise term " << _current_term << " when get last_log_id"; + << " raise term " << _current_term + << " when get last_log_id"; return; } @@ -1649,8 +1691,8 @@ void NodeImpl::pre_vote(std::unique_lock* lck, bool triggered) { std::set peers; _conf.list_peers(&peers); - for (std::set::const_iterator - iter = peers.begin(); iter != peers.end(); ++iter) { + for (std::set::const_iterator iter = peers.begin(); + iter != peers.end(); ++iter) { if (*iter == _server_id) { continue; } @@ -1666,12 +1708,12 @@ void NodeImpl::pre_vote(std::unique_lock* lck, bool triggered) { } OnPreVoteRPCDone* done = new OnPreVoteRPCDone( - *iter, _current_term, _pre_vote_ctx.version(), this); + *iter, _current_term, _pre_vote_ctx.version(), this); done->cntl.set_timeout_ms(_options.election_timeout_ms); done->request.set_group_id(_group_id); done->request.set_server_id(_server_id.to_string()); done->request.set_peer_id(iter->to_string()); - done->request.set_term(_current_term + 1); // next term + done->request.set_term(_current_term + 1); // next term done->request.set_last_log_index(last_log_id.index); done->request.set_last_log_term(last_log_id.term); @@ -1682,10 +1724,10 @@ void NodeImpl::pre_vote(std::unique_lock* lck, bool triggered) { } // in lock -void NodeImpl::elect_self(std::unique_lock* lck, +void NodeImpl::elect_self(std::unique_lock* lck, bool old_leader_stepped_down) { - LOG(INFO) << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " start vote and grant vote self"; + LOG(INFO) << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " start vote and grant vote self"; if (_conf.empty()) { LOG(WARNING) << "node " << _group_id << ':' << _server_id << " can't do elect_self as _conf is empty"; @@ -1693,8 +1735,8 @@ void NodeImpl::elect_self(std::unique_lock* lck, } // cancel follower election timer if (_state == STATE_FOLLOWER) { - BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " stop election_timer"; + BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " stop election_timer"; _election_timer.stop(); } // reset leader_id before vote @@ -1702,21 +1744,23 @@ void NodeImpl::elect_self(std::unique_lock* lck, const int64_t leader_term = _current_term; PeerId empty_id; butil::Status status; - status.set_error(ERAFTTIMEDOUT, "A follower's leader_id is reset to NULL " - "as it begins to request_vote."); + status.set_error(ERAFTTIMEDOUT, + "A follower's leader_id is reset to NULL " + "as it begins to request_vote."); reset_leader_id(empty_id, status); _state = STATE_CANDIDATE; _current_term++; _voted_id = _server_id; - BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " term " << _current_term << " start vote_timer"; + BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " start vote_timer"; _vote_timer.start(); _pre_vote_ctx.reset(this); _vote_ctx.init(this, false); if (old_leader_stepped_down) { - _vote_ctx.set_disrupted_leader(DisruptedLeader(old_leader, leader_term)); + _vote_ctx.set_disrupted_leader( + DisruptedLeader(old_leader, leader_term)); _follower_lease.expire(); } @@ -1729,7 +1773,8 @@ void NodeImpl::elect_self(std::unique_lock* lck, if (old_term != _current_term) { // term changed cause by step_down LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " raise term " << _current_term << " when get last_log_id"; + << " raise term " << _current_term + << " when get last_log_id"; return; } _vote_ctx.set_last_log_id(last_log_id); @@ -1738,13 +1783,14 @@ void NodeImpl::elect_self(std::unique_lock* lck, _conf.list_peers(&peers); request_peers_to_vote(peers, _vote_ctx.disrupted_leader()); - //TODO: outof lock - status = _meta_storage-> - set_term_and_votedfor(_current_term, _server_id, _v_group_id); + // TODO: outof lock + status = _meta_storage->set_term_and_votedfor(_current_term, _server_id, + _v_group_id); if (!status.ok()) { LOG(ERROR) << "node " << _group_id << ":" << _server_id << " fail to set_term_and_votedfor itself when elect_self," - " error: " << status; + " error: " + << status; // reset _voted_id to avoid inconsistent cases // return immediately without granting _vote_ctx _voted_id.reset(); @@ -1755,8 +1801,8 @@ void NodeImpl::elect_self(std::unique_lock* lck, void NodeImpl::request_peers_to_vote(const std::set& peers, const DisruptedLeader& disrupted_leader) { - for (std::set::const_iterator - iter = peers.begin(); iter != peers.end(); ++iter) { + for (std::set::const_iterator iter = peers.begin(); + iter != peers.end(); ++iter) { if (*iter == _server_id) { continue; } @@ -1771,8 +1817,8 @@ void NodeImpl::request_peers_to_vote(const std::set& peers, continue; } - OnRequestVoteRPCDone* done = - new OnRequestVoteRPCDone(*iter, _current_term, _vote_ctx.version(), this); + OnRequestVoteRPCDone* done = new OnRequestVoteRPCDone( + *iter, _current_term, _vote_ctx.version(), this); done->cntl.set_timeout_ms(_options.election_timeout_ms); done->request.set_group_id(_group_id); done->request.set_server_id(_server_id.to_string()); @@ -1782,10 +1828,10 @@ void NodeImpl::request_peers_to_vote(const std::set& peers, done->request.set_last_log_term(_vote_ctx.last_log_id().term); if (disrupted_leader.peer_id != ANY_PEER) { - done->request.mutable_disrupted_leader() - ->set_peer_id(disrupted_leader.peer_id.to_string()); - done->request.mutable_disrupted_leader() - ->set_term(disrupted_leader.term); + done->request.mutable_disrupted_leader()->set_peer_id( + disrupted_leader.peer_id.to_string()); + done->request.mutable_disrupted_leader()->set_term( + disrupted_leader.term); } RaftService_Stub stub(&channel); @@ -1794,13 +1840,12 @@ void NodeImpl::request_peers_to_vote(const std::set& peers, } // in lock -void NodeImpl::step_down(const int64_t term, bool wakeup_a_candidate, +void NodeImpl::step_down(const int64_t term, bool wakeup_a_candidate, const butil::Status& status) { - BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " term " << _current_term - << " stepdown from " << state2str(_state) - << " new_term " << term - << " wakeup_a_candidate=" << wakeup_a_candidate; + BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " stepdown from " << state2str(_state) + << " new_term " << term + << " wakeup_a_candidate=" << wakeup_a_candidate; if (!is_active_state(_state)) { return; @@ -1822,8 +1867,8 @@ void NodeImpl::step_down(const int64_t term, bool wakeup_a_candidate, _fsm_caller->on_leader_stop(status); } } - - // reset leader_id + + // reset leader_id PeerId empty_id; reset_leader_id(empty_id, status); @@ -1843,13 +1888,14 @@ void NodeImpl::step_down(const int64_t term, bool wakeup_a_candidate, if (term > _current_term) { _current_term = term; _voted_id.reset(); - //TODO: outof lock - butil::Status status = _meta_storage-> - set_term_and_votedfor(term, _voted_id, _v_group_id); + // TODO: outof lock + butil::Status status = + _meta_storage->set_term_and_votedfor(term, _voted_id, _v_group_id); if (!status.ok()) { - LOG(ERROR) << "node " << _group_id << ":" << _server_id - << " fail to set_term_and_votedfor when step_down, error: " - << status; + LOG(ERROR) + << "node " << _group_id << ":" << _server_id + << " fail to set_term_and_votedfor when step_down, error: " + << status; // TODO report error } } @@ -1857,11 +1903,11 @@ void NodeImpl::step_down(const int64_t term, bool wakeup_a_candidate, // stop stagging new node if (wakeup_a_candidate) { _replicator_group.stop_all_and_find_the_next_candidate( - &_waking_candidate, _conf); + &_waking_candidate, _conf); // FIXME: We issue the RPC in the critical section, which is fine now // since the Node is going to quit when reaching the branch - Replicator::send_timeout_now_and_stop( - _waking_candidate, _options.election_timeout_ms); + Replicator::send_timeout_now_and_stop(_waking_candidate, + _options.election_timeout_ms); } else { _replicator_group.stop_all(); } @@ -1879,20 +1925,20 @@ void NodeImpl::step_down(const int64_t term, bool wakeup_a_candidate, _election_timer.start(); } // in lock -void NodeImpl::reset_leader_id(const PeerId& new_leader_id, - const butil::Status& status) { +void NodeImpl::reset_leader_id(const PeerId& new_leader_id, + const butil::Status& status) { if (new_leader_id.is_empty()) { if (!_leader_id.is_empty() && _state > STATE_TRANSFERRING) { - LeaderChangeContext stop_following_context(_leader_id, - _current_term, status); + LeaderChangeContext stop_following_context(_leader_id, + _current_term, status); _fsm_caller->on_stop_following(stop_following_context); } _leader_id.reset(); } else { if (_leader_id.is_empty()) { _pre_vote_ctx.reset(this); - LeaderChangeContext start_following_context(new_leader_id, - _current_term, status); + LeaderChangeContext start_following_context(new_leader_id, + _current_term, status); _fsm_caller->on_start_following(start_following_context); } _leader_id = new_leader_id; @@ -1900,30 +1946,35 @@ void NodeImpl::reset_leader_id(const PeerId& new_leader_id, } // in lock -void NodeImpl::check_step_down(const int64_t request_term, const PeerId& server_id) { +void NodeImpl::check_step_down(const int64_t request_term, + const PeerId& server_id) { butil::Status status; if (request_term > _current_term) { - status.set_error(ENEWLEADER, "Raft node receives message from " - "new leader with higher term."); + status.set_error(ENEWLEADER, + "Raft node receives message from " + "new leader with higher term."); step_down(request_term, false, status); - } else if (_state != STATE_FOLLOWER) { - status.set_error(ENEWLEADER, "Candidate receives message " - "from new leader with the same term."); + } else if (_state != STATE_FOLLOWER) { + status.set_error(ENEWLEADER, + "Candidate receives message " + "from new leader with the same term."); step_down(request_term, false, status); } else if (_leader_id.is_empty()) { - status.set_error(ENEWLEADER, "Follower receives message " - "from new leader with the same term."); - step_down(request_term, false, status); + status.set_error(ENEWLEADER, + "Follower receives message " + "from new leader with the same term."); + step_down(request_term, false, status); } // save current leader - if (_leader_id.is_empty()) { + if (_leader_id.is_empty()) { reset_leader_id(server_id, status); } } class LeaderStartClosure : public Closure { -public: - LeaderStartClosure(StateMachine* fsm, int64_t term) : _fsm(fsm), _term(term) {} + public: + LeaderStartClosure(StateMachine* fsm, int64_t term) + : _fsm(fsm), _term(term) {} ~LeaderStartClosure() {} void Run() { if (status().ok()) { @@ -1931,7 +1982,8 @@ class LeaderStartClosure : public Closure { } delete this; } -private: + + private: StateMachine* _fsm; int64_t _term; }; @@ -1939,9 +1991,8 @@ class LeaderStartClosure : public Closure { // in lock void NodeImpl::become_leader() { CHECK(_state == STATE_CANDIDATE); - LOG(INFO) << "node " << _group_id << ":" << _server_id - << " term " << _current_term - << " become leader of group " << _conf.conf + LOG(INFO) << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " become leader of group " << _conf.conf << " " << _conf.old_conf; // cancel candidate vote timer _vote_timer.stop(); @@ -1956,16 +2007,15 @@ void NodeImpl::become_leader() { std::set peers; _conf.list_peers(&peers); - for (std::set::const_iterator - iter = peers.begin(); iter != peers.end(); ++iter) { + for (std::set::const_iterator iter = peers.begin(); + iter != peers.end(); ++iter) { if (*iter == _server_id) { continue; } - BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " term " << _current_term - << " add replicator " << *iter; - //TODO: check return code + BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " term " + << _current_term << " add replicator " << *iter; + // TODO: check return code _replicator_group.add_replicator(*iter); } @@ -1980,48 +2030,45 @@ void NodeImpl::become_leader() { } class LeaderStableClosure : public LogManager::StableClosure { -public: + public: void Run(); -private: - LeaderStableClosure(const NodeId& node_id, - size_t nentries, + + private: + LeaderStableClosure(const NodeId& node_id, size_t nentries, BallotBox* ballot_box); ~LeaderStableClosure() {} -friend class NodeImpl; + friend class NodeImpl; NodeId _node_id; size_t _nentries; BallotBox* _ballot_box; }; -LeaderStableClosure::LeaderStableClosure(const NodeId& node_id, - size_t nentries, +LeaderStableClosure::LeaderStableClosure(const NodeId& node_id, size_t nentries, BallotBox* ballot_box) - : _node_id(node_id), _nentries(nentries), _ballot_box(ballot_box) -{ -} + : _node_id(node_id), _nentries(nentries), _ballot_box(ballot_box) {} void LeaderStableClosure::Run() { if (status().ok()) { if (_ballot_box) { // ballot_box check quorum ok, will call fsm_caller - _ballot_box->commit_at( - _first_log_index, _first_log_index + _nentries - 1, _node_id.peer_id); + _ballot_box->commit_at(_first_log_index, + _first_log_index + _nentries - 1, + _node_id.peer_id); } int64_t now = butil::cpuwide_time_us(); - if (FLAGS_raft_trace_append_entry_latency && - now - metric.start_time_us > (int64_t)FLAGS_raft_append_entry_high_lat_us) { - LOG(WARNING) << "leader append entry latency us " << (now - metric.start_time_us) - << " greater than " - << FLAGS_raft_append_entry_high_lat_us - << metric - << " node " << _node_id - << " log_index [" << _first_log_index - << ", " << _first_log_index + _nentries - 1 - << "]"; + if (FLAGS_raft_trace_append_entry_latency && + now - metric.start_time_us > + (int64_t)FLAGS_raft_append_entry_high_lat_us) { + LOG(WARNING) << "leader append entry latency us " + << (now - metric.start_time_us) << " greater than " + << FLAGS_raft_append_entry_high_lat_us << metric + << " node " << _node_id << " log_index [" + << _first_log_index << ", " + << _first_log_index + _nentries - 1 << "]"; } } else { - LOG(ERROR) << "node " << _node_id << " append [" << _first_log_index << ", " - << _first_log_index + _nentries - 1 << "] failed"; + LOG(ERROR) << "node " << _node_id << " append [" << _first_log_index + << ", " << _first_log_index + _nentries - 1 << "] failed"; } delete this; } @@ -2043,7 +2090,8 @@ void NodeImpl::apply(LogEntryAndClosure tasks[], size_t size) { st.set_error(EBUSY, "is transferring leadership"); } lck.unlock(); - BRAFT_VLOG << "node " << _group_id << ":" << _server_id << " can't apply : " << st; + BRAFT_VLOG << "node " << _group_id << ":" << _server_id + << " can't apply : " << st; for (size_t i = 0; i < size; ++i) { tasks[i].entry->Release(); if (tasks[i].done) { @@ -2054,14 +2102,18 @@ void NodeImpl::apply(LogEntryAndClosure tasks[], size_t size) { return; } for (size_t i = 0; i < size; ++i) { - if (tasks[i].expected_term != -1 && tasks[i].expected_term != _current_term) { + if (tasks[i].expected_term != -1 && + tasks[i].expected_term != _current_term) { BRAFT_VLOG << "node " << _group_id << ":" << _server_id - << " can't apply taks whose expected_term=" << tasks[i].expected_term - << " doesn't match current_term=" << _current_term; + << " can't apply taks whose expected_term=" + << tasks[i].expected_term + << " doesn't match current_term=" << _current_term; if (tasks[i].done) { tasks[i].done->status().set_error( - EPERM, "expected_term=%" PRId64 " doesn't match current_term=%" PRId64, - tasks[i].expected_term, _current_term); + EPERM, + "expected_term=%" PRId64 + " doesn't match current_term=%" PRId64, + tasks[i].expected_term, _current_term); run_closure_in_bthread(tasks[i].done); } tasks[i].entry->Release(); @@ -2074,11 +2126,9 @@ void NodeImpl::apply(LogEntryAndClosure tasks[], size_t size) { _conf.conf, _conf.stable() ? nullptr : &_conf.old_conf, tasks[i].done); } - _log_manager->append_entries(&entries, - new LeaderStableClosure( - NodeId(_group_id, _server_id), - entries.size(), - _ballot_box)); + _log_manager->append_entries( + &entries, new LeaderStableClosure(NodeId(_group_id, _server_id), + entries.size(), _ballot_box)); // update _conf.first _log_manager->check_and_set_configuration(&_conf); } @@ -2098,31 +2148,32 @@ void NodeImpl::unsafe_apply_configuration(const Configuration& new_conf, old_conf->list_peers(entry->old_peers); } ConfigurationChangeDone* configuration_change_done = - new ConfigurationChangeDone(this, _current_term, leader_start, _leader_lease.lease_epoch()); + new ConfigurationChangeDone(this, _current_term, leader_start, + _leader_lease.lease_epoch()); // Use the new_conf to deal the quorum of this very log - _ballot_box->append_pending_task(new_conf, old_conf, configuration_change_done); + _ballot_box->append_pending_task(new_conf, old_conf, + configuration_change_done); std::vector entries; entries.push_back(entry); - _log_manager->append_entries(&entries, - new LeaderStableClosure( - NodeId(_group_id, _server_id), - 1u, _ballot_box)); + _log_manager->append_entries( + &entries, new LeaderStableClosure(NodeId(_group_id, _server_id), 1u, + _ballot_box)); _log_manager->check_and_set_configuration(&_conf); } int NodeImpl::handle_pre_vote_request(const RequestVoteRequest* request, RequestVoteResponse* response) { std::unique_lock lck(_mutex); - + if (!is_active_state(_state)) { const int64_t saved_current_term = _current_term; const State saved_state = _state; lck.unlock(); - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " is not in active state " << "current_term " - << saved_current_term - << " state " << state2str(saved_state); + LOG(WARNING) << "node " << _group_id << ":" << _server_id + << " is not in active state " + << "current_term " << saved_current_term << " state " + << state2str(saved_state); return EINVAL; } @@ -2141,8 +2192,8 @@ int NodeImpl::handle_pre_vote_request(const RequestVoteRequest* request, // ignore older term LOG(INFO) << "node " << _group_id << ":" << _server_id << " ignore PreVote from " << request->server_id() - << " in term " << request->term() - << " current_term " << _current_term; + << " in term " << request->term() << " current_term " + << _current_term; break; } @@ -2152,8 +2203,8 @@ int NodeImpl::handle_pre_vote_request(const RequestVoteRequest* request, lck.lock(); // pre_vote not need ABA check after unlock&lock int64_t votable_time = _follower_lease.votable_time_from_now(); - bool grantable = (LogId(request->last_log_index(), request->last_log_term()) - >= last_log_id); + bool grantable = (LogId(request->last_log_index(), + request->last_log_term()) >= last_log_id); if (grantable) { granted = (votable_time == 0); rejected_by_lease = (votable_time > 0); @@ -2161,9 +2212,8 @@ int NodeImpl::handle_pre_vote_request(const RequestVoteRequest* request, LOG(INFO) << "node " << _group_id << ":" << _server_id << " received PreVote from " << request->server_id() - << " in term " << request->term() - << " current_term " << _current_term - << " granted " << granted + << " in term " << request->term() << " current_term " + << _current_term << " granted " << granted << " rejected_by_lease " << rejected_by_lease; } while (0); @@ -2185,10 +2235,10 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, const int64_t saved_current_term = _current_term; const State saved_state = _state; lck.unlock(); - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " is not in active state " << "current_term " - << saved_current_term - << " state " << state2str(saved_state); + LOG(WARNING) << "node " << _group_id << ":" << _server_id + << " is not in active state " + << "current_term " << saved_current_term << " state " + << state2str(saved_state); return EINVAL; } @@ -2201,11 +2251,10 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, } PeerId disrupted_leader_id; - if (_state == STATE_FOLLOWER && - request->has_disrupted_leader() && - _current_term == request->disrupted_leader().term() && - 0 == disrupted_leader_id.parse(request->disrupted_leader().peer_id()) && - _leader_id == disrupted_leader_id) { + if (_state == STATE_FOLLOWER && request->has_disrupted_leader() && + _current_term == request->disrupted_leader().term() && + 0 == disrupted_leader_id.parse(request->disrupted_leader().peer_id()) && + _leader_id == disrupted_leader_id) { // The candidate has already disrupted the old leader, we // can expire the lease safely. _follower_lease.expire(); @@ -2220,8 +2269,8 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, // ignore older term LOG(INFO) << "node " << _group_id << ":" << _server_id << " ignore RequestVote from " << request->server_id() - << " in term " << request->term() - << " current_term " << _current_term; + << " in term " << request->term() << " current_term " + << _current_term; break; } @@ -2233,19 +2282,19 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, // vote need ABA check after unlock&lock if (previous_term != _current_term) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " raise term " << _current_term << " when get last_log_id"; + << " raise term " << _current_term + << " when get last_log_id"; break; } - bool log_is_ok = (LogId(request->last_log_index(), request->last_log_term()) - >= last_log_id); + bool log_is_ok = (LogId(request->last_log_index(), + request->last_log_term()) >= last_log_id); int64_t votable_time = _follower_lease.votable_time_from_now(); LOG(INFO) << "node " << _group_id << ":" << _server_id << " received RequestVote from " << request->server_id() - << " in term " << request->term() - << " current_term " << _current_term - << " log_is_ok " << log_is_ok + << " in term " << request->term() << " current_term " + << _current_term << " log_is_ok " << log_is_ok << " votable_time " << votable_time; // if the vote is rejected by lease, tell the candidate @@ -2257,8 +2306,9 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, // increase current term, change state to follower if (request->term() > _current_term) { butil::Status status; - status.set_error(EHIGHERTERMREQUEST, "Raft node receives higher term " - "request_vote_request."); + status.set_error(EHIGHERTERMREQUEST, + "Raft node receives higher term " + "request_vote_request."); disrupted = (_state <= STATE_TRANSFERRING); step_down(request->term(), false, status); } @@ -2266,20 +2316,21 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, // save if (log_is_ok && _voted_id.is_empty()) { butil::Status status; - status.set_error(EVOTEFORCANDIDATE, "Raft node votes for some candidate, " - "step down to restart election_timer."); + status.set_error(EVOTEFORCANDIDATE, + "Raft node votes for some candidate, " + "step down to restart election_timer."); step_down(request->term(), false, status); _voted_id = candidate_id; - //TODO: outof lock - status = _meta_storage-> - set_term_and_votedfor(_current_term, candidate_id, _v_group_id); + // TODO: outof lock + status = _meta_storage->set_term_and_votedfor( + _current_term, candidate_id, _v_group_id); if (!status.ok()) { LOG(ERROR) << "node " << _group_id << ":" << _server_id << " refuse to vote for " << request->server_id() << " because failed to set_votedfor it, error: " << status; // reset _voted_id to response set_granted(false) - _voted_id.reset(); + _voted_id.reset(); } } } while (0); @@ -2287,34 +2338,33 @@ int NodeImpl::handle_request_vote_request(const RequestVoteRequest* request, response->set_disrupted(disrupted); response->set_previous_term(previous_term); response->set_term(_current_term); - response->set_granted(request->term() == _current_term && _voted_id == candidate_id); + response->set_granted(request->term() == _current_term && + _voted_id == candidate_id); response->set_rejected_by_lease(rejected_by_lease); return 0; } class FollowerStableClosure : public LogManager::StableClosure { -public: - FollowerStableClosure( - brpc::Controller* cntl, - const AppendEntriesRequest* request, - AppendEntriesResponse* response, - google::protobuf::Closure* done, - NodeImpl* node, - int64_t term) - : _cntl(cntl) - , _request(request) - , _response(response) - , _done(done) - , _node(node) - , _term(term) - { + public: + FollowerStableClosure(brpc::Controller* cntl, + const AppendEntriesRequest* request, + AppendEntriesResponse* response, + google::protobuf::Closure* done, NodeImpl* node, + int64_t term) + : _cntl(cntl), + _request(request), + _response(response), + _done(done), + _node(node), + _term(term) { _node->AddRef(); } void Run() { run(); delete this; } -private: + + private: ~FollowerStableClosure() { if (_node) { _node->Release(); @@ -2356,26 +2406,27 @@ class FollowerStableClosure : public LogManager::StableClosure { _response->set_term(_term); const int64_t committed_index = - std::min(_request->committed_index(), - // ^^^ committed_index is likely less than the - // last_log_index - _request->prev_log_index() + _request->entries_size() - // ^^^ The logs after the appended entries are - // untrustable so we can't commit them even if their - // indexes are less than request->committed_index() - ); + std::min(_request->committed_index(), + // ^^^ committed_index is likely less than the + // last_log_index + _request->prev_log_index() + _request->entries_size() + // ^^^ The logs after the appended entries are + // untrustable so we can't commit them even if their + // indexes are less than request->committed_index() + ); //_ballot_box is thread safe and tolerates disorder. _node->_ballot_box->set_last_committed_index(committed_index); int64_t now = butil::cpuwide_time_us(); - if (FLAGS_raft_trace_append_entry_latency && now - metric.start_time_us > + if (FLAGS_raft_trace_append_entry_latency && + now - metric.start_time_us > (int64_t)FLAGS_raft_append_entry_high_lat_us) { - LOG(WARNING) << "follower append entry latency us " << (now - metric.start_time_us) - << " greater than " - << FLAGS_raft_append_entry_high_lat_us - << metric - << " node " << _node->node_id() - << " log_index [" << _request->prev_log_index() + 1 - << ", " << _request->prev_log_index() + _request->entries_size() - 1 + LOG(WARNING) << "follower append entry latency us " + << (now - metric.start_time_us) << " greater than " + << FLAGS_raft_append_entry_high_lat_us << metric + << " node " << _node->node_id() << " log_index [" + << _request->prev_log_index() + 1 << ", " + << _request->prev_log_index() + + _request->entries_size() - 1 << "]"; } } @@ -2388,11 +2439,10 @@ class FollowerStableClosure : public LogManager::StableClosure { int64_t _term; }; -void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, - const AppendEntriesRequest* request, - AppendEntriesResponse* response, - google::protobuf::Closure* done, - bool from_append_entries_cache) { +void NodeImpl::handle_append_entries_request( + brpc::Controller* cntl, const AppendEntriesRequest* request, + AppendEntriesResponse* response, google::protobuf::Closure* done, + bool from_append_entries_cache) { std::vector entries; entries.reserve(request->entries_size()); brpc::ClosureGuard done_guard(done); @@ -2405,11 +2455,13 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, const int64_t saved_current_term = _current_term; const State saved_state = _state; lck.unlock(); - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " is not in active state " << "current_term " << saved_current_term - << " state " << state2str(saved_state); - cntl->SetFailed(EINVAL, "node %s:%s is not in active state, state %s", - _group_id.c_str(), _server_id.to_string().c_str(), state2str(saved_state)); + LOG(WARNING) << "node " << _group_id << ":" << _server_id + << " is not in active state " + << "current_term " << saved_current_term << " state " + << state2str(saved_state); + cntl->SetFailed(EINVAL, "node %s:%s is not in active state, state %s", + _group_id.c_str(), _server_id.to_string().c_str(), + state2str(saved_state)); return; } @@ -2419,8 +2471,7 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, LOG(WARNING) << "node " << _group_id << ":" << _server_id << " received AppendEntries from " << request->server_id() << " server_id bad format"; - cntl->SetFailed(brpc::EREQUEST, - "Fail to parse server_id `%s'", + cntl->SetFailed(brpc::EREQUEST, "Fail to parse server_id `%s'", request->server_id().c_str()); return; } @@ -2430,8 +2481,8 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, const int64_t saved_current_term = _current_term; lck.unlock(); LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " ignore stale AppendEntries from " << request->server_id() - << " in term " << request->term() + << " ignore stale AppendEntries from " + << request->server_id() << " in term " << request->term() << " current_term " << saved_current_term; response->set_success(false); response->set_term(saved_current_term); @@ -2439,16 +2490,18 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, } // check term and state to step down - check_step_down(request->term(), server_id); - + check_step_down(request->term(), server_id); + if (server_id != _leader_id) { LOG(ERROR) << "Another peer " << _group_id << ":" << server_id - << " declares that it is the leader at term=" << _current_term + << " declares that it is the leader at term=" + << _current_term << " which was occupied by leader=" << _leader_id; - // Increase the term by 1 and make both leaders step down to minimize the - // loss of split brain + // Increase the term by 1 and make both leaders step down to minimize + // the loss of split brain butil::Status status; - status.set_error(ELEADERCONFLICT, "More than one leader in the same term."); + status.set_error(ELEADERCONFLICT, + "More than one leader in the same term."); step_down(request->term() + 1, false, status); response->set_success(false); response->set_term(request->term() + 1); @@ -2461,8 +2514,7 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, } if (request->entries_size() > 0 && - (_snapshot_executor - && _snapshot_executor->is_installing_snapshot())) { + (_snapshot_executor && _snapshot_executor->is_installing_snapshot())) { LOG(WARNING) << "node " << _group_id << ":" << _server_id << " received append entries while installing snapshot"; cntl->SetFailed(EBUSY, "Is installing snapshot"); @@ -2475,24 +2527,23 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, if (local_prev_log_term != prev_log_term) { int64_t last_index = _log_manager->last_log_index(); int64_t saved_term = request->term(); - int saved_entries_size = request->entries_size(); + int saved_entries_size = request->entries_size(); std::string rpc_server_id = request->server_id(); if (!from_append_entries_cache && - handle_out_of_order_append_entries( - cntl, request, response, done, last_index)) { - // It's not safe to touch cntl/request/response/done after this point, - // since the ownership is tranfered to the cache. + handle_out_of_order_append_entries(cntl, request, response, done, + last_index)) { + // It's not safe to touch cntl/request/response/done after this + // point, since the ownership is tranfered to the cache. lck.unlock(); done_guard.release(); LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " cache out-of-order AppendEntries from " - << rpc_server_id - << " in term " << saved_term + << " cache out-of-order AppendEntries from " + << rpc_server_id << " in term " << saved_term << " prev_log_index " << prev_log_index << " prev_log_term " << prev_log_term << " local_prev_log_term " << local_prev_log_term - << " last_log_index " << last_index - << " entries_size " << saved_entries_size; + << " last_log_index " << last_index << " entries_size " + << saved_entries_size; return; } @@ -2502,15 +2553,16 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, lck.unlock(); if (local_prev_log_term != 0) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " reject term_unmatched AppendEntries from " - << request->server_id() - << " in term " << request->term() - << " prev_log_index " << request->prev_log_index() - << " prev_log_term " << request->prev_log_term() - << " local_prev_log_term " << local_prev_log_term - << " last_log_index " << last_index - << " entries_size " << request->entries_size() - << " from_append_entries_cache: " << from_append_entries_cache; + << " reject term_unmatched AppendEntries from " + << request->server_id() << " in term " + << request->term() << " prev_log_index " + << request->prev_log_index() << " prev_log_term " + << request->prev_log_term() << " local_prev_log_term " + << local_prev_log_term << " last_log_index " + << last_index << " entries_size " + << request->entries_size() + << " from_append_entries_cache: " + << from_append_entries_cache; } return; } @@ -2523,8 +2575,7 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, lck.unlock(); // see the comments at FollowerStableClosure::run() _ballot_box->set_last_committed_index( - std::min(request->committed_index(), - prev_log_index)); + std::min(request->committed_index(), prev_log_index)); return; } @@ -2568,8 +2619,7 @@ void NodeImpl::handle_append_entries_request(brpc::Controller* cntl, check_append_entries_cache(index); FollowerStableClosure* c = new FollowerStableClosure( - cntl, request, response, done_guard.release(), - this, _current_term); + cntl, request, response, done_guard.release(), this, _current_term); _log_manager->append_entries(&entries, c); // update configuration after _log_manager updated its memory status @@ -2585,9 +2635,7 @@ int NodeImpl::increase_term_to(int64_t new_term, const butil::Status& status) { return 0; } -void NodeImpl::after_shutdown(NodeImpl* node) { - return node->after_shutdown(); -} +void NodeImpl::after_shutdown(NodeImpl* node) { return node->after_shutdown(); } void NodeImpl::after_shutdown() { std::vector saved_done; @@ -2606,10 +2654,9 @@ void NodeImpl::after_shutdown() { } } -void NodeImpl::handle_install_snapshot_request(brpc::Controller* cntl, - const InstallSnapshotRequest* request, - InstallSnapshotResponse* response, - google::protobuf::Closure* done) { +void NodeImpl::handle_install_snapshot_request( + brpc::Controller* cntl, const InstallSnapshotRequest* request, + InstallSnapshotResponse* response, google::protobuf::Closure* done) { brpc::ClosureGuard done_guard(done); if (_snapshot_executor == NULL) { cntl->SetFailed(EINVAL, "Not support snapshot"); @@ -2622,40 +2669,44 @@ void NodeImpl::handle_install_snapshot_request(brpc::Controller* cntl, return; } std::unique_lock lck(_mutex); - + if (!is_active_state(_state)) { const int64_t saved_current_term = _current_term; const State saved_state = _state; lck.unlock(); - LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " is not in active state " << "current_term " - << saved_current_term << " state " << state2str(saved_state); - cntl->SetFailed(EINVAL, "node %s:%s is not in active state, state %s", - _group_id.c_str(), _server_id.to_string().c_str(), state2str(saved_state)); + LOG(WARNING) << "node " << _group_id << ":" << _server_id + << " is not in active state " + << "current_term " << saved_current_term << " state " + << state2str(saved_state); + cntl->SetFailed(EINVAL, "node %s:%s is not in active state, state %s", + _group_id.c_str(), _server_id.to_string().c_str(), + state2str(saved_state)); return; } // check stale term if (request->term() < _current_term) { LOG(WARNING) << "node " << _group_id << ":" << _server_id - << " ignore stale InstallSnapshot from " << request->server_id() - << " in term " << request->term() + << " ignore stale InstallSnapshot from " + << request->server_id() << " in term " << request->term() << " current_term " << _current_term; response->set_term(_current_term); response->set_success(false); return; } - + check_step_down(request->term(), server_id); if (server_id != _leader_id) { LOG(ERROR) << "Another peer " << _group_id << ":" << server_id - << " declares that it is the leader at term=" << _current_term + << " declares that it is the leader at term=" + << _current_term << " which was occupied by leader=" << _leader_id; - // Increase the term by 1 and make both leaders step down to minimize the - // loss of split brain + // Increase the term by 1 and make both leaders step down to minimize + // the loss of split brain butil::Status status; - status.set_error(ELEADERCONFLICT, "More than one leader in the same term."); + status.set_error(ELEADERCONFLICT, + "More than one leader in the same term."); step_down(request->term() + 1, false, status); response->set_success(false); response->set_term(request->term() + 1); @@ -2668,11 +2719,10 @@ void NodeImpl::handle_install_snapshot_request(brpc::Controller* cntl, << " last_included_log_index=" << request->meta().last_included_index() << " last_include_log_term=" - << request->meta().last_included_term() - << " from " << server_id + << request->meta().last_included_term() << " from " << server_id << " when last_log_id=" << _log_manager->last_log_id(); - return _snapshot_executor->install_snapshot( - cntl, request, response, done_guard.release()); + return _snapshot_executor->install_snapshot(cntl, request, response, + done_guard.release()); } void NodeImpl::update_configuration_after_installing_snapshot() { @@ -2680,19 +2730,25 @@ void NodeImpl::update_configuration_after_installing_snapshot() { _log_manager->check_and_set_configuration(&_conf); } -butil::Status NodeImpl::read_committed_user_log(const int64_t index, UserLog* user_log) { +butil::Status NodeImpl::read_committed_user_log(const int64_t index, + UserLog* user_log) { if (index <= 0) { - return butil::Status(EINVAL, "request index:%" PRId64 " is invalid.", index); + return butil::Status(EINVAL, "request index:%" PRId64 " is invalid.", + index); } const int64_t saved_last_applied_index = _fsm_caller->last_applied_index(); if (index > saved_last_applied_index) { - return butil::Status(ENOMOREUSERLOG, "request index:%" PRId64 " is greater" - " than last_applied_index:%" PRId64, index, saved_last_applied_index); + return butil::Status(ENOMOREUSERLOG, + "request index:%" PRId64 + " is greater" + " than last_applied_index:%" PRId64, + index, saved_last_applied_index); } int64_t cur_index = index; LogEntry* entry = _log_manager->get_entry(cur_index); if (entry == NULL) { - return butil::Status(ELOGDELETED, "user log is deleted at index:%" PRId64, index); + return butil::Status(ELOGDELETED, + "user log is deleted at index:%" PRId64, index); } do { if (entry->type == ENTRY_TYPE_DATA) { @@ -2705,14 +2761,17 @@ butil::Status NodeImpl::read_committed_user_log(const int64_t index, UserLog* us ++cur_index; } if (cur_index > saved_last_applied_index) { - return butil::Status(ENOMOREUSERLOG, "no user log between index:%" PRId64 - " and last_applied_index:%" PRId64, index, saved_last_applied_index); + return butil::Status(ENOMOREUSERLOG, + "no user log between index:%" PRId64 + " and last_applied_index:%" PRId64, + index, saved_last_applied_index); } entry = _log_manager->get_entry(cur_index); } while (entry != NULL); - // entry is likely to be NULL because snapshot is done after + // entry is likely to be NULL because snapshot is done after // getting saved_last_applied_index. - return butil::Status(ELOGDELETED, "user log is deleted at index:%" PRId64, cur_index); + return butil::Status(ELOGDELETED, "user log is deleted at index:%" PRId64, + cur_index); } void NodeImpl::describe(std::ostream& os, bool use_html) { @@ -2725,13 +2784,13 @@ void NodeImpl::describe(std::ostream& os, bool use_html) { } const int64_t term = _current_term; const int64_t conf_index = _conf.id.index; - //const int ref_count = ref_count_; + // const int ref_count = ref_count_; std::vector peers; _conf.conf.list_peers(&peers); const std::string is_changing_conf = _conf_ctx.is_busy() ? "YES" : "NO"; - const char* conf_statge = _conf_ctx.stage_str(); - // new_peers and old_peers during all conf-change stages, namely + const char* conf_statge = _conf_ctx.stage_str(); + // new_peers and old_peers during all conf-change stages, namely // STAGE_CATCHING_UP->STAGE_JOINT->STAGE_STABLE std::vector new_peers; _conf_ctx.list_new_peers(&new_peers); @@ -2743,7 +2802,7 @@ void NodeImpl::describe(std::ostream& os, bool use_html) { const int64_t leader_timestamp = _follower_lease.last_leader_timestamp(); const bool readonly = (_node_readonly || _majority_nodes_readonly); lck.unlock(); - const char *newline = use_html ? "
" : "\r\n"; + const char* newline = use_html ? "
" : "\r\n"; os << "peer_id: " << _server_id << newline; os << "state: " << state2str(st) << newline; os << "readonly: " << readonly << newline; @@ -2753,8 +2812,8 @@ void NodeImpl::describe(std::ostream& os, bool use_html) { for (size_t j = 0; j < peers.size(); ++j) { os << ' '; if (use_html && peers[j] != _server_id) { - os << ""; + os << ""; } os << peers[j]; if (use_html && peers[j] != _server_id) { @@ -2766,15 +2825,15 @@ void NodeImpl::describe(std::ostream& os, bool use_html) { // info of configuration change if (st == STATE_LEADER) { os << "changing_conf: " << is_changing_conf - << " stage: " << conf_statge << newline; + << " stage: " << conf_statge << newline; } if (!new_peers.empty()) { os << "new_peers:"; for (size_t j = 0; j < new_peers.size(); ++j) { os << ' '; if (use_html && new_peers[j] != _server_id) { - os << ""; + os << ""; } os << new_peers[j]; if (use_html && new_peers[j] != _server_id) { @@ -2788,8 +2847,8 @@ void NodeImpl::describe(std::ostream& os, bool use_html) { for (size_t j = 0; j < old_peers.size(); ++j) { os << ' '; if (use_html && old_peers[j] != _server_id) { - os << ""; + os << ""; } os << old_peers[j]; if (use_html && old_peers[j] != _server_id) { @@ -2802,15 +2861,14 @@ void NodeImpl::describe(std::ostream& os, bool use_html) { if (st == STATE_FOLLOWER) { os << "leader: "; if (use_html) { - os << "" - << leader << ""; + os << "" << leader << ""; } else { os << leader; } os << newline; - os << "last_msg_to_now: " << butil::monotonic_time_ms() - leader_timestamp - << newline; + os << "last_msg_to_now: " + << butil::monotonic_time_ms() - leader_timestamp << newline; } // Show timers @@ -2853,8 +2911,7 @@ void NodeImpl::get_status(NodeStatus* status) { _conf.conf.list_peers(&peers); _replicator_group.list_replicators(&replicators); - if (status->state == STATE_LEADER || - status->state == STATE_TRANSFERRING) { + if (status->state == STATE_LEADER || status->state == STATE_TRANSFERRING) { status->leader_id = _server_id; } else if (status->state == STATE_FOLLOWER) { status->leader_id = _leader_id; @@ -2876,7 +2933,7 @@ void NodeImpl::get_status(NodeStatus* status) { status->pending_queue_size = ballot_box_status.pending_queue_size; status->applying_index = _fsm_caller->applying_index(); - + if (replicators.size() == 0) { return; } @@ -2892,8 +2949,9 @@ void NodeImpl::get_status(NodeStatus* status) { NodeStatus::PeerStatusMap::iterator it = status->stable_followers.find(replicators[i].first); if (it == status->stable_followers.end()) { - it = status->unstable_followers.insert( - std::make_pair(replicators[i].first, PeerStatus())).first; + it = status->unstable_followers + .insert(std::make_pair(replicators[i].first, PeerStatus())) + .first; } Replicator::get_status(replicators[i].second, &(it->second)); } @@ -2901,26 +2959,26 @@ void NodeImpl::get_status(NodeStatus* status) { void NodeImpl::stop_replicator(const std::set& keep, const std::set& drop) { - for (std::set::const_iterator - iter = drop.begin(); iter != drop.end(); ++iter) { + for (std::set::const_iterator iter = drop.begin(); + iter != drop.end(); ++iter) { if (keep.find(*iter) == keep.end() && *iter != _server_id) { _replicator_group.stop_replicator(*iter); } } } -bool NodeImpl::handle_out_of_order_append_entries(brpc::Controller* cntl, - const AppendEntriesRequest* request, - AppendEntriesResponse* response, - google::protobuf::Closure* done, - int64_t local_last_index) { +bool NodeImpl::handle_out_of_order_append_entries( + brpc::Controller* cntl, const AppendEntriesRequest* request, + AppendEntriesResponse* response, google::protobuf::Closure* done, + int64_t local_last_index) { if (!FLAGS_raft_enable_append_entries_cache || local_last_index >= request->prev_log_index() || request->entries_size() == 0) { return false; } if (!_append_entries_cache) { - _append_entries_cache = new AppendEntriesCache(this, ++_append_entries_cache_version); + _append_entries_cache = + new AppendEntriesCache(this, ++_append_entries_cache_version); } AppendEntriesRpc* rpc = new AppendEntriesRpc; rpc->cntl = cntl; @@ -2957,7 +3015,8 @@ void NodeImpl::clear_append_entries_cache() { } void* NodeImpl::handle_append_entries_from_cache(void* arg) { - HandleAppendEntriesFromCacheArg* handle_arg = (HandleAppendEntriesFromCacheArg*)arg; + HandleAppendEntriesFromCacheArg* handle_arg = + (HandleAppendEntriesFromCacheArg*)arg; NodeImpl* node = handle_arg->node; butil::LinkedList& rpcs = handle_arg->rpcs; while (!rpcs.empty()) { @@ -2974,9 +3033,9 @@ void* NodeImpl::handle_append_entries_from_cache(void* arg) { void NodeImpl::on_append_entries_cache_timedout(void* arg) { bthread_t tid; - if (bthread_start_background( - &tid, NULL, NodeImpl::handle_append_entries_cache_timedout, - arg) != 0) { + if (bthread_start_background(&tid, NULL, + NodeImpl::handle_append_entries_cache_timedout, + arg) != 0) { PLOG(ERROR) << "Fail to start bthread"; NodeImpl::handle_append_entries_cache_timedout(arg); } @@ -2995,9 +3054,10 @@ void* NodeImpl::handle_append_entries_cache_timedout(void* arg) { std::unique_lock lck(node->_mutex); if (node->_append_entries_cache && - timer_arg->cache_version == node->_append_entries_cache->cache_version()) { + timer_arg->cache_version == + node->_append_entries_cache->cache_version()) { node->_append_entries_cache->do_handle_append_entries_cache_timedout( - timer_arg->timer_version, timer_arg->timer_start_ms); + timer_arg->timer_version, timer_arg->timer_start_ms); if (node->_append_entries_cache->empty()) { delete node->_append_entries_cache; node->_append_entries_cache = NULL; @@ -3019,9 +3079,7 @@ int64_t NodeImpl::AppendEntriesCache::cache_version() const { return _cache_version; } -bool NodeImpl::AppendEntriesCache::empty() const { - return _rpc_map.empty(); -} +bool NodeImpl::AppendEntriesCache::empty() const { return _rpc_map.empty(); } bool NodeImpl::AppendEntriesCache::store(AppendEntriesRpc* rpc) { if (!_rpc_map.empty()) { @@ -3031,14 +3089,15 @@ bool NodeImpl::AppendEntriesCache::store(AppendEntriesRpc* rpc) { int64_t rpc_prev_index = rpc->request->prev_log_index(); int64_t rpc_last_index = rpc_prev_index + rpc->request->entries_size(); - // Some rpcs with the overlap log index alredy exist, means retransmission - // happend, simplely clean all out of order requests, and store the new - // one. + // Some rpcs with the overlap log index alredy exist, means + // retransmission happend, simplely clean all out of order requests, and + // store the new one. if (it != _rpc_map.begin()) { --it; AppendEntriesRpc* prev_rpc = it->second; if (prev_rpc->request->prev_log_index() + - prev_rpc->request->entries_size() > rpc_prev_index) { + prev_rpc->request->entries_size() > + rpc_prev_index) { need_clear = true; } ++it; @@ -3082,17 +3141,19 @@ bool NodeImpl::AppendEntriesCache::store(AppendEntriesRpc* rpc) { return true; } -void NodeImpl::AppendEntriesCache::process_runable_rpcs(int64_t local_last_index) { +void NodeImpl::AppendEntriesCache::process_runable_rpcs( + int64_t local_last_index) { CHECK(!_rpc_map.empty()); CHECK(!_rpc_queue.empty()); HandleAppendEntriesFromCacheArg* arg = NULL; for (std::map::iterator it = _rpc_map.begin(); - it != _rpc_map.end();) { + it != _rpc_map.end();) { AppendEntriesRpc* rpc = it->second; if (rpc->request->prev_log_index() > local_last_index) { break; } - local_last_index = rpc->request->prev_log_index() + rpc->request->entries_size(); + local_last_index = + rpc->request->prev_log_index() + rpc->request->entries_size(); _rpc_map.erase(it++); rpc->RemoveFromList(); if (arg == NULL) { @@ -3130,13 +3191,13 @@ void NodeImpl::AppendEntriesCache::ack_fail(AppendEntriesRpc* rpc) { delete rpc; } -void NodeImpl::AppendEntriesCache::start_to_handle(HandleAppendEntriesFromCacheArg* arg) { +void NodeImpl::AppendEntriesCache::start_to_handle( + HandleAppendEntriesFromCacheArg* arg) { _node->AddRef(); bthread_t tid; // Sequence if not important if (bthread_start_background( - &tid, NULL, NodeImpl::handle_append_entries_from_cache, - arg) != 0) { + &tid, NULL, NodeImpl::handle_append_entries_from_cache, arg) != 0) { PLOG(ERROR) << "Fail to start bthread"; // We cant't call NodeImpl::handle_append_entries_from_cache // here since we are in the mutex, which will cause dead lock, just @@ -3160,12 +3221,12 @@ bool NodeImpl::AppendEntriesCache::start_timer() { timer_arg->cache_version = _cache_version; timer_arg->timer_start_ms = _rpc_queue.head()->value()->receive_time_ms; timespec duetime = butil::milliseconds_from( - butil::milliseconds_to_timespec(timer_arg->timer_start_ms), - std::max(_node->_options.election_timeout_ms >> 2, 1)); + butil::milliseconds_to_timespec(timer_arg->timer_start_ms), + std::max(_node->_options.election_timeout_ms >> 2, 1)); _node->AddRef(); - if (bthread_timer_add( - &_timer, duetime, NodeImpl::on_append_entries_cache_timedout, - timer_arg) != 0) { + if (bthread_timer_add(&_timer, duetime, + NodeImpl::on_append_entries_cache_timedout, + timer_arg) != 0) { LOG(ERROR) << "Fail to add timer"; delete timer_arg; _node->Release(); @@ -3186,14 +3247,14 @@ void NodeImpl::AppendEntriesCache::stop_timer() { } void NodeImpl::AppendEntriesCache::do_handle_append_entries_cache_timedout( - int64_t timer_version, int64_t timer_start_ms) { + int64_t timer_version, int64_t timer_start_ms) { if (timer_version != _timer_version) { return; } CHECK(!_rpc_map.empty()); CHECK(!_rpc_queue.empty()); - // If the head of out-of-order requests is not be handled, clear the entire cache, - // otherwise, start a new timer. + // If the head of out-of-order requests is not be handled, clear the entire + // cache, otherwise, start a new timer. if (_rpc_queue.head()->value()->receive_time_ms <= timer_start_ms) { clear(); return; @@ -3229,19 +3290,20 @@ void NodeImpl::ConfigurationCtx::start(const Configuration& old_conf, ss << ", begin caughtup."; LOG(INFO) << ss.str(); adding.list_peers(&_adding_peers); - for (std::set::const_iterator iter - = _adding_peers.begin(); iter != _adding_peers.end(); ++iter) { + for (std::set::const_iterator iter = _adding_peers.begin(); + iter != _adding_peers.end(); ++iter) { if (_node->_replicator_group.add_replicator(*iter) != 0) { LOG(ERROR) << "node " << _node->node_id() << " start replicator failed, peer " << *iter; return on_caughtup(_version, *iter, false); } - OnCaughtUp* caught_up = new OnCaughtUp( - _node, _node->_current_term, *iter, _version); + OnCaughtUp* caught_up = + new OnCaughtUp(_node, _node->_current_term, *iter, _version); timespec due_time = butil::milliseconds_from_now( - _node->_options.get_catchup_timeout_ms()); + _node->_options.get_catchup_timeout_ms()); if (_node->_replicator_group.wait_caughtup( - *iter, _node->_options.catchup_margin, &due_time, caught_up) != 0) { + *iter, _node->_options.catchup_margin, &due_time, caught_up) != + 0) { LOG(WARNING) << "node " << _node->node_id() << " wait_caughtup failed, peer " << *iter; delete caught_up; @@ -3263,15 +3325,14 @@ void NodeImpl::ConfigurationCtx::flush(const Configuration& conf, } _node->unsafe_apply_configuration(conf, old_conf.empty() ? NULL : &old_conf, true); - } -void NodeImpl::ConfigurationCtx::on_caughtup( - int64_t version, const PeerId& peer_id, bool succ) { +void NodeImpl::ConfigurationCtx::on_caughtup(int64_t version, + const PeerId& peer_id, bool succ) { if (version != _version) { LOG(WARNING) << "Node " << _node->node_id() - << " on_caughtup with unmatched version=" << version - << ", expect version=" << _version; + << " on_caughtup with unmatched version=" << version + << ", expect version=" << _version; return; } CHECK_EQ(STAGE_CATCHING_UP, _stage); @@ -3283,10 +3344,9 @@ void NodeImpl::ConfigurationCtx::on_caughtup( return; } // Fail - LOG(WARNING) << "Node " << _node->node_id() - << " fail to catch up peer " << peer_id - << " when trying to change peers from " - << Configuration(_old_peers) << " to " + LOG(WARNING) << "Node " << _node->node_id() << " fail to catch up peer " + << peer_id << " when trying to change peers from " + << Configuration(_old_peers) << " to " << Configuration(_new_peers); butil::Status err(ECATCHUP, "Peer %s failed to catch up", peer_id.to_string().c_str()); @@ -3296,55 +3356,60 @@ void NodeImpl::ConfigurationCtx::on_caughtup( void NodeImpl::ConfigurationCtx::next_stage() { CHECK(is_busy()); switch (_stage) { - case STAGE_CATCHING_UP: - if (_nchanges > 1) { - _stage = STAGE_JOINT; - Configuration old_conf(_old_peers); - return _node->unsafe_apply_configuration( + case STAGE_CATCHING_UP: + if (_nchanges > 1) { + _stage = STAGE_JOINT; + Configuration old_conf(_old_peers); + return _node->unsafe_apply_configuration( Configuration(_new_peers), &old_conf, false); - } - // Skip joint consensus since only one peer has been changed here. Make - // it a one-stage change to be compitible with the legacy - // implementation. - case STAGE_JOINT: - _stage = STAGE_STABLE; - TEST_SYNC_POINT_CALLBACK("NodeImpl::ConfigurationCtx:StableStage:BeforeApplyConfiguration", _node); - return _node->unsafe_apply_configuration( - Configuration(_new_peers), NULL, false); - case STAGE_STABLE: - { - bool should_step_down = + } + // Skip joint consensus since only one peer has been changed here. + // Make it a one-stage change to be compitible with the legacy + // implementation. + case STAGE_JOINT: + _stage = STAGE_STABLE; + TEST_SYNC_POINT_CALLBACK( + "NodeImpl::ConfigurationCtx:StableStage:" + "BeforeApplyConfiguration", + _node); + return _node->unsafe_apply_configuration(Configuration(_new_peers), + NULL, false); + case STAGE_STABLE: { + bool should_step_down = _new_peers.find(_node->_server_id) == _new_peers.end(); butil::Status st = butil::Status::OK(); reset(&st); if (should_step_down) { - _node->step_down(_node->_current_term, true, - butil::Status(ELEADERREMOVED, "This node was removed")); + _node->step_down( + _node->_current_term, true, + butil::Status(ELEADERREMOVED, "This node was removed")); } return; } - case STAGE_NONE: - CHECK(false) << "Can't reach here"; - return; + case STAGE_NONE: + CHECK(false) << "Can't reach here"; + return; } } void NodeImpl::ConfigurationCtx::reset(butil::Status* st) { // reset() should be called only once if (_stage == STAGE_NONE) { - BRAFT_VLOG << "node " << _node->node_id() - << " reset ConfigurationCtx when stage is STAGE_NONE already"; + BRAFT_VLOG + << "node " << _node->node_id() + << " reset ConfigurationCtx when stage is STAGE_NONE already"; return; } LOG(INFO) << "node " << _node->node_id() - << " reset ConfigurationCtx, new_peers: " << Configuration(_new_peers) + << " reset ConfigurationCtx, new_peers: " + << Configuration(_new_peers) << ", old_peers: " << Configuration(_old_peers); if (st && st->ok()) { _node->stop_replicator(_new_peers, _old_peers); } else { - // leader step_down may stop replicators of catching up nodes, leading to - // run catchup_closure + // leader step_down may stop replicators of catching up nodes, leading + // to run catchup_closure _node->stop_replicator(_old_peers, _new_peers); } _new_peers.clear(); @@ -3368,7 +3433,7 @@ void NodeImpl::ConfigurationCtx::reset(butil::Status* st) { void NodeImpl::enter_readonly_mode() { BAIDU_SCOPED_LOCK(_mutex); if (!_node_readonly) { - LOG(INFO) << "node " << _group_id << ":" << _server_id + LOG(INFO) << "node " << _group_id << ":" << _server_id << " enter readonly mode"; _node_readonly = true; } @@ -3377,7 +3442,7 @@ void NodeImpl::enter_readonly_mode() { void NodeImpl::leave_readonly_mode() { BAIDU_SCOPED_LOCK(_mutex); if (_node_readonly) { - LOG(INFO) << "node " << _group_id << ":" << _server_id + LOG(INFO) << "node " << _group_id << ":" << _server_id << " leave readonly mode"; _node_readonly = false; } @@ -3388,7 +3453,8 @@ bool NodeImpl::readonly() { return _node_readonly || _majority_nodes_readonly; } -int NodeImpl::change_readonly_config(int64_t term, const PeerId& peer_id, bool readonly) { +int NodeImpl::change_readonly_config(int64_t term, const PeerId& peer_id, + bool readonly) { BAIDU_SCOPED_LOCK(_mutex); if (term != _current_term && _state != STATE_LEADER) { return EINVAL; @@ -3411,7 +3477,7 @@ void NodeImpl::check_majority_nodes_readonly(const Configuration& conf) { size_t readonly_nodes = 0; for (size_t i = 0; i < peers.size(); i++) { if (peers[i] == _server_id) { - readonly_nodes += ((_node_readonly) ? 1: 0); + readonly_nodes += ((_node_readonly) ? 1 : 0); continue; } if (_replicator_group.readonly(peers[i])) { @@ -3422,9 +3488,10 @@ void NodeImpl::check_majority_nodes_readonly(const Configuration& conf) { bool prev_readonly = _majority_nodes_readonly; _majority_nodes_readonly = !(writable_nodes >= (peers.size() / 2 + 1)); if (prev_readonly != _majority_nodes_readonly) { - LOG(INFO) << "node " << _group_id << ":" << _server_id - << " majority readonly change from " << (prev_readonly ? "enable" : "disable") - << " to " << (_majority_nodes_readonly ? " enable" : "disable"); + LOG(INFO) << "node " << _group_id << ":" << _server_id + << " majority readonly change from " + << (prev_readonly ? "enable" : "disable") << " to " + << (_majority_nodes_readonly ? " enable" : "disable"); } } @@ -3466,7 +3533,8 @@ void NodeImpl::get_leader_lease_status(LeaderLeaseStatus* lease_status) { int64_t last_active_timestamp = last_leader_active_timestamp(); _leader_lease.renew(last_active_timestamp); _leader_lease.get_lease_info(&internal_info); - if (internal_info.state != LeaderLease::VALID && internal_info.state != LeaderLease::DISABLED) { + if (internal_info.state != LeaderLease::VALID && + internal_info.state != LeaderLease::DISABLED) { butil::Status status; status.set_error(ERAFTTIMEDOUT, "Leader lease expired"); step_down(_current_term, false, status); @@ -3482,11 +3550,12 @@ void NodeImpl::get_leader_lease_status(LeaderLeaseStatus* lease_status) { void NodeImpl::VoteBallotCtx::init(NodeImpl* node, bool triggered) { reset(node); - _ballot.init(node->_conf.conf, std::optional() ); + _ballot.init(node->_conf.conf, std::optional()); _triggered = triggered; } -void NodeImpl::VoteBallotCtx::start_grant_self_timer(int64_t wait_ms, NodeImpl* node) { +void NodeImpl::VoteBallotCtx::start_grant_self_timer(int64_t wait_ms, + NodeImpl* node) { timespec duetime = butil::milliseconds_from_now(wait_ms); GrantSelfArg* timer_arg = new GrantSelfArg; timer_arg->node = node; @@ -3494,9 +3563,8 @@ void NodeImpl::VoteBallotCtx::start_grant_self_timer(int64_t wait_ms, NodeImpl* timer_arg->vote_ctx = this; node->AddRef(); _grant_self_arg = timer_arg; - if (bthread_timer_add( - &_timer, duetime, NodeImpl::on_grant_self_timedout, - timer_arg) != 0) { + if (bthread_timer_add(&_timer, duetime, NodeImpl::on_grant_self_timedout, + timer_arg) != 0) { LOG(ERROR) << "Fail to add timer"; delete timer_arg; _grant_self_arg = NULL; @@ -3529,11 +3597,13 @@ void NodeImpl::VoteBallotCtx::reserve(const PeerId& peer) { _reserved_peers.insert(peer); } -void NodeImpl::VoteBallotCtx::set_disrupted_leader(const DisruptedLeader& peer) { +void NodeImpl::VoteBallotCtx::set_disrupted_leader( + const DisruptedLeader& peer) { _disrupted_leader = peer; } -const NodeImpl::DisruptedLeader& NodeImpl::VoteBallotCtx::disrupted_leader() const { +const NodeImpl::DisruptedLeader& NodeImpl::VoteBallotCtx::disrupted_leader() + const { return _disrupted_leader; } @@ -3553,8 +3623,10 @@ const LogId& NodeImpl::VoteBallotCtx::last_log_id() const { return _last_log_id; } -void NodeImpl::grant_self(VoteBallotCtx* vote_ctx, std::unique_lock* lck) { - // If follower lease expired, we can safely grant self. Otherwise, we wait util: +void NodeImpl::grant_self(VoteBallotCtx* vote_ctx, + std::unique_lock* lck) { + // If follower lease expired, we can safely grant self. Otherwise, we wait + // util: // 1. last active leader vote the node, and we grant two votes together; // 2. follower lease expire. int64_t wait_ms = _follower_lease.votable_time_from_now(); @@ -3576,18 +3648,17 @@ void NodeImpl::grant_self(VoteBallotCtx* vote_ctx, std::unique_locknode; + GrantSelfArg* grant_arg = (GrantSelfArg*)arg; + NodeImpl* node = grant_arg->node; VoteBallotCtx* vote_ctx = grant_arg->vote_ctx; - int64_t vote_ctx_version = grant_arg->vote_ctx_version; + int64_t vote_ctx_version = grant_arg->vote_ctx_version; delete grant_arg; @@ -3607,23 +3678,22 @@ void* NodeImpl::handle_grant_self_timedout(void* arg) { void NodeImpl::leader_lease_start(int64_t lease_epoch) { BAIDU_SCOPED_LOCK(_mutex); if (_state == STATE_LEADER) { - _leader_lease.on_lease_start( - lease_epoch, last_leader_active_timestamp()); + _leader_lease.on_lease_start(lease_epoch, + last_leader_active_timestamp()); } } int64_t NodeImpl::last_leader_active_timestamp() { int64_t timestamp = last_leader_active_timestamp(_conf.conf); if (!_conf.old_conf.empty()) { - timestamp = std::min(timestamp, last_leader_active_timestamp(_conf.old_conf)); + timestamp = + std::min(timestamp, last_leader_active_timestamp(_conf.old_conf)); } return timestamp; } struct LastActiveTimestampCompare { - bool operator()(const int64_t& a, const int64_t& b) { - return a > b; - } + bool operator()(const int64_t& a, const int64_t& b) { return a > b; } }; int64_t NodeImpl::last_leader_active_timestamp(const Configuration& conf) { @@ -3638,9 +3708,11 @@ int64_t NodeImpl::last_leader_active_timestamp(const Configuration& conf) { int64_t timestamp = _replicator_group.last_rpc_send_timestamp(peers[i]); last_rpc_send_timestamps.push_back(timestamp); - std::push_heap(last_rpc_send_timestamps.begin(), last_rpc_send_timestamps.end(), compare); + std::push_heap(last_rpc_send_timestamps.begin(), + last_rpc_send_timestamps.end(), compare); if (last_rpc_send_timestamps.size() > peers.size() / 2) { - std::pop_heap(last_rpc_send_timestamps.begin(), last_rpc_send_timestamps.end(), compare); + std::pop_heap(last_rpc_send_timestamps.begin(), + last_rpc_send_timestamps.end(), compare); last_rpc_send_timestamps.pop_back(); } } @@ -3648,7 +3720,8 @@ int64_t NodeImpl::last_leader_active_timestamp(const Configuration& conf) { if (last_rpc_send_timestamps.empty()) { return butil::monotonic_time_ms(); } - std::pop_heap(last_rpc_send_timestamps.begin(), last_rpc_send_timestamps.end(), compare); + std::pop_heap(last_rpc_send_timestamps.begin(), + last_rpc_send_timestamps.end(), compare); return last_rpc_send_timestamps.back(); } @@ -3667,28 +3740,20 @@ void NodeTimer::on_destroy() { } } -void ElectionTimer::run() { - _node->handle_election_timeout(); -} +void ElectionTimer::run() { _node->handle_election_timeout(); } int ElectionTimer::adjust_timeout_ms(int timeout_ms) { return random_timeout(timeout_ms); } -void VoteTimer::run() { - _node->handle_vote_timeout(); -} +void VoteTimer::run() { _node->handle_vote_timeout(); } int VoteTimer::adjust_timeout_ms(int timeout_ms) { return random_timeout(timeout_ms); } -void StepdownTimer::run() { - _node->handle_stepdown_timeout(); -} +void StepdownTimer::run() { _node->handle_stepdown_timeout(); } -void SnapshotTimer::run() { - _node->handle_snapshot_timeout(); -} +void SnapshotTimer::run() { _node->handle_snapshot_timeout(); } } // namespace braft diff --git a/src/braft/node.h b/src/braft/node.h index b9dd3e8..2314f9c 100644 --- a/src/braft/node.h +++ b/src/braft/node.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,23 +19,25 @@ #ifndef BRAFT_RAFT_NODE_H #define BRAFT_RAFT_NODE_H -#include +#include +#include #include -#include #include -#include -#include -#include "braft/raft.h" -#include "braft/log_manager.h" +#include + +#include + #include "braft/ballot_box.h" -#include "braft/storage.h" -#include "braft/raft_service.h" -#include "braft/fsm_caller.h" -#include "braft/replicator.h" -#include "braft/util.h" #include "braft/closure_queue.h" #include "braft/configuration_manager.h" +#include "braft/fsm_caller.h" +#include "braft/log_manager.h" +#include "braft/raft.h" +#include "braft/raft_service.h" #include "braft/repeated_timer_task.h" +#include "braft/replicator.h" +#include "braft/storage.h" +#include "braft/util.h" namespace braft { @@ -47,57 +49,59 @@ class StopTransferArg; class NodeImpl; class NodeTimer : public RepeatedTimerTask { -public: + public: NodeTimer() : _node(NULL) {} virtual ~NodeTimer() {} int init(NodeImpl* node, int timeout_ms); virtual void run() = 0; -protected: + + protected: void on_destroy(); NodeImpl* _node; }; class ElectionTimer : public NodeTimer { -protected: + protected: void run(); int adjust_timeout_ms(int timeout_ms); }; class VoteTimer : public NodeTimer { -protected: + protected: void run(); int adjust_timeout_ms(int timeout_ms); }; class StepdownTimer : public NodeTimer { -protected: + protected: void run(); }; class SnapshotTimer : public NodeTimer { -public: + public: SnapshotTimer() : _first_schedule(true) {} -protected: + + protected: void run(); int adjust_timeout_ms(int timeout_ms); -private: + + private: bool _first_schedule; }; -class BAIDU_CACHELINE_ALIGNMENT NodeImpl - : public butil::RefCountedThreadSafe { -friend class RaftServiceImpl; -friend class RaftStatImpl; -friend class FollowerStableClosure; -friend class ConfigurationChangeDone; -friend class VoteBallotCtx; -public: +class BAIDU_CACHELINE_ALIGNMENT NodeImpl + : public butil::RefCountedThreadSafe { + friend class RaftServiceImpl; + friend class RaftStatImpl; + friend class FollowerStableClosure; + friend class ConfigurationChangeDone; + friend class VoteBallotCtx; + + public: NodeImpl(const GroupId& group_id, const PeerId& peer_id); NodeImpl(); - NodeId node_id() const { - return NodeId(_group_id, _server_id); - } + NodeId node_id() const { return NodeId(_group_id, _server_id); } PeerId leader_id() { BAIDU_SCOPED_LOCK(_mutex); @@ -115,7 +119,8 @@ friend class VoteBallotCtx; int init(const NodeOptions& options); // shutdown local replica - // done is user defined function, maybe response to client or clean some resource + // done is user defined function, maybe response to client or clean some + // resource void shutdown(Closure* done); // Block the thread until the node is successfully stopped. @@ -124,7 +129,7 @@ friend class VoteBallotCtx; // apply task to the replicated-state-machine // // About the ownership: - // |task.data|: for the performance consideration, we will take way the + // |task.data|: for the performance consideration, we will take way the // content. If you want keep the content, copy it before call // this function // |task.done|: If the data is successfully committed to the raft group. We @@ -149,7 +154,8 @@ friend class VoteBallotCtx; // reset the election_timeout for the very node butil::Status reset_election_timeout_ms(int election_timeout_ms); - void reset_election_timeout_ms(int election_timeout_ms, int max_clock_drift_ms); + void reset_election_timeout_ms(int election_timeout_ms, + int max_clock_drift_ms); // rpc request proc func // @@ -158,20 +164,20 @@ friend class VoteBallotCtx; RequestVoteResponse* response); // handle received RequestVote int handle_request_vote_request(const RequestVoteRequest* request, - RequestVoteResponse* response); + RequestVoteResponse* response); // handle received AppendEntries void handle_append_entries_request(brpc::Controller* cntl, - const AppendEntriesRequest* request, - AppendEntriesResponse* response, - google::protobuf::Closure* done, - bool from_append_entries_cache = false); + const AppendEntriesRequest* request, + AppendEntriesResponse* response, + google::protobuf::Closure* done, + bool from_append_entries_cache = false); // handle received InstallSnapshot void handle_install_snapshot_request(brpc::Controller* controller, - const InstallSnapshotRequest* request, - InstallSnapshotResponse* response, - google::protobuf::Closure* done); + const InstallSnapshotRequest* request, + InstallSnapshotResponse* response, + google::protobuf::Closure* done); void handle_timeout_now_request(brpc::Controller* controller, const TimeoutNowRequest* request, @@ -192,8 +198,8 @@ friend class VoteBallotCtx; void handle_request_vote_response(const PeerId& peer_id, const int64_t term, const int64_t ctx_version, const RequestVoteResponse& response); - void on_caughtup(const PeerId& peer, int64_t term, - int64_t version, const butil::Status& st); + void on_caughtup(const PeerId& peer, int64_t term, int64_t version, + const butil::Status& st); // other func // // called when leader change configuration done, ref with FSMCaller @@ -202,23 +208,25 @@ friend class VoteBallotCtx; // Called when leader lease is safe to start. void leader_lease_start(int64_t lease_epoch); - // called when leader recv greater term in AppendEntriesResponse, ref with Replicator + // called when leader recv greater term in AppendEntriesResponse, ref with + // Replicator int increase_term_to(int64_t new_term, const butil::Status& status); // Temporary solution void update_configuration_after_installing_snapshot(); void describe(std::ostream& os, bool use_html); - - // Get the internal status of this node, the information is mostly the same as we - // see from the website, which is generated by |describe| actually. + + // Get the internal status of this node, the information is mostly the same + // as we see from the website, which is generated by |describe| actually. void get_status(NodeStatus* status); // Readonly mode func void enter_readonly_mode(); void leave_readonly_mode(); bool readonly(); - int change_readonly_config(int64_t term, const PeerId& peer_id, bool readonly); + int change_readonly_config(int64_t term, const PeerId& peer_id, + bool readonly); void check_majority_nodes_readonly(); void check_majority_nodes_readonly(const Configuration& conf); @@ -234,15 +242,17 @@ friend class VoteBallotCtx; void on_error(const Error& e); int transfer_leadership_to(const PeerId& peer); - - butil::Status read_committed_user_log(const int64_t index, UserLog* user_log); + + butil::Status read_committed_user_log(const int64_t index, + UserLog* user_log); int bootstrap(const BootstrapOptions& options); bool disable_cli() const { return _options.disable_cli; } bool is_witness() const { return _options.witness; } -private: -friend class butil::RefCountedThreadSafe; + + private: + friend class butil::RefCountedThreadSafe; virtual ~NodeImpl(); // internal init func @@ -263,11 +273,13 @@ friend class butil::RefCountedThreadSafe; void step_down(const int64_t term, bool wakeup_a_candidate, const butil::Status& status); - // reset leader_id. - // When new_leader_id is NULL, it means this node just stop following a leader; - // otherwise, it means setting this node's leader_id to new_leader_id. - // status gives the situation under which this method is called. - void reset_leader_id(const PeerId& new_leader_id, const butil::Status& status); + // reset leader_id. + // When new_leader_id is NULL, it means this node just stop following a + // leader; otherwise, it means setting this node's leader_id to + // new_leader_id. status gives the situation under which this method is + // called. + void reset_leader_id(const PeerId& new_leader_id, + const butil::Status& status); // check weather to step_down when receiving append_entries/install_snapshot // requests. @@ -277,14 +289,15 @@ friend class butil::RefCountedThreadSafe; void pre_vote(std::unique_lock* lck, bool triggered); // elect self to candidate - // If old leader has already stepped down, the candidate can vote without + // If old leader has already stepped down, the candidate can vote without // taking account of leader lease - void elect_self(std::unique_lock* lck, + void elect_self(std::unique_lock* lck, bool old_leader_stepped_down = false); // grant self a vote class VoteBallotCtx; - void grant_self(VoteBallotCtx* vote_ctx, std::unique_lock* lck); + void grant_self(VoteBallotCtx* vote_ctx, + std::unique_lock* lck); static void on_grant_self_timedout(void* arg); static void* handle_grant_self_timedout(void* arg); @@ -302,7 +315,7 @@ friend class butil::RefCountedThreadSafe; struct LogEntryAndClosure; static int execute_applying_tasks( - void* meta, bthread::TaskIterator& iter); + void* meta, bthread::TaskIterator& iter); void apply(LogEntryAndClosure tasks[], size_t size); void check_dead_nodes(const Configuration& conf, int64_t now_ms); void check_witness(const Configuration& conf); @@ -326,11 +339,11 @@ friend class butil::RefCountedThreadSafe; void request_peers_to_vote(const std::set& peers, const DisruptedLeader& disrupted_leader); -private: - + private: class ConfigurationCtx { - DISALLOW_COPY_AND_ASSIGN(ConfigurationCtx); - public: + DISALLOW_COPY_AND_ASSIGN(ConfigurationCtx); + + public: enum Stage { // Don't change the order if you are not sure about the usage STAGE_NONE = 0, @@ -338,8 +351,8 @@ friend class butil::RefCountedThreadSafe; STAGE_JOINT = 2, STAGE_STABLE = 3, }; - ConfigurationCtx(NodeImpl* node) : - _node(node), _stage(STAGE_NONE), _version(0), _done(NULL) {} + ConfigurationCtx(NodeImpl* node) + : _node(node), _stage(STAGE_NONE), _version(0), _done(NULL) {} void list_new_peers(std::vector* new_peers) const { new_peers->clear(); std::set::iterator it; @@ -355,8 +368,12 @@ friend class butil::RefCountedThreadSafe; } } const char* stage_str() { - const char* str[] = {"STAGE_NONE", "STAGE_CATCHING_UP", - "STAGE_JOINT", "STAGE_STABLE", }; + const char* str[] = { + "STAGE_NONE", + "STAGE_CATCHING_UP", + "STAGE_JOINT", + "STAGE_STABLE", + }; if (_stage <= STAGE_STABLE) { return str[(int)_stage]; } else { @@ -367,16 +384,15 @@ friend class butil::RefCountedThreadSafe; void reset(butil::Status* st = NULL); bool is_busy() const { return _stage != STAGE_NONE; } // Start change configuration. - void start(const Configuration& old_conf, - const Configuration& new_conf, - Closure * done); + void start(const Configuration& old_conf, const Configuration& new_conf, + Closure* done); // Invoked when this node becomes the leader, write a configuration // change log as the first log - void flush(const Configuration& conf, - const Configuration& old_conf); + void flush(const Configuration& conf, const Configuration& old_conf); void next_stage(); void on_caughtup(int64_t version, const PeerId& peer_id, bool succ); - private: + + private: NodeImpl* _node; Stage _stage; int _nchanges; @@ -408,10 +424,12 @@ friend class butil::RefCountedThreadSafe; // A simple cache to temporaryly store out-of-order AppendEntries requests. class AppendEntriesCache { - public: + public: AppendEntriesCache(NodeImpl* node, int64_t version) - : _node(node), _timer(bthread_timer_t()) - , _cache_version(0), _timer_version(0) {} + : _node(node), + _timer(bthread_timer_t()), + _cache_version(0), + _timer_version(0) {} int64_t first_index() const; int64_t cache_version() const; @@ -419,10 +437,10 @@ friend class butil::RefCountedThreadSafe; bool store(AppendEntriesRpc* rpc); void process_runable_rpcs(int64_t local_last_index); void clear(); - void do_handle_append_entries_cache_timedout( - int64_t timer_version, int64_t timer_start_ms); + void do_handle_append_entries_cache_timedout(int64_t timer_version, + int64_t timer_start_ms); - private: + private: void ack_fail(AppendEntriesRpc* rpc); void start_to_handle(HandleAppendEntriesFromCacheArg* arg); bool start_timer(); @@ -446,20 +464,16 @@ friend class butil::RefCountedThreadSafe; // A versioned ballot for vote and prevote struct GrantSelfArg; class VoteBallotCtx { - public: - VoteBallotCtx() : _timer(bthread_timer_t()), _version(0) - , _grant_self_arg(NULL), _triggered(false) { - } + public: + VoteBallotCtx() + : _timer(bthread_timer_t()), + _version(0), + _grant_self_arg(NULL), + _triggered(false) {} void init(NodeImpl* node, bool triggered); - void grant(const PeerId& peer) { - _ballot.grant(peer); - } - bool granted() { - return _ballot.granted(); - } - int64_t version() { - return _version; - } + void grant(const PeerId& peer) { _ballot.grant(peer); } + bool granted() { return _ballot.granted(); } + int64_t version() { return _version; } void start_grant_self_timer(int64_t wait_ms, NodeImpl* node); void stop_grant_self_timer(NodeImpl* node); void reset(NodeImpl* node); @@ -470,7 +484,8 @@ friend class butil::RefCountedThreadSafe; void pop_grantable_peers(std::set* peers); void set_last_log_id(const LogId& log_id); const LogId& last_log_id() const; - private: + + private: bthread_timer_t _timer; Ballot _ballot; // Each time the vote ctx restarted, increase the version to avoid @@ -493,8 +508,8 @@ friend class butil::RefCountedThreadSafe; int64_t _current_term; PeerId _leader_id; PeerId _voted_id; - VoteBallotCtx _vote_ctx; // candidate vote ctx - VoteBallotCtx _pre_vote_ctx; // prevote ctx + VoteBallotCtx _vote_ctx; // candidate vote ctx + VoteBallotCtx _pre_vote_ctx; // prevote ctx ConfigurationEntry _conf; GroupId _group_id; @@ -535,6 +550,6 @@ friend class butil::RefCountedThreadSafe; FollowerLease _follower_lease; }; -} +} // namespace braft -#endif //~BRAFT_RAFT_NODE_H +#endif //~BRAFT_RAFT_NODE_H diff --git a/src/braft/node_manager.cpp b/src/braft/node_manager.cpp index 84bf940..9e193a3 100644 --- a/src/braft/node_manager.cpp +++ b/src/braft/node_manager.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,11 +14,12 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#include "braft/node.h" #include "braft/node_manager.h" -#include "braft/file_service.h" + #include "braft/builtin_service_impl.h" #include "braft/cli_service.h" +#include "braft/file_service.h" +#include "braft/node.h" namespace braft { @@ -42,7 +43,7 @@ void NodeManager::remove_address(butil::EndPoint addr) { _addr_set.erase(addr); } -int NodeManager::add_service(brpc::Server* server, +int NodeManager::add_service(brpc::Server* server, const butil::EndPoint& listen_address) { if (server == NULL) { LOG(ERROR) << "server is NULL"; @@ -52,14 +53,14 @@ int NodeManager::add_service(brpc::Server* server, return 0; } - if (0 != server->AddService(file_service(), brpc::SERVER_DOESNT_OWN_SERVICE)) { + if (0 != + server->AddService(file_service(), brpc::SERVER_DOESNT_OWN_SERVICE)) { LOG(ERROR) << "Fail to add FileService"; return -1; } - if (0 != server->AddService( - new RaftServiceImpl(listen_address), - brpc::SERVER_OWNS_SERVICE)) { + if (0 != server->AddService(new RaftServiceImpl(listen_address), + brpc::SERVER_OWNS_SERVICE)) { LOG(ERROR) << "Fail to add RaftService"; return -1; } @@ -68,7 +69,8 @@ int NodeManager::add_service(brpc::Server* server, LOG(ERROR) << "Fail to add RaftStatService"; return -1; } - if (0 != server->AddService(new CliServiceImpl, brpc::SERVER_OWNS_SERVICE)) { + if (0 != + server->AddService(new CliServiceImpl, brpc::SERVER_OWNS_SERVICE)) { LOG(ERROR) << "Fail to add CliService"; return -1; } @@ -83,10 +85,10 @@ int NodeManager::add_service(brpc::Server* server, size_t NodeManager::_add_node(Maps& m, const NodeImpl* node) { NodeId node_id = node->node_id(); std::pair ret = m.node_map.insert( - NodeMap::value_type(node_id, const_cast(node))); + NodeMap::value_type(node_id, const_cast(node))); if (ret.second) { - m.group_map.insert(GroupMap::value_type( - node_id.group_id, const_cast(node))); + m.group_map.insert(GroupMap::value_type(node_id.group_id, + const_cast(node))); return 1; } return 0; @@ -95,13 +97,13 @@ size_t NodeManager::_add_node(Maps& m, const NodeImpl* node) { size_t NodeManager::_remove_node(Maps& m, const NodeImpl* node) { NodeMap::iterator iter = m.node_map.find(node->node_id()); if (iter == m.node_map.end() || iter->second.get() != node) { - // ^^ - // Avoid duplicated nodes + // ^^ + // Avoid duplicated nodes return 0; } m.node_map.erase(iter); - std::pair - range = m.group_map.equal_range(node->node_id().group_id); + std::pair range = + m.group_map.equal_range(node->node_id().group_id); for (GroupMap::iterator it = range.first; it != range.second; ++it) { if (it->second == node) { m.group_map.erase(it); @@ -125,7 +127,8 @@ bool NodeManager::remove(NodeImpl* node) { return _nodes.Modify(_remove_node, node) != 0; } -scoped_refptr NodeManager::get(const GroupId& group_id, const PeerId& peer_id) { +scoped_refptr NodeManager::get(const GroupId& group_id, + const PeerId& peer_id) { butil::DoublyBufferedData::ScopedPtr ptr; if (_nodes.Read(&ptr) != 0) { return NULL; @@ -138,15 +141,14 @@ scoped_refptr NodeManager::get(const GroupId& group_id, const PeerId& } void NodeManager::get_nodes_by_group_id( - const GroupId& group_id, std::vector >* nodes) { - + const GroupId& group_id, std::vector >* nodes) { nodes->clear(); butil::DoublyBufferedData::ScopedPtr ptr; if (_nodes.Read(&ptr) != 0) { return; } - std::pair - range = ptr->group_map.equal_range(group_id); + std::pair range = + ptr->group_map.equal_range(group_id); for (GroupMap::const_iterator it = range.first; it != range.second; ++it) { nodes->push_back(it->second); } @@ -159,11 +161,10 @@ void NodeManager::get_all_nodes(std::vector >* nodes) { return; } nodes->reserve(ptr->group_map.size()); - for (GroupMap::const_iterator - it = ptr->group_map.begin(); it != ptr->group_map.end(); ++it) { + for (GroupMap::const_iterator it = ptr->group_map.begin(); + it != ptr->group_map.end(); ++it) { nodes->push_back(it->second); } } } // namespace braft - diff --git a/src/braft/node_manager.h b/src/braft/node_manager.h index 5d44399..d8486f8 100644 --- a/src/braft/node_manager.h +++ b/src/braft/node_manager.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,11 +14,12 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_NODE_MANAGER_H -#define BRAFT_NODE_MANAGER_H +#ifndef BRAFT_NODE_MANAGER_H +#define BRAFT_NODE_MANAGER_H -#include #include +#include + #include "braft/raft.h" #include "braft/util.h" @@ -27,10 +28,8 @@ namespace braft { class NodeImpl; class NodeManager { -public: - static NodeManager* GetInstance() { - return Singleton::get(); - } + public: + static NodeManager* GetInstance() { return Singleton::get(); } // add raft node bool add(NodeImpl* node); @@ -42,14 +41,13 @@ class NodeManager { scoped_refptr get(const GroupId& group_id, const PeerId& peer_id); // get all the nodes of |group_id| - void get_nodes_by_group_id(const GroupId& group_id, + void get_nodes_by_group_id(const GroupId& group_id, std::vector >* nodes); void get_all_nodes(std::vector >* nodes); // Add service to |server| at |listen_addr| - int add_service(brpc::Server* server, - const butil::EndPoint& listen_addr); + int add_service(brpc::Server* server, const butil::EndPoint& listen_addr); // Return true if |addr| is reachable by a RPC Server bool server_exists(butil::EndPoint addr); @@ -57,17 +55,17 @@ class NodeManager { // Remove the addr from _addr_set when the backing service is destroyed void remove_address(butil::EndPoint addr); -private: + private: NodeManager(); ~NodeManager(); DISALLOW_COPY_AND_ASSIGN(NodeManager); friend struct DefaultSingletonTraits; - + // TODO(chenzhangyi01): replace std::map with FlatMap // To make implementation simplicity, we use two maps here, although // it works practically with only one GroupMap typedef std::map > NodeMap; - typedef std::multimap GroupMap; + typedef std::multimap GroupMap; struct Maps { NodeMap node_map; GroupMap group_map; @@ -84,6 +82,6 @@ class NodeManager { #define global_node_manager NodeManager::GetInstance() -} // namespace braft +} // namespace braft #endif // BRAFT_NODE_MANAGER_H diff --git a/src/braft/protobuf_file.cpp b/src/braft/protobuf_file.cpp index 72ed249..eee7b89 100644 --- a/src/braft/protobuf_file.cpp +++ b/src/braft/protobuf_file.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,21 +14,21 @@ // Authors: Wang,Yao(wangyao02@baidu.com) +#include "braft/protobuf_file.h" + #include #include -#include "braft/protobuf_file.h" - namespace braft { -ProtoBufFile::ProtoBufFile(const char* path, FileSystemAdaptor* fs) +ProtoBufFile::ProtoBufFile(const char* path, FileSystemAdaptor* fs) : _path(path), _fs(fs) { if (_fs == NULL) { _fs = default_file_system(); } } -ProtoBufFile::ProtoBufFile(const std::string& path, FileSystemAdaptor* fs) +ProtoBufFile::ProtoBufFile(const std::string& path, FileSystemAdaptor* fs) : _path(path), _fs(fs) { if (_fs == NULL) { _fs = default_file_system(); @@ -40,10 +40,11 @@ int ProtoBufFile::save(const google::protobuf::Message* message, bool sync) { tmp_path.append(".tmp"); butil::File::Error e; - FileAdaptor* file = _fs->open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, NULL, &e); + FileAdaptor* file = + _fs->open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, NULL, &e); if (!file) { - LOG(WARNING) << "open file failed, path: " << _path - << ": " << butil::File::ErrorToString(e); + LOG(WARNING) << "open file failed, path: " << _path << ": " + << butil::File::ErrorToString(e); return -1; } std::unique_ptr > guard(file); @@ -78,7 +79,8 @@ int ProtoBufFile::save(const google::protobuf::Message* message, bool sync) { // rename if (!_fs->rename(tmp_path, _path)) { - LOG(WARNING) << "rename failed, old: " << tmp_path << " , new: " << _path; + LOG(WARNING) << "rename failed, old: " << tmp_path + << " , new: " << _path; return -1; } return 0; @@ -88,8 +90,8 @@ int ProtoBufFile::load(google::protobuf::Message* message) { butil::File::Error e; FileAdaptor* file = _fs->open(_path, O_RDONLY, NULL, &e); if (!file) { - LOG(WARNING) << "open file failed, path: " << _path - << ": " << butil::File::ErrorToString(e); + LOG(WARNING) << "open file failed, path: " << _path << ": " + << butil::File::ErrorToString(e); return -1; } @@ -119,4 +121,4 @@ int ProtoBufFile::load(google::protobuf::Message* message) { return 0; } -} +} // namespace braft diff --git a/src/braft/protobuf_file.h b/src/braft/protobuf_file.h index 2f987a3..c9feacd 100644 --- a/src/braft/protobuf_file.h +++ b/src/braft/protobuf_file.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,8 +17,10 @@ #ifndef BRAFT_PROTOBUF_FILE_H #define BRAFT_PROTOBUF_FILE_H -#include #include + +#include + #include "braft/file_system_adaptor.h" namespace braft { @@ -27,7 +29,7 @@ namespace braft { // len [4B, in network order] // protobuf data class ProtoBufFile { -public: + public: ProtoBufFile(const char* path, FileSystemAdaptor* fs = NULL); ProtoBufFile(const std::string& path, FileSystemAdaptor* fs = NULL); ~ProtoBufFile() {} @@ -35,11 +37,11 @@ class ProtoBufFile { int save(const ::google::protobuf::Message* message, bool sync); int load(::google::protobuf::Message* message); -private: + private: std::string _path; scoped_refptr _fs; }; -} +} // namespace braft -#endif //~BRAFT_PROTOBUF_FILE_H +#endif //~BRAFT_PROTOBUF_FILE_H diff --git a/src/braft/raft.cpp b/src/braft/raft.cpp index 6069f70..dae8b10 100644 --- a/src/braft/raft.cpp +++ b/src/braft/raft.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,45 +15,46 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) // Wang,Yao(wangyao02@baidu.com) +#include "braft/raft.h" + +#include +#include #include #include -#include -#include -#include "braft/raft.h" -#include "braft/node.h" -#include "braft/storage.h" -#include "braft/node_manager.h" + +#include "braft/fsm_caller.h" // IteratorImpl #include "braft/log.h" #include "braft/memory_log.h" +#include "braft/node.h" +#include "braft/node_manager.h" #include "braft/raft_meta.h" #include "braft/snapshot.h" -#include "braft/fsm_caller.h" // IteratorImpl +#include "braft/storage.h" namespace braft { static void print_revision(std::ostream& os, void*) { #if defined(BRAFT_REVISION) - os << BRAFT_REVISION; + os << BRAFT_REVISION; #else - os << "undefined"; + os << "undefined"; #endif } -static bvar::PassiveStatus s_raft_revision( - "raft_revision", print_revision, NULL); - +static bvar::PassiveStatus s_raft_revision("raft_revision", + print_revision, NULL); static pthread_once_t global_init_once = PTHREAD_ONCE_INIT; struct GlobalExtension { SegmentLogStorage local_log; MemoryLogStorage memory_log; - + // manage only one raft instance FileBasedSingleMetaStorage single_meta; // manage a batch of raft instances KVBasedMergedMetaStorage merged_meta; - // mix two types for double write when upgrade and downgrade + // mix two types for double write when upgrade and downgrade MixedMetaStorage mixed_meta; LocalSnapshotStorage local_snapshot; @@ -64,7 +65,7 @@ static void global_init_or_die_impl() { log_storage_extension()->RegisterOrDie("local", &s_ext.local_log); log_storage_extension()->RegisterOrDie("memory", &s_ext.memory_log); - + // uri = local://{single_path} // |single_path| usually ends with `/meta' // NOTICE: not change "local" to "local-single" because of compatibility @@ -74,7 +75,7 @@ static void global_init_or_die_impl() { meta_storage_extension()->RegisterOrDie("local-merged", &s_ext.merged_meta); // uri = local-mixed://merged_path={merged_path}&&single_path={single_path} meta_storage_extension()->RegisterOrDie("local-mixed", &s_ext.mixed_meta); - + snapshot_storage_extension()->RegisterOrDie("local", &s_ext.local_snapshot); } @@ -105,7 +106,7 @@ int add_service(brpc::Server* server, const char* listen_ip_and_port) { return add_service(server, addr); } -// GC +// GC int gc_raft_data(const GCOptions& gc_options) { const VersionedGroupId vgid = gc_options.vgid; const std::string log_uri = gc_options.log_uri; @@ -116,20 +117,23 @@ int gc_raft_data(const GCOptions& gc_options) { butil::Status status = LogStorage::destroy(log_uri); if (!status.ok()) { is_success = false; - LOG(WARNING) << "Group " << vgid << " failed to gc raft log, uri " << log_uri; + LOG(WARNING) << "Group " << vgid << " failed to gc raft log, uri " + << log_uri; } // TODO encode vgid into raft_meta_uri ? status = RaftMetaStorage::destroy(raft_meta_uri, vgid); if (!status.ok()) { is_success = false; - LOG(WARNING) << "Group " << vgid << " failed to gc raft stable, uri " << raft_meta_uri; + LOG(WARNING) << "Group " << vgid << " failed to gc raft stable, uri " + << raft_meta_uri; } status = SnapshotStorage::destroy(snapshot_uri); if (!status.ok()) { is_success = false; - LOG(WARNING) << "Group " << vgid << " failed to gc raft snapshot, uri " << snapshot_uri; + LOG(WARNING) << "Group " << vgid << " failed to gc raft snapshot, uri " + << snapshot_uri; } - return is_success ? 0 : -1; + return is_success ? 0 : -1; } // ------------- Node @@ -146,41 +150,25 @@ Node::~Node() { } } -NodeId Node::node_id() { - return _impl->node_id(); -} +NodeId Node::node_id() { return _impl->node_id(); } -PeerId Node::leader_id() { - return _impl->leader_id(); -} +PeerId Node::leader_id() { return _impl->leader_id(); } -bool Node::is_leader() { - return _impl->is_leader(); -} +bool Node::is_leader() { return _impl->is_leader(); } -bool Node::is_leader_lease_valid() { - return _impl->is_leader_lease_valid(); -} +bool Node::is_leader_lease_valid() { return _impl->is_leader_lease_valid(); } void Node::get_leader_lease_status(LeaderLeaseStatus* status) { return _impl->get_leader_lease_status(status); } -int Node::init(const NodeOptions& options) { - return _impl->init(options); -} +int Node::init(const NodeOptions& options) { return _impl->init(options); } -void Node::shutdown(Closure* done) { - _impl->shutdown(done); -} +void Node::shutdown(Closure* done) { _impl->shutdown(done); } -void Node::join() { - _impl->join(); -} +void Node::join() { _impl->join(); } -void Node::apply(const Task& task) { - _impl->apply(task); -} +void Node::apply(const Task& task) { _impl->apply(task); } butil::Status Node::list_peers(std::vector* peers) { return _impl->list_peers(peers); @@ -202,9 +190,7 @@ butil::Status Node::reset_peers(const Configuration& new_peers) { return _impl->reset_peers(new_peers); } -void Node::snapshot(Closure* done) { - _impl->snapshot(done); -} +void Node::snapshot(Closure* done) { _impl->snapshot(done); } butil::Status Node::vote(int election_timeout) { return _impl->vote(election_timeout); @@ -214,7 +200,8 @@ butil::Status Node::reset_election_timeout_ms(int election_timeout_ms) { return _impl->reset_election_timeout_ms(election_timeout_ms); } -void Node::reset_election_timeout_ms(int election_timeout_ms, int max_clock_drift_ms) { +void Node::reset_election_timeout_ms(int election_timeout_ms, + int max_clock_drift_ms) { _impl->reset_election_timeout_ms(election_timeout_ms, max_clock_drift_ms); } @@ -222,25 +209,18 @@ int Node::transfer_leadership_to(const PeerId& peer) { return _impl->transfer_leadership_to(peer); } -butil::Status Node::read_committed_user_log(const int64_t index, UserLog* user_log) { +butil::Status Node::read_committed_user_log(const int64_t index, + UserLog* user_log) { return _impl->read_committed_user_log(index, user_log); } -void Node::get_status(NodeStatus* status) { - return _impl->get_status(status); -} +void Node::get_status(NodeStatus* status) { return _impl->get_status(status); } -void Node::enter_readonly_mode() { - return _impl->enter_readonly_mode(); -} +void Node::enter_readonly_mode() { return _impl->enter_readonly_mode(); } -void Node::leave_readonly_mode() { - return _impl->leave_readonly_mode(); -} +void Node::leave_readonly_mode() { return _impl->leave_readonly_mode(); } -bool Node::readonly() { - return _impl->readonly(); -} +bool Node::readonly() { return _impl->readonly(); } // ------------- Iterator void Iterator::next() { @@ -257,13 +237,9 @@ int64_t Iterator::index() const { return _impl->index(); } int64_t Iterator::term() const { return _impl->entry()->id.term; } -const butil::IOBuf& Iterator::data() const { - return _impl->entry()->data; -} +const butil::IOBuf& Iterator::data() const { return _impl->entry()->data; } -Closure* Iterator::done() const { - return _impl->done(); -} +Closure* Iterator::done() const { return _impl->done(); } void Iterator::set_error_and_rollback(size_t ntail, const butil::Status* st) { return _impl->set_error_and_rollback(ntail, st); @@ -279,7 +255,7 @@ void StateMachine::on_snapshot_save(SnapshotWriter* writer, Closure* done) { LOG(ERROR) << butil::class_name_str(*this) << " didn't implement on_snapshot_save"; done->status().set_error(-1, "%s didn't implement on_snapshot_save", - butil::class_name_str(*this).c_str()); + butil::class_name_str(*this).c_str()); done->Run(); } @@ -294,11 +270,12 @@ int StateMachine::on_snapshot_load(SnapshotReader* reader) { void StateMachine::on_leader_start(int64_t) {} void StateMachine::on_leader_stop(const butil::Status&) {} void StateMachine::on_error(const Error& e) { - LOG(ERROR) << "Encountered an error=" << e << " on StateMachine " - << butil::class_name_str(*this) - << ", it's highly recommended to implement this interface" - " as raft stops working since some error ocurrs," - " you should figure out the cause and repair or remove this node"; + LOG(ERROR) + << "Encountered an error=" << e << " on StateMachine " + << butil::class_name_str(*this) + << ", it's highly recommended to implement this interface" + " as raft stops working since some error ocurrs," + " you should figure out the cause and repair or remove this node"; } void StateMachine::on_configuration_committed(const Configuration& conf) { @@ -306,7 +283,8 @@ void StateMachine::on_configuration_committed(const Configuration& conf) { return; } -void StateMachine::on_configuration_committed(const Configuration& conf, int64_t index) { +void StateMachine::on_configuration_committed(const Configuration& conf, + int64_t index) { (void)index; return on_configuration_committed(conf); } @@ -315,11 +293,10 @@ void StateMachine::on_stop_following(const LeaderChangeContext&) {} void StateMachine::on_start_following(const LeaderChangeContext&) {} BootstrapOptions::BootstrapOptions() - : last_log_index(0) - , fsm(NULL) - , node_owns_fsm(false) - , usercode_in_pthread(false) -{} + : last_log_index(0), + fsm(NULL), + node_owns_fsm(false), + usercode_in_pthread(false) {} int bootstrap(const BootstrapOptions& options) { global_init_once_or_die(); @@ -331,4 +308,4 @@ int bootstrap(const BootstrapOptions& options) { return rc; } -} +} // namespace braft diff --git a/src/braft/raft.h b/src/braft/raft.h index cb80163..7dae932 100644 --- a/src/braft/raft.h +++ b/src/braft/raft.h @@ -3,9 +3,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,17 +20,19 @@ #ifndef BRAFT_RAFT_H #define BRAFT_RAFT_H -#include - -#include +#include #include +#include #include -#include + +#include + #include "braft/configuration.h" #include "braft/enum.pb.h" #include "braft/errno.pb.h" -template class scoped_refptr; +template +class scoped_refptr; namespace brpc { class Server; @@ -51,17 +53,17 @@ const PeerId ANY_PEER(butil::EndPoint(butil::IP_ANY, 0), 0); // Raft-specific closure which encloses a butil::Status to report if the // operation was successful. class Closure : public google::protobuf::Closure { -public: + public: butil::Status& status() { return _st; } const butil::Status& status() const { return _st; } - -private: + + private: butil::Status _st; }; // Describe a specific error class Error { -public: + public: Error() : _type(ERROR_TYPE_NONE) {} Error(const Error& e) : _type(e._type), _st(e._st) {} ErrorType type() const { return _type; } @@ -74,7 +76,8 @@ class Error { _st = rhs._st; return *this; } -private: + + private: // Intentionally copyable ErrorType _type; butil::Status _st; @@ -82,25 +85,24 @@ class Error { inline const char* errortype2str(ErrorType t) { switch (t) { - case ERROR_TYPE_NONE: - return "None"; - case ERROR_TYPE_LOG: - return "LogError"; - case ERROR_TYPE_STABLE: - return "StableError"; - case ERROR_TYPE_SNAPSHOT: - return "SnapshotError"; - case ERROR_TYPE_STATE_MACHINE: - return "StateMachineError"; + case ERROR_TYPE_NONE: + return "None"; + case ERROR_TYPE_LOG: + return "LogError"; + case ERROR_TYPE_STABLE: + return "StableError"; + case ERROR_TYPE_SNAPSHOT: + return "SnapshotError"; + case ERROR_TYPE_STATE_MACHINE: + return "StateMachineError"; } return "Unknown"; } inline std::ostream& operator<<(std::ostream& os, const Error& e) { - os << "{type=" << errortype2str(e.type()) - << ", error_code=" << e.status().error_code() - << ", error_text=`" << e.status().error_cstr() - << "'}"; + os << "{type=" << errortype2str(e.type()) + << ", error_code=" << e.status().error_code() << ", error_text=`" + << e.status().error_cstr() << "'}"; return os; } @@ -133,16 +135,17 @@ class IteratorImpl; // } class Iterator { DISALLOW_COPY_AND_ASSIGN(Iterator); -public: + + public: // Move to the next task. void next(); - // Return a unique and monotonically increasing identifier of the current + // Return a unique and monotonically increasing identifier of the current // task: - // - Uniqueness guarantees that committed tasks in different peers with + // - Uniqueness guarantees that committed tasks in different peers with // the same index are always the same and kept unchanged. - // - Monotonicity guarantees that for any index pair i, j (i < j), task - // at index |i| must be applied before task at index |j| in all the + // - Monotonicity guarantees that for any index pair i, j (i < j), task + // at index |i| must be applied before task at index |j| in all the // peers from the group. int64_t index() const; @@ -157,10 +160,10 @@ class Iterator { // task no matter this operation succeeds or fails, otherwise the // corresponding resources would leak. // - // If this task is proposed by this Node when it was the leader of this - // group and the leadership has not changed before this point, done() is - // exactly what was passed to Node::apply which may stand for some - // continuation (such as respond to the client) after updating the + // If this task is proposed by this Node when it was the leader of this + // group and the leadership has not changed before this point, done() is + // exactly what was passed to Node::apply which may stand for some + // continuation (such as respond to the client) after updating the // StateMachine with the given task. Otherweise done() must be NULL. Closure* done() const; @@ -169,19 +172,20 @@ class Iterator { // batch of tasks or some error has occurred bool valid() const; - // Invoked when some critical error occurred. And we will consider the last + // Invoked when some critical error occurred. And we will consider the last // |ntail| tasks (starting from the last iterated one) as not applied. After - // this point, no further changes on the StateMachine as well as the Node - // would be allowed and you should try to repair this replica or just drop + // this point, no further changes on the StateMachine as well as the Node + // would be allowed and you should try to repair this replica or just drop // it. // // If |st| is not NULL, it should describe the detail of the error. - void set_error_and_rollback(size_t ntail = 1, const butil::Status* st = NULL); + void set_error_and_rollback(size_t ntail = 1, + const butil::Status* st = NULL); -private: -friend class FSMCaller; + private: + friend class FSMCaller; Iterator(IteratorImpl* impl) : _impl(impl) {} - ~Iterator() {}; + ~Iterator(){}; // The ownership of _impl belongs to FSMCaller; IteratorImpl* _impl; @@ -190,18 +194,18 @@ friend class FSMCaller; // |StateMachine| is the sink of all the events of a very raft node. // Implement a specific StateMachine for your own business logic. // -// NOTE: All the interfaces are not guaranteed to be thread safe and they are -// called sequentially, saying that every single operation will block all the +// NOTE: All the interfaces are not guaranteed to be thread safe and they are +// called sequentially, saying that every single operation will block all the // following ones. class StateMachine { -public: + public: virtual ~StateMachine(); // Update the StateMachine with a batch a tasks that can be accessed // through |iterator|. // // Invoked when one or more tasks that were passed to Node::apply have been - // committed to the raft group (quorum of the group peers have received + // committed to the raft group (quorum of the group peers have received // those tasks and stored them on the backing storage). // // Once this function returns to the caller, we will regard all the iterated @@ -243,22 +247,23 @@ class StateMachine { // Invoked when a configuration has been committed to the group virtual void on_configuration_committed(const ::braft::Configuration& conf); - virtual void on_configuration_committed(const ::braft::Configuration& conf, int64_t index); - - // this method is called when a follower stops following a leader and its leader_id becomes NULL, - // situations including: - // 1. handle election_timeout and start pre_vote - // 2. receive requests with higher term such as vote_request from a candidate - // or append_entries_request from a new leader + virtual void on_configuration_committed(const ::braft::Configuration& conf, + int64_t index); + + // this method is called when a follower stops following a leader and its + // leader_id becomes NULL, situations including: + // 1. handle election_timeout and start pre_vote + // 2. receive requests with higher term such as vote_request from a + // candidate or append_entries_request from a new leader // 3. receive timeout_now_request from current leader and start request_vote // the parameter ctx gives the information(leader_id, term and status) about // the very leader whom the follower followed before. // User can reset the node's information as it stops following some leader. virtual void on_stop_following(const ::braft::LeaderChangeContext& ctx); - // this method is called when a follower or candidate starts following a leader and its leader_id - // (should be NULL before the method is called) is set to the leader's id, - // situations including: + // this method is called when a follower or candidate starts following a + // leader and its leader_id (should be NULL before the method is called) is + // set to the leader's id, situations including: // 1. a candidate receives append_entries from a leader // 2. a follower(without leader) receives append_entries from a leader // the parameter ctx gives the information(leader_id, term and status) about @@ -281,8 +286,10 @@ enum State { }; inline const char* state2str(State state) { - const char* str[] = {"LEADER", "TRANSFERRING", "CANDIDATE", "FOLLOWER", - "ERROR", "UNINITIALIZED", "SHUTTING", "SHUTDOWN", }; + const char* str[] = { + "LEADER", "TRANSFERRING", "CANDIDATE", "FOLLOWER", + "ERROR", "UNINITIALIZED", "SHUTTING", "SHUTDOWN", + }; if (state < STATE_END) { return str[(int)state - 1]; } else { @@ -296,44 +303,43 @@ inline bool is_active_state(State s) { return s < STATE_ERROR; } -// This class encapsulates the parameter of on_start_following and on_stop_following interfaces. +// This class encapsulates the parameter of on_start_following and +// on_stop_following interfaces. class LeaderChangeContext { DISALLOW_COPY_AND_ASSIGN(LeaderChangeContext); -public: - LeaderChangeContext(const PeerId& leader_id, int64_t term, const butil::Status& status) - : _leader_id(leader_id) - , _term(term) - , _st(status) - {}; + + public: + LeaderChangeContext(const PeerId& leader_id, int64_t term, + const butil::Status& status) + : _leader_id(leader_id), _term(term), _st(status){}; // for on_start_following, the leader_id and term are of the new leader; // for on_stop_following, the leader_id and term are of the old leader. const PeerId& leader_id() const { return _leader_id; } int64_t term() const { return _term; } - // return the information about why on_start_following or on_stop_following is called. + // return the information about why on_start_following or on_stop_following + // is called. const butil::Status& status() const { return _st; } - -private: + + private: PeerId _leader_id; int64_t _term; butil::Status _st; }; -inline std::ostream& operator<<(std::ostream& os, const LeaderChangeContext& ctx) { - os << "{ leader_id=" << ctx.leader_id() - << ", term=" << ctx.term() - << ", status=" << ctx.status() - << "}"; +inline std::ostream& operator<<(std::ostream& os, + const LeaderChangeContext& ctx) { + os << "{ leader_id=" << ctx.leader_id() << ", term=" << ctx.term() + << ", status=" << ctx.status() << "}"; return os; } class UserLog { DISALLOW_COPY_AND_ASSIGN(UserLog); -public: - UserLog() {}; + + public: + UserLog(){}; UserLog(int64_t log_index, const butil::IOBuf& log_data) - : _index(log_index) - , _data(log_data) - {}; + : _index(log_index), _data(log_data){}; int64_t log_index() const { return _index; } const butil::IOBuf& log_data() const { return _data; } void set_log_index(const int64_t log_index) { _index = log_index; } @@ -343,34 +349,37 @@ class UserLog { _data.clear(); } -private: + private: int64_t _index; butil::IOBuf _data; }; inline std::ostream& operator<<(std::ostream& os, const UserLog& user_log) { os << "{user_log: index=" << user_log.log_index() - << ", data size=" << user_log.log_data().size() - << "}"; + << ", data size=" << user_log.log_data().size() << "}"; return os; } // Status of a peer struct PeerStatus { PeerStatus() - : valid(false), installing_snapshot(false), blocking(false), next_index(0) - , last_rpc_send_timestamp(0), flying_append_entries_size(0) - , readonly_index(0), consecutive_error_times(0) - {} - - bool valid; - bool installing_snapshot; - bool blocking; + : valid(false), + installing_snapshot(false), + blocking(false), + next_index(0), + last_rpc_send_timestamp(0), + flying_append_entries_size(0), + readonly_index(0), + consecutive_error_times(0) {} + + bool valid; + bool installing_snapshot; + bool blocking; int64_t next_index; int64_t last_rpc_send_timestamp; int64_t flying_append_entries_size; int64_t readonly_index; - int consecutive_error_times; + int consecutive_error_times; }; // Status of Node @@ -378,10 +387,17 @@ struct NodeStatus { typedef std::map PeerStatusMap; NodeStatus() - : state(STATE_END), readonly(false), term(0), committed_index(0), known_applied_index(0) - , pending_index(0), pending_queue_size(0), applying_index(0), first_index(0) - , last_index(-1), disk_index(0) - {} + : state(STATE_END), + readonly(false), + term(0), + committed_index(0), + known_applied_index(0), + pending_index(0), + pending_queue_size(0), + applying_index(0), + first_index(0), + last_index(-1), + disk_index(0) {} State state; PeerId peer_id; @@ -393,7 +409,7 @@ struct NodeStatus { // The start index of the logs waiting to be committed. // If the value is 0, means no pending logs. - // + // // WARNING: if this value is not 0, and keep the same in a long time, // means something happened to prevent the node to commit logs in a // large probability, and users should check carefully to find out @@ -401,10 +417,10 @@ struct NodeStatus { int64_t pending_index; // How many pending logs waiting to be committed. - // - // WARNING: too many pending logs, means the processing rate can't catup with - // the writing rate. Users can consider to slow down the writing rate to avoid - // exhaustion of resources. + // + // WARNING: too many pending logs, means the processing rate can't catup + // with the writing rate. Users can consider to slow down the writing rate + // to avoid exhaustion of resources. int64_t pending_queue_size; // The current applying index. If the value is 0, means no applying log. @@ -433,13 +449,13 @@ struct NodeStatus { }; // State of a lease. Following is a typical lease state change diagram: -// +// // event: become leader become follower // ^ on leader start ^ on leader stop // | ^ | ^ // time: ----------|-----------|-----------------|---|------- -// lease state: EXPIRED | NOT_READY | VALID | EXPIRED -// +// lease state: EXPIRED | NOT_READY | VALID | EXPIRED +// enum LeaseState { // Lease is disabled, this state will only be returned when // |raft_enable_leader_lease == false|. @@ -448,7 +464,8 @@ enum LeaseState { // Lease is expired, this node is not leader any more. LEASE_EXPIRED = 2, - // This node is leader, but we are not sure the data is up to date. This state + // This node is leader, but we are not sure the data is up to date. This + // state // continue until |on_leader_start| or the leader step down. LEASE_NOT_READY = 3, @@ -458,28 +475,26 @@ enum LeaseState { // Status of a leader lease. struct LeaderLeaseStatus { - LeaderLeaseStatus() - : state(LEASE_DISABLED), term(0), lease_epoch(0) - {} + LeaderLeaseStatus() : state(LEASE_DISABLED), term(0), lease_epoch(0) {} LeaseState state; // These following fields are only meaningful when |state == LEASE_VALID|. - + // The term of this lease int64_t term; - // A specific term may have more than one lease, when transfer leader timeout - // happen. Lease epoch will be guranteed to be monotinically increase, in the - // life cycle of a node. + // A specific term may have more than one lease, when transfer leader + // timeout happen. Lease epoch will be guranteed to be monotinically + // increase, in the life cycle of a node. int64_t lease_epoch; }; struct NodeOptions { - // A follower would become a candidate if it doesn't receive any message + // A follower would become a candidate if it doesn't receive any message // from the leader in |election_timeout_ms| milliseconds // Default: 1000 (1s) - int election_timeout_ms; //follower to candidate timeout + int election_timeout_ms; // follower to candidate timeout // wait new peer to catchup log in |catchup_timeout_ms| milliseconds // if set to 0, it will same as election_timeout_ms @@ -513,7 +528,7 @@ struct NodeOptions { Configuration initial_conf; // Run the user callbacks and user closures in pthread rather than bthread - // + // // Default: false bool usercode_in_pthread; @@ -527,8 +542,8 @@ struct NodeOptions { // Default: false bool node_owns_fsm; - // The specific LogStorage implemented at the business layer, which should be a valid - // instance, otherwise use SegmentLogStorage by default. + // The specific LogStorage implemented at the business layer, which should + // be a valid instance, otherwise use SegmentLogStorage by default. // // Default: null LogStorage* log_storage; @@ -552,7 +567,7 @@ struct NodeOptions { // typical format: local://${node_path} // 2. type=local-merged // KVBasedMergedMetaStorage will be used, whose under layer is based - // on KV storage and manages a batch of Nodes one the same disk. It's + // on KV storage and manages a batch of Nodes one the same disk. It's // designed to solve performance problems caused by lots of small // synchronous IO during leader electing, when there are huge number of // Nodes in Multi-raft situation. @@ -562,7 +577,7 @@ struct NodeOptions { // two types of meta storages when upgrade an downgrade. // typical format: // local-mixed://merged_path=${disk_path}&&single_path=${node_path} - // + // // Upgrade and Downgrade steps: // upgrade from Single to Merged: local -> mixed -> merged // downgrade from Merged to Single: merged -> mixed -> local @@ -572,15 +587,15 @@ struct NodeOptions { std::string snapshot_uri; // If enable, we will filter duplicate files before copy remote snapshot, - // to avoid useless transmission. Two files in local and remote are duplicate, - // only if they has the same filename and the same checksum (stored in file meta). - // Default: false + // to avoid useless transmission. Two files in local and remote are + // duplicate, only if they has the same filename and the same checksum + // (stored in file meta). Default: false bool filter_before_copy_remote; - // If non-null, we will pass this snapshot_file_system_adaptor to SnapshotStorage - // Default: NULL - scoped_refptr* snapshot_file_system_adaptor; - + // If non-null, we will pass this snapshot_file_system_adaptor to + // SnapshotStorage Default: NULL + scoped_refptr* snapshot_file_system_adaptor; + // If non-null, we will pass this snapshot_throttle to SnapshotExecutor // Default: NULL scoped_refptr* snapshot_throttle; @@ -591,16 +606,20 @@ struct NodeOptions { // If true, this node is a witness. // 1. FLAGS_raft_enable_witness_to_leader = false - // It will never be elected as leader. So we don't need to init _vote_timer and _election_timer. + // It will never be elected as leader. So we don't need to init + // _vote_timer and _election_timer. // 2. FLAGS_raft_enable_witness_to_leader = true - // It can be electd as leader, but should transfer leader to normal replica as soon as possible. - // - // Warning: + // It can be electd as leader, but should transfer leader to normal + // replica as soon as possible. + // + // Warning: // 1. FLAGS_raft_enable_witness_to_leader = false - // When leader down and witness had newer log entry, it may cause leader election fail. + // When leader down and witness had newer log entry, it may cause leader + // election fail. // 2. FLAGS_raft_enable_witness_to_leader = true - // When leader shutdown and witness was elected as leader, if follower delay over one snapshot, - // it may cause data lost because witness had truncated log entry before snapshot. + // When leader shutdown and witness was elected as leader, if follower + // delay over one snapshot, it may cause data lost because witness had + // truncated log entry before snapshot. // Default: false bool witness = false; // Construct a default instance @@ -609,22 +628,21 @@ struct NodeOptions { int get_catchup_timeout_ms(); }; -inline NodeOptions::NodeOptions() - : election_timeout_ms(1000) - , catchup_timeout_ms(0) - , max_clock_drift_ms(1000) - , snapshot_interval_s(3600) - , catchup_margin(1000) - , usercode_in_pthread(false) - , fsm(NULL) - , node_owns_fsm(false) - , log_storage(NULL) - , node_owns_log_storage(true) - , filter_before_copy_remote(false) - , snapshot_file_system_adaptor(NULL) - , snapshot_throttle(NULL) - , disable_cli(false) -{} +inline NodeOptions::NodeOptions() + : election_timeout_ms(1000), + catchup_timeout_ms(0), + max_clock_drift_ms(1000), + snapshot_interval_s(3600), + catchup_margin(1000), + usercode_in_pthread(false), + fsm(NULL), + node_owns_fsm(false), + log_storage(NULL), + node_owns_log_storage(true), + filter_before_copy_remote(false), + snapshot_file_system_adaptor(NULL), + snapshot_throttle(NULL), + disable_cli(false) {} inline int NodeOptions::get_catchup_timeout_ms() { return (catchup_timeout_ms == 0) ? election_timeout_ms : catchup_timeout_ms; @@ -632,7 +650,7 @@ inline int NodeOptions::get_catchup_timeout_ms() { class NodeImpl; class Node { -public: + public: Node(const GroupId& group_id, const PeerId& peer_id); virtual ~Node(); @@ -648,9 +666,11 @@ class Node { // Return true if this is the leader, and leader lease is valid. It's always // false when |raft_enable_leader_lease == false|. // In the following situations, the returned true is unbeleivable: - // - Not all nodes in the raft group set |raft_enable_leader_lease| to true, + // - Not all nodes in the raft group set |raft_enable_leader_lease| to + // true, // and tranfer leader/vote interfaces are used; - // - In the raft group, the value of |election_timeout_ms| in one node is larger + // - In the raft group, the value of |election_timeout_ms| in one node + // is larger // than |election_timeout_ms + max_clock_drift_ms| in another peer. bool is_leader_lease_valid(); @@ -661,8 +681,8 @@ class Node { int init(const NodeOptions& options); // shutdown local replica. - // done is user defined function, maybe response to client or clean some resource - // [NOTE] code after apply can't access resource in done + // done is user defined function, maybe response to client or clean some + // resource [NOTE] code after apply can't access resource in done void shutdown(Closure* done); // Block the thread until the node is successfully stopped. @@ -672,7 +692,7 @@ class Node { // apply task to the replicated-state-machine // // About the ownership: - // |task.data|: for the performance consideration, we will take away the + // |task.data|: for the performance consideration, we will take away the // content. If you want keep the content, copy it before call // this function // |task.done|: If the data is successfully committed to the raft group. We @@ -682,8 +702,9 @@ class Node { void apply(const Task& task); // list peers of this raft group, only leader retruns ok - // [NOTE] when list_peers concurrency with add_peer/remove_peer, maybe return peers is staled. - // because add_peer/remove_peer immediately modify configuration in memory + // [NOTE] when list_peers concurrency with add_peer/remove_peer, maybe + // return peers is staled. because add_peer/remove_peer immediately modify + // configuration in memory butil::Status list_peers(std::vector* peers); // Add a new peer to the raft group. done->Run() would be invoked after this @@ -717,29 +738,35 @@ class Node { // higher probability butil::Status vote(int election_timeout); - // Reset the |election_timeout_ms| for the very node, the |max_clock_drift_ms| - // is also adjusted to keep the sum of |election_timeout_ms| and |the max_clock_drift_ms| - // unchanged. + // Reset the |election_timeout_ms| for the very node, the + // |max_clock_drift_ms| is also adjusted to keep the sum of + // |election_timeout_ms| and |the max_clock_drift_ms| unchanged. butil::Status reset_election_timeout_ms(int election_timeout_ms); - // Forcely reset |election_timeout_ms| and |max_clock_drift_ms|. It may break - // leader lease safety, should be careful. - // Following are suggestions for you to change |election_timeout_ms| safely. + // Forcely reset |election_timeout_ms| and |max_clock_drift_ms|. It may + // break leader lease safety, should be careful. Following are suggestions + // for you to change |election_timeout_ms| safely. // 1. Three steps to safely upgrade |election_timeout_ms| to a larger one: // - Enlarge |max_clock_drift_ms| in all peers to make sure // |old election_timeout_ms + new max_clock_drift_ms| larger than // |new election_timeout_ms + old max_clock_drift_ms|. - // - Wait at least |old election_timeout_ms + new max_clock_drift_ms| times to make + // - Wait at least |old election_timeout_ms + new max_clock_drift_ms| + // times to make // sure all previous elections complete. - // - Upgrade |election_timeout_ms| to new one, meanwhiles |max_clock_drift_ms| + // - Upgrade |election_timeout_ms| to new one, meanwhiles + // |max_clock_drift_ms| // can set back to the old value. // 2. Three steps to safely upgrade |election_timeout_ms| to a smaller one: - // - Adjust |election_timeout_ms| and |max_clock_drift_ms| at the same time, - // to make the sum of |election_timeout_ms + max_clock_drift_ms| unchanged. - // - Wait at least |election_timeout_ms + max_clock_drift_ms| times to make + // - Adjust |election_timeout_ms| and |max_clock_drift_ms| at the same + // time, + // to make the sum of |election_timeout_ms + max_clock_drift_ms| + // unchanged. + // - Wait at least |election_timeout_ms + max_clock_drift_ms| times to + // make // sure all previous elections complete. // - Upgrade |max_clock_drift_ms| back to the old value. - void reset_election_timeout_ms(int election_timeout_ms, int max_clock_drift_ms); + void reset_election_timeout_ms(int election_timeout_ms, + int max_clock_drift_ms); // Try transferring leadership to |peer|. // If peer is ANY_PEER, a proper follower will be chosen as the leader for @@ -748,35 +775,37 @@ class Node { int transfer_leadership_to(const PeerId& peer); // Read the first committed user log from the given index. - // Return OK on success and user_log is assigned with the very data. Be awared - // that the user_log may be not the exact log at the given index, but the - // first available user log from the given index to last_committed_index. - // Otherwise, appropriate errors are returned: + // Return OK on success and user_log is assigned with the very data. Be + // awared that the user_log may be not the exact log at the given index, but + // the first available user log from the given index to + // last_committed_index. Otherwise, appropriate errors are returned: // - return ELOGDELETED when the log has been deleted; - // - return ENOMOREUSERLOG when we can't get a user log even reaching last_committed_index. - // [NOTE] in consideration of safety, we use last_applied_index instead of last_committed_index - // in code implementation. - butil::Status read_committed_user_log(const int64_t index, UserLog* user_log); - - // Get the internal status of this node, the information is mostly the same as we - // see from the website. + // - return ENOMOREUSERLOG when we can't get a user log even reaching + // last_committed_index. + // [NOTE] in consideration of safety, we use last_applied_index instead of + // last_committed_index in code implementation. + butil::Status read_committed_user_log(const int64_t index, + UserLog* user_log); + + // Get the internal status of this node, the information is mostly the same + // as we see from the website. void get_status(NodeStatus* status); // Make this node enter readonly mode. - // Readonly mode should only be used to protect the system in some extreme cases. - // For example, in a storage system, too many write requests flood into the system - // unexpectly, and the system is in the danger of exhaust capacity. There's not enough - // time to add new machines, and wait for capacity balance. Once many disks become - // full, quorum dead happen to raft groups. One choice in this example is readonly - // mode, to let leader reject new write requests, but still handle reads request, - // and configuration changes. - // If a follower become readonly, the leader stop replicate new logs to it. This - // may cause the data far behind the leader, in the case that the leader is still - // writable. After the follower exit readonly mode, the leader will resume to - // replicate missing logs. - // A leader is readonly, if the node itself is readonly, or writable nodes (nodes that - // are not marked as readonly) in the group is less than majority. Once a leader become - // readonly, no new users logs will be acceptted. + // Readonly mode should only be used to protect the system in some extreme + // cases. For example, in a storage system, too many write requests flood + // into the system unexpectly, and the system is in the danger of exhaust + // capacity. There's not enough time to add new machines, and wait for + // capacity balance. Once many disks become full, quorum dead happen to raft + // groups. One choice in this example is readonly mode, to let leader reject + // new write requests, but still handle reads request, and configuration + // changes. If a follower become readonly, the leader stop replicate new + // logs to it. This may cause the data far behind the leader, in the case + // that the leader is still writable. After the follower exit readonly mode, + // the leader will resume to replicate missing logs. A leader is readonly, + // if the node itself is readonly, or writable nodes (nodes that are not + // marked as readonly) in the group is less than majority. Once a leader + // become readonly, no new users logs will be acceptted. void enter_readonly_mode(); // Node leave readonly node. @@ -785,16 +814,16 @@ class Node { // Check if this node is readonly. // There are two situations that if a node is readonly: // - This node is marked as readonly, by calling enter_readonly_mode(); - // - This node is a leader, and the count of writable nodes in the group + // - This node is a leader, and the count of writable nodes in the + // group // is less than the majority. bool readonly(); -private: + private: NodeImpl* _impl; }; struct BootstrapOptions { - // Containing the initial member of this raft group // Default: empty conf Configuration group_conf; @@ -803,7 +832,7 @@ struct BootstrapOptions { // Default: 0 int64_t last_log_index; - // The specific StateMachine which is going to dump the first snapshot + // The specific StateMachine which is going to dump the first snapshot // If last_log_index isn't 0, fsm must be a valid instance. // Default: NULL StateMachine* fsm; @@ -815,7 +844,7 @@ struct BootstrapOptions { bool node_owns_fsm; // Run the user callbacks and user closures in pthread rather than bthread - // + // // Default: false bool usercode_in_pthread; @@ -830,17 +859,16 @@ struct BootstrapOptions { // Construct default options BootstrapOptions(); - }; -// Bootstrap a non-empty raft node, +// Bootstrap a non-empty raft node, int bootstrap(const BootstrapOptions& options); // Attach raft services to |server|, this makes the raft services share the same // listening address with the user services. // // NOTE: Now we only allow the backing Server to be started with a specific -// listen address, if the Server is going to be started from a range of ports, +// listen address, if the Server is going to be started from a range of ports, // the behavior is undefined. // Returns 0 on success, -1 otherwise. int add_service(brpc::Server* server, const butil::EndPoint& listen_addr); @@ -849,19 +877,19 @@ int add_service(brpc::Server* server, const char* listen_ip_and_port); // GC struct GCOptions { - // Versioned-groupid of this raft instance. - // Version is necessary because instance with the same groupid may be created - // again very soon after destroyed. + // Versioned-groupid of this raft instance. + // Version is necessary because instance with the same groupid may be + // created again very soon after destroyed. VersionedGroupId vgid; std::string log_uri; std::string raft_meta_uri; std::string snapshot_uri; }; -// TODO What if a disk is dropped and added again without released from +// TODO What if a disk is dropped and added again without released from // global_mss_manager? It seems ok because all the instance on that disk would -// be destroyed before dropping the disk itself, so there would be no garbage. -// +// be destroyed before dropping the disk itself, so there would be no garbage. +// // GC the data of a raft instance when destroying the instance by some reason. // // Returns 0 on success, -1 otherwise. @@ -869,4 +897,4 @@ int gc_raft_data(const GCOptions& gc_options); } // namespace braft -#endif //BRAFT_RAFT_H +#endif // BRAFT_RAFT_H diff --git a/src/braft/raft_meta.cpp b/src/braft/raft_meta.cpp index fc791e7..8edfc96 100644 --- a/src/braft/raft_meta.cpp +++ b/src/braft/raft_meta.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,21 +15,24 @@ // Authors: Wang,Yao(wangyao02@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#include -#include +#include "braft/raft_meta.h" + +#include +#include // butil::CreateDirectory #include -#include // butil::CreateDirectory +#include +#include #include -#include -#include "braft/util.h" -#include "braft/protobuf_file.h" + #include "braft/local_storage.pb.h" -#include "braft/raft_meta.h" +#include "braft/protobuf_file.h" +#include "braft/util.h" namespace braft { -DEFINE_int32(raft_meta_write_batch, 128, - "Max number of tasks that can be written into db in a single batch"); +DEFINE_int32( + raft_meta_write_batch, 128, + "Max number of tasks that can be written into db in a single batch"); BRPC_VALIDATE_GFLAG(raft_meta_write_batch, brpc::PositiveInteger); static bvar::LatencyRecorder g_load_pb_raft_meta("raft_load_pb_raft_meta"); @@ -39,37 +42,36 @@ static bvar::LatencyRecorder g_save_kv_raft_meta("raft_save_kv_raft_meta"); static bvar::LatencyRecorder g_delete_kv_raft_meta("raft_delete_kv_raft_meta"); static bvar::CounterRecorder g_save_kv_raft_meta_batch_counter( - "raft_save_kv_raft_meta_batch_counter"); + "raft_save_kv_raft_meta_batch_counter"); const char* FileBasedSingleMetaStorage::_s_raft_meta = "raft_meta"; // MetaStorageManager // // To manage all KVBasedMergedMetaStorageImpl of all the raft instances. -// Typically nodes on the same disk will share a KVBasedMergedMetaStorageImpl, +// Typically nodes on the same disk will share a KVBasedMergedMetaStorageImpl, // so we use disk_path as the KEY to manage all the instances. class MetaStorageManager { -public: + public: static MetaStorageManager* GetInstance() { return Singleton::get(); } - scoped_refptr - register_meta_storage(const std::string& path) { - scoped_refptr mss = get_meta_storage(path); + scoped_refptr register_meta_storage( + const std::string& path) { + scoped_refptr mss = + get_meta_storage(path); if (mss != NULL) { return mss; } - + mss = new KVBasedMergedMetaStorageImpl(path); - { - _ss_map.Modify(_add, path, mss); - } - return get_meta_storage(path); + { _ss_map.Modify(_add, path, mss); } + return get_meta_storage(path); } - - scoped_refptr - get_meta_storage(const std::string& path) { + + scoped_refptr get_meta_storage( + const std::string& path) { DoublyBufferedMetaStorageMap::ScopedPtr ptr; CHECK_EQ(0, _ss_map.Read(&ptr)); MetaStorageMap::const_iterator it = ptr->find(path); @@ -78,13 +80,13 @@ class MetaStorageManager { } return NULL; } - - // GC an invalid item in KVBasedMergedMetaStorageImpl when destroying + + // GC an invalid item in KVBasedMergedMetaStorageImpl when destroying // an raft instance on the disk for some reason, such as IO error. - int remove_instance_from_meta_storage(const std::string& path, - const VersionedGroupId& v_group_id) { - scoped_refptr mss = - get_meta_storage(path); + int remove_instance_from_meta_storage(const std::string& path, + const VersionedGroupId& v_group_id) { + scoped_refptr mss = + get_meta_storage(path); if (mss == NULL) { return 0; } @@ -95,26 +97,27 @@ class MetaStorageManager { return 0; } -private: - MetaStorageManager() {}; - ~MetaStorageManager() {}; + private: + MetaStorageManager(){}; + ~MetaStorageManager(){}; DISALLOW_COPY_AND_ASSIGN(MetaStorageManager); friend struct DefaultSingletonTraits; - - typedef std::map > - MetaStorageMap; - typedef butil::DoublyBufferedData DoublyBufferedMetaStorageMap; - - static size_t _add(MetaStorageMap& m, const std::string& path, + + typedef std::map > + MetaStorageMap; + typedef butil::DoublyBufferedData + DoublyBufferedMetaStorageMap; + + static size_t _add(MetaStorageMap& m, const std::string& path, const scoped_refptr& mss) { - std::pair iter = - m.insert(std::make_pair(path, mss)); + std::pair iter = + m.insert(std::make_pair(path, mss)); if (iter.second) { return 1lu; } return 0lu; } - + static size_t _remove(MetaStorageMap& m, const std::string& path) { return m.erase(path); } @@ -126,32 +129,32 @@ class MetaStorageManager { // MixedMetaStorage // -// Uri of Multi-raft using mixed stable storage is: +// Uri of Multi-raft using mixed stable storage is: // local-mixed://merged_path={merged_path}&&single_path={single_path} -int MixedMetaStorage::parse_mixed_path(const std::string& uri, - std::string& merged_path, - std::string& single_path) { +int MixedMetaStorage::parse_mixed_path(const std::string& uri, + std::string& merged_path, + std::string& single_path) { // here uri has removed protocol already, check just for safety butil::StringPiece copied_uri(uri); size_t pos = copied_uri.find("://"); if (pos != butil::StringPiece::npos) { - copied_uri.remove_prefix(pos + 3/* length of '://' */); + copied_uri.remove_prefix(pos + 3 /* length of '://' */); } - + pos = copied_uri.find("merged_path="); if (pos == butil::StringPiece::npos) { return -1; } - copied_uri.remove_prefix(pos + 12/* length of 'merged_path=' */); - + copied_uri.remove_prefix(pos + 12 /* length of 'merged_path=' */); + pos = copied_uri.find("&&single_path="); if (pos == butil::StringPiece::npos) { return -1; } merged_path = copied_uri.substr(0, pos).as_string(); - copied_uri.remove_prefix(pos + 14/* length of '&&single_path=' */); + copied_uri.remove_prefix(pos + 14 /* length of '&&single_path=' */); single_path = copied_uri.as_string(); - + return 0; } @@ -165,9 +168,9 @@ MixedMetaStorage::MixedMetaStorage(const std::string& uri) { int ret = parse_mixed_path(uri, merged_path, single_path); if (ret != 0) { LOG(ERROR) << "node parse mixed path failed, uri " << uri; - _is_bad = true; + _is_bad = true; } else { - // Use single_path as the path of MixedMetaStorage as it usually + // Use single_path as the path of MixedMetaStorage as it usually // contains group_id _path = single_path; @@ -177,7 +180,8 @@ MixedMetaStorage::MixedMetaStorage(const std::string& uri) { if (!_single_impl || !_merged_impl) { // Both _single_impl and _merged_impl are needed in MixedMetaStorage LOG(ERROR) << "MixedMetaStorage failed to create both" - " sub stable storage, uri " << uri; + " sub stable storage, uri " + << uri; _is_bad = true; } } @@ -187,7 +191,7 @@ MixedMetaStorage::~MixedMetaStorage() { if (_single_impl) { delete _single_impl; _single_impl = NULL; - } + } if (_merged_impl) { _merged_impl = NULL; } @@ -195,28 +199,30 @@ MixedMetaStorage::~MixedMetaStorage() { butil::Status MixedMetaStorage::init() { butil::Status status; - if (_is_inited) { + if (_is_inited) { return status; } // check bad if (_is_bad) { - status.set_error(EINVAL, "MixedMetaStorage is bad, path %s", + status.set_error(EINVAL, "MixedMetaStorage is bad, path %s", _path.c_str()); return status; } - + // both _single_impl and _merged_impl are valid since _is_bad is false status = _single_impl->init(); if (!status.ok()) { LOG(ERROR) << "Init Mixed stable storage failed because init Single" - " stable storage failed, path " << _path; + " stable storage failed, path " + << _path; return status; } status = _merged_impl->init(); if (!status.ok()) { LOG(ERROR) << "Init Mixed stable storage failed because init merged" - " stable storage failed, path " << _path; + " stable storage failed, path " + << _path; return status; } @@ -226,15 +232,14 @@ butil::Status MixedMetaStorage::init() { } class StableMetaClosure : public Closure { -public: - StableMetaClosure(const int64_t term, const PeerId& votedfor, - const VersionedGroupId& vgid, const std::string& path) - : _term(term) - , _votedfor(votedfor) - , _vgid(vgid) - , _path(path) - , _start_time_us(butil::cpuwide_time_us()) - {} + public: + StableMetaClosure(const int64_t term, const PeerId& votedfor, + const VersionedGroupId& vgid, const std::string& path) + : _term(term), + _votedfor(votedfor), + _vgid(vgid), + _path(path), + _start_time_us(butil::cpuwide_time_us()) {} ~StableMetaClosure() {} @@ -246,19 +251,17 @@ class StableMetaClosure : public Closure { } else { int64_t u_elapsed = butil::cpuwide_time_us() - _start_time_us; g_save_kv_raft_meta << u_elapsed; - LOG(INFO) << "Saved merged stable meta, path " << _path - << " group " << _vgid - << " term " << _term - << " votedfor " << _votedfor - << " time: " << u_elapsed; + LOG(INFO) << "Saved merged stable meta, path " << _path << " group " + << _vgid << " term " << _term << " votedfor " << _votedfor + << " time: " << u_elapsed; } - + _sync.Run(); } void wait() { _sync.wait(); } -private: + private: int64_t _term; PeerId _votedfor; VersionedGroupId _vgid; @@ -267,19 +270,21 @@ class StableMetaClosure : public Closure { SynchronizedClosure _sync; }; -butil::Status MixedMetaStorage::set_term_and_votedfor(const int64_t term, - const PeerId& peer_id, const VersionedGroupId& group) { +butil::Status MixedMetaStorage::set_term_and_votedfor( + const int64_t term, const PeerId& peer_id, const VersionedGroupId& group) { butil::Status status; if (!_is_inited) { LOG(WARNING) << "MixedMetaStorage not init, path: " << _path; - status.set_error(EINVAL, "MixedMetaStorage of group %s not init, path: %s", + status.set_error(EINVAL, + "MixedMetaStorage of group %s not init, path: %s", group.c_str(), _path.c_str()); return status; } status = _single_impl->set_term_and_votedfor(term, peer_id, group); if (!status.ok()) { - LOG(WARNING) << "node " << group + LOG(WARNING) + << "node " << group << " single stable storage failed to set_term_and_votedfor, path: " << _path; return status; @@ -291,46 +296,51 @@ butil::Status MixedMetaStorage::set_term_and_votedfor(const int64_t term, return done.status(); } -// [NOTICE] Conflict cases may occur in this mode, it's important to ensure consistency +// [NOTICE] Conflict cases may occur in this mode, it's important to ensure +// consistency // 1. Single is newer than Merged: // case 1: upgrade storage from Single to Mixed, data in Merged is stale -// case 2: last set_term_and_votedfor succeeded in Single but failed in Merged +// case 2: last set_term_and_votedfor succeeded in Single but failed in +// Merged // 2. Merged is newer than Single: // case: downgrade storage from Merged to Mixed, data in Single is stale -butil::Status MixedMetaStorage::get_term_and_votedfor(int64_t* term, PeerId* peer_id, - const VersionedGroupId& group) { +butil::Status MixedMetaStorage::get_term_and_votedfor( + int64_t* term, PeerId* peer_id, const VersionedGroupId& group) { butil::Status status; if (!_is_inited) { LOG(WARNING) << "MixedMetaStorage not init, path: " << _path; - status.set_error(EINVAL, "MixedMetaStorage of group %s not init, path: %s", + status.set_error(EINVAL, + "MixedMetaStorage of group %s not init, path: %s", group.c_str(), _path.c_str()); return status; } - - // If data from single stable storage is newer than that from merged stable storage, - // merged stable storage should catch up the newer data to ensure safety; Vice versa. + + // If data from single stable storage is newer than that from merged stable + // storage, merged stable storage should catch up the newer data to ensure + // safety; Vice versa. bool single_newer_than_merged = false; - + int64_t term_1; PeerId peer_id_1; status = _single_impl->get_term_and_votedfor(&term_1, &peer_id_1, group); if (!status.ok()) { - LOG(WARNING) << "node " << group + LOG(WARNING) + << "node " << group << " single stable storage failed to get_term_and_votedfor, path: " << _path << ", error: " << status.error_cstr(); return status; } - + int64_t term_2; PeerId peer_id_2; - status = _merged_impl->get_term_and_votedfor(&term_2, - &peer_id_2, group); + status = _merged_impl->get_term_and_votedfor(&term_2, &peer_id_2, group); if (!status.ok()) { LOG(WARNING) << "node " << group - << " merged stable storage failed to get_term_and_votdfor," - << " path: " << _path << ", error: " << status.error_cstr(); + << " merged stable storage failed to get_term_and_votdfor," + << " path: " << _path + << ", error: " << status.error_cstr(); return status; - // check consistency of two stable storage + // check consistency of two stable storage } else if (term_1 == term_2 && peer_id_1 == peer_id_2) { // if two results are consistent, just return success *term = term_1; @@ -338,23 +348,25 @@ butil::Status MixedMetaStorage::get_term_and_votedfor(int64_t* term, PeerId* pee return status; } - // this case is theoretically impossible, pay much attention to it if happens - if (term_1 == term_2 && peer_id_1 != ANY_PEER - && peer_id_2 != ANY_PEER) { - CHECK(false) << "Unexpected conflict when mixed stable storage of " - << group << " get_term_and_votedfor, the same term " << term_1 - << ", but different non-empty votdfor(" << peer_id_1 - << " from single stable storage and " << peer_id_2 - << " from merged stable storage), path: " << _path; + // this case is theoretically impossible, pay much attention to it if + // happens + if (term_1 == term_2 && peer_id_1 != ANY_PEER && peer_id_2 != ANY_PEER) { + CHECK(false) << "Unexpected conflict when mixed stable storage of " + << group << " get_term_and_votedfor, the same term " + << term_1 << ", but different non-empty votdfor(" + << peer_id_1 << " from single stable storage and " + << peer_id_2 + << " from merged stable storage), path: " << _path; status.set_error(EINVAL, "Unexpected conflict"); return status; } // if two results are not consistent, check out which is newer and catch up - // data for the stale one - single_newer_than_merged = term_1 > term_2 || + // data for the stale one + single_newer_than_merged = + term_1 > term_2 || (term_1 == term_2 && peer_id_1 != ANY_PEER && peer_id_2 == ANY_PEER); - + if (single_newer_than_merged) { *term = term_1; *peer_id = peer_id_1; @@ -363,35 +375,35 @@ butil::Status MixedMetaStorage::get_term_and_votedfor(int64_t* term, PeerId* pee done.wait(); status = done.status(); if (!status.ok()) { - LOG(WARNING) << "node " << group - << " merged stable storage failed to set term " << *term - << " and vote for peer " << *peer_id - << " when catch up data, path " << _path - << ", error: " << status.error_cstr(); + LOG(WARNING) << "node " << group + << " merged stable storage failed to set term " + << *term << " and vote for peer " << *peer_id + << " when catch up data, path " << _path + << ", error: " << status.error_cstr(); return status; } - LOG(INFO) << "node " << group - << " merged stable storage succeed to set term " << *term - << " and vote for peer " << *peer_id - << " when catch up data, path " << _path; + LOG(INFO) << "node " << group + << " merged stable storage succeed to set term " << *term + << " and vote for peer " << *peer_id + << " when catch up data, path " << _path; } else { LOG(WARNING) << "LocalMetaStorage not init(), path: " << _path; *term = term_2; *peer_id = peer_id_2; status = _single_impl->set_term_and_votedfor(*term, *peer_id, group); if (!status.ok()) { - LOG(WARNING) << "node " << group - << " single stable storage failed to set term " << *term - << " and vote for peer " << *peer_id - << " when catch up data, path " << _path - << ", error: " << status.error_cstr(); + LOG(WARNING) << "node " << group + << " single stable storage failed to set term " + << *term << " and vote for peer " << *peer_id + << " when catch up data, path " << _path + << ", error: " << status.error_cstr(); return status; - } - LOG(INFO) << "node " << group - << " single stable storage succeed to set term " << *term - << " and vote for peer " << *peer_id - << " when catch up data, path " << _path; - } + } + LOG(INFO) << "node " << group + << " single stable storage succeed to set term " << *term + << " and vote for peer " << *peer_id + << " when catch up data, path " << _path; + } return status; } @@ -400,8 +412,8 @@ RaftMetaStorage* MixedMetaStorage::new_instance(const std::string& uri) const { return new MixedMetaStorage(uri); } -butil::Status MixedMetaStorage::gc_instance(const std::string& uri, - const VersionedGroupId& vgid) const { +butil::Status MixedMetaStorage::gc_instance( + const std::string& uri, const VersionedGroupId& vgid) const { butil::Status status; std::string merged_path; std::string single_path; @@ -409,29 +421,31 @@ butil::Status MixedMetaStorage::gc_instance(const std::string& uri, int ret = parse_mixed_path(uri, merged_path, single_path); if (ret != 0) { LOG(WARNING) << "node parse mixed path failed, uri " << uri; - status.set_error(EINVAL, "Group %s failed to parse mixed path, uri %s", + status.set_error(EINVAL, "Group %s failed to parse mixed path, uri %s", vgid.c_str(), uri.c_str()); return status; } if (0 != gc_dir(single_path)) { - LOG(WARNING) << "Group " << vgid << " failed to gc path " << single_path; - status.set_error(EIO, "Group %s failed to gc path %s", - vgid.c_str(), single_path.c_str()); + LOG(WARNING) << "Group " << vgid << " failed to gc path " + << single_path; + status.set_error(EIO, "Group %s failed to gc path %s", vgid.c_str(), + single_path.c_str()); return status; } - if (0 != global_mss_manager-> - remove_instance_from_meta_storage(merged_path, vgid)) { - LOG(ERROR) << "Group " << vgid << " failed to gc kv from path: " - << merged_path; - status.set_error(EIO, "Group %s failed to gc kv from path %s", + if (0 != global_mss_manager->remove_instance_from_meta_storage(merged_path, + vgid)) { + LOG(ERROR) << "Group " << vgid + << " failed to gc kv from path: " << merged_path; + status.set_error(EIO, "Group %s failed to gc kv from path %s", vgid.c_str(), merged_path.c_str()); return status; } - LOG(INFO) << "Group " << vgid << " succeed to gc from single path: " - << single_path << " and merged path: " << merged_path; - return status; - } - + LOG(INFO) << "Group " << vgid + << " succeed to gc from single path: " << single_path + << " and merged path: " << merged_path; + return status; +} + // FileBasedSingleMetaStorage butil::Status FileBasedSingleMetaStorage::init() { butil::Status status; @@ -442,19 +456,24 @@ butil::Status FileBasedSingleMetaStorage::init() { butil::FilePath dir_path(_path); butil::File::Error e; if (!butil::CreateDirectoryAndGetError( - dir_path, &e, FLAGS_raft_create_parent_directories)) { + dir_path, &e, FLAGS_raft_create_parent_directories)) { LOG(ERROR) << "Fail to create " << dir_path.value() << " : " << e; - status.set_error(e, "Fail to create dir when init SingleMetaStorage, " - "path: %s", _path.c_str()); + status.set_error(e, + "Fail to create dir when init SingleMetaStorage, " + "path: %s", + _path.c_str()); return status; } int ret = load(); if (ret != 0) { LOG(ERROR) << "Fail to load pb meta when init single stable storage" - ", path: " << _path; - status.set_error(EIO, "Fail to load pb meta when init stabel storage" - ", path: %s", _path.c_str()); + ", path: " + << _path; + status.set_error(EIO, + "Fail to load pb meta when init stabel storage" + ", path: %s", + _path.c_str()); return status; } @@ -462,32 +481,33 @@ butil::Status FileBasedSingleMetaStorage::init() { return status; } -butil::Status FileBasedSingleMetaStorage::set_term_and_votedfor(const int64_t term, - const PeerId& peer_id, const VersionedGroupId&) { +butil::Status FileBasedSingleMetaStorage::set_term_and_votedfor( + const int64_t term, const PeerId& peer_id, const VersionedGroupId&) { butil::Status status; if (!_is_inited) { - status.set_error(EINVAL, "SingleMetaStorage not init, path: %s", + status.set_error(EINVAL, "SingleMetaStorage not init, path: %s", _path.c_str()); return status; - } + } _term = term; _votedfor = peer_id; if (save() != 0) { - status.set_error(EIO, "SingleMetaStorage failed to save pb meta, path: %s", + status.set_error(EIO, + "SingleMetaStorage failed to save pb meta, path: %s", _path.c_str()); return status; } return status; } - -butil::Status FileBasedSingleMetaStorage::get_term_and_votedfor(int64_t* term, - PeerId* peer_id, const VersionedGroupId& group) { + +butil::Status FileBasedSingleMetaStorage::get_term_and_votedfor( + int64_t* term, PeerId* peer_id, const VersionedGroupId& group) { butil::Status status; if (!_is_inited) { - status.set_error(EINVAL, "SingleMetaStorage not init, path: %s", + status.set_error(EINVAL, "SingleMetaStorage not init, path: %s", _path.c_str()); return status; - } + } *term = _term; *peer_id = _votedfor; return status; @@ -496,7 +516,7 @@ butil::Status FileBasedSingleMetaStorage::get_term_and_votedfor(int64_t* term, int FileBasedSingleMetaStorage::load() { butil::Timer timer; timer.start(); - + std::string path(_path); path.append("/"); path.append(_s_raft_meta); @@ -513,14 +533,13 @@ int FileBasedSingleMetaStorage::load() { } else { PLOG(ERROR) << "Fail to load meta from " << path; } - + timer.stop(); // Only reload process will load stable meta of raft instances, // reading just go through memory g_load_pb_raft_meta << timer.u_elapsed(); - LOG(INFO) << "Loaded single stable meta, path " << _path - << " term " << _term - << " votedfor " << _votedfor.to_string() + LOG(INFO) << "Loaded single stable meta, path " << _path << " term " + << _term << " votedfor " << _votedfor.to_string() << " time: " << timer.u_elapsed(); return ret; } @@ -543,30 +562,35 @@ int FileBasedSingleMetaStorage::save() { timer.stop(); g_save_pb_raft_meta << timer.u_elapsed(); - LOG(INFO) << "Saved single stable meta, path " << _path - << " term " << _term - << " votedfor " << _votedfor.to_string() + LOG(INFO) << "Saved single stable meta, path " << _path << " term " << _term + << " votedfor " << _votedfor.to_string() << " time: " << timer.u_elapsed(); return ret; } RaftMetaStorage* FileBasedSingleMetaStorage::new_instance( - const std::string& uri) const { + const std::string& uri) const { return new FileBasedSingleMetaStorage(uri); } -butil::Status FileBasedSingleMetaStorage::gc_instance(const std::string& uri, - const VersionedGroupId& vgid) const { +butil::Status FileBasedSingleMetaStorage::gc_instance( + const std::string& uri, const VersionedGroupId& vgid) const { butil::Status status; if (0 != gc_dir(uri)) { - LOG(WARNING) << "Group " << vgid << " failed to gc single stable storage" - ", path: " << uri; - status.set_error(EIO, "Group %s failed to gc single stable storage" - ", path: %s", vgid.c_str(), uri.c_str()); + LOG(WARNING) << "Group " << vgid + << " failed to gc single stable storage" + ", path: " + << uri; + status.set_error(EIO, + "Group %s failed to gc single stable storage" + ", path: %s", + vgid.c_str(), uri.c_str()); return status; } - LOG(INFO) << "Group " << vgid << " succeed to gc single stable storage" - ", path: " << uri; + LOG(INFO) << "Group " << vgid + << " succeed to gc single stable storage" + ", path: " + << uri; return status; } @@ -581,36 +605,33 @@ KVBasedMergedMetaStorage::~KVBasedMergedMetaStorage() { } } -butil::Status KVBasedMergedMetaStorage::init() { - return _merged_impl->init(); -}; +butil::Status KVBasedMergedMetaStorage::init() { return _merged_impl->init(); }; -butil::Status KVBasedMergedMetaStorage::set_term_and_votedfor(const int64_t term, - const PeerId& peer_id, const VersionedGroupId& group) { +butil::Status KVBasedMergedMetaStorage::set_term_and_votedfor( + const int64_t term, const PeerId& peer_id, const VersionedGroupId& group) { StableMetaClosure done(term, peer_id, group, ""); _merged_impl->set_term_and_votedfor(term, peer_id, group, &done); done.wait(); - + return done.status(); }; -butil::Status KVBasedMergedMetaStorage::get_term_and_votedfor(int64_t* term, - PeerId* peer_id, const VersionedGroupId& group) { +butil::Status KVBasedMergedMetaStorage::get_term_and_votedfor( + int64_t* term, PeerId* peer_id, const VersionedGroupId& group) { return _merged_impl->get_term_and_votedfor(term, peer_id, group); }; RaftMetaStorage* KVBasedMergedMetaStorage::new_instance( - const std::string& uri) const { + const std::string& uri) const { return new KVBasedMergedMetaStorage(uri); } -butil::Status KVBasedMergedMetaStorage::gc_instance(const std::string& uri, - const VersionedGroupId& vgid) const { +butil::Status KVBasedMergedMetaStorage::gc_instance( + const std::string& uri, const VersionedGroupId& vgid) const { butil::Status status; - if (0 != global_mss_manager-> - remove_instance_from_meta_storage(uri, vgid)) { + if (0 != global_mss_manager->remove_instance_from_meta_storage(uri, vgid)) { LOG(WARNING) << "Group " << vgid << " failed to gc kv, path: " << uri; - status.set_error(EIO, "Group %s failed to gc kv in path: %s", + status.set_error(EIO, "Group %s failed to gc kv in path: %s", vgid.c_str(), uri.c_str()); return status; } @@ -619,90 +640,92 @@ butil::Status KVBasedMergedMetaStorage::gc_instance(const std::string& uri, }; butil::Status KVBasedMergedMetaStorage::delete_meta( - const VersionedGroupId& group) { + const VersionedGroupId& group) { return _merged_impl->delete_meta(group); }; // KVBasedMergedMetaStorageImpl butil::Status KVBasedMergedMetaStorageImpl::init() { - std::unique_lock lck(_mutex); + std::unique_lock lck(_mutex); butil::Status status; if (_is_inited) { return status; } - + butil::FilePath dir_path(_path); butil::File::Error e; if (!butil::CreateDirectoryAndGetError( - dir_path, &e, FLAGS_raft_create_parent_directories)) { + dir_path, &e, FLAGS_raft_create_parent_directories)) { lck.unlock(); LOG(ERROR) << "Fail to create " << dir_path.value() << " : " << e; - status.set_error(e, "Fail to create dir when init MergedMetaStorage, " - "path: %s", _path.c_str()); + status.set_error(e, + "Fail to create dir when init MergedMetaStorage, " + "path: %s", + _path.c_str()); return status; } leveldb::Options options; options.create_if_missing = true; - //options.error_if_exists = true; + // options.error_if_exists = true; leveldb::Status st; st = leveldb::DB::Open(options, _path.c_str(), &_db); if (!st.ok()) { lck.unlock(); - LOG(ERROR) << "Fail to open db: " << st.ToString() << " path: " << _path; - status.set_error(EIO, "Fail to open db, path: %s, error: %s", + LOG(ERROR) << "Fail to open db: " << st.ToString() + << " path: " << _path; + status.set_error(EIO, "Fail to open db, path: %s, error: %s", _path.c_str(), st.ToString().c_str()); return status; - } + } // start execution_queue bthread::ExecutionQueueOptions execq_opt; execq_opt.bthread_attr = BTHREAD_ATTR_NORMAL; - //execq_opt.max_tasks_size = 256; - if (bthread::execution_queue_start(&_queue_id, - &execq_opt, + // execq_opt.max_tasks_size = 256; + if (bthread::execution_queue_start(&_queue_id, &execq_opt, KVBasedMergedMetaStorageImpl::run, this) != 0) { - status.set_error(EINVAL, "Fail to start execution_queue, path: %s", + status.set_error(EINVAL, "Fail to start execution_queue, path: %s", _path.c_str()); return status; - } + } _is_inited = true; return status; } - -void KVBasedMergedMetaStorageImpl::run_tasks(leveldb::WriteBatch& updates, - Closure* dones[], size_t size) { - g_save_kv_raft_meta_batch_counter << size; - +void KVBasedMergedMetaStorageImpl::run_tasks(leveldb::WriteBatch& updates, + Closure* dones[], size_t size) { + g_save_kv_raft_meta_batch_counter << size; + leveldb::WriteOptions options; - options.sync = raft_sync_meta(); + options.sync = raft_sync_meta(); leveldb::Status st = _db->Write(options, &updates); if (!st.ok()) { LOG(ERROR) << "Fail to write batch into db, path: " << _path << ", error: " << st.ToString(); butil::Status status; - status.set_error(EIO, "MergedMetaStorage failed to write batch" - ", path: %s, error: %s", - _path.c_str(), st.ToString().c_str()); + status.set_error(EIO, + "MergedMetaStorage failed to write batch" + ", path: %s, error: %s", + _path.c_str(), st.ToString().c_str()); for (size_t i = 0; i < size; ++i) { - dones[i]->status() = status; + dones[i]->status() = status; run_closure_in_bthread_nosig(dones[i]); } } else { for (size_t i = 0; i < size; ++i) { run_closure_in_bthread_nosig(dones[i]); - } + } } bthread_flush(); } -int KVBasedMergedMetaStorageImpl::run(void* meta, - bthread::TaskIterator& iter) { +int KVBasedMergedMetaStorageImpl::run(void* meta, + bthread::TaskIterator& iter) { if (iter.is_queue_stopped()) { return 0; } @@ -715,7 +738,7 @@ int KVBasedMergedMetaStorageImpl::run(void* meta, for (; iter; ++iter) { if (cur_size == batch_size) { - mss->run_tasks(updates, dones, cur_size); + mss->run_tasks(updates, dones, cur_size); updates.Clear(); cur_size = 0; } @@ -723,9 +746,9 @@ int KVBasedMergedMetaStorageImpl::run(void* meta, const int64_t term = iter->term; const PeerId votedfor = iter->votedfor; const VersionedGroupId vgid = iter->vgid; - Closure* done = iter->done; - // get key and value - leveldb::Slice key(vgid.data(), vgid.size()); + Closure* done = iter->done; + // get key and value + leveldb::Slice key(vgid.data(), vgid.size()); StablePBMeta meta; meta.set_term(term); meta.set_votedfor(votedfor.to_string()); @@ -745,14 +768,16 @@ int KVBasedMergedMetaStorageImpl::run(void* meta, } void KVBasedMergedMetaStorageImpl::set_term_and_votedfor( - const int64_t term, const PeerId& peer_id, - const VersionedGroupId& group, Closure* done) { + const int64_t term, const PeerId& peer_id, const VersionedGroupId& group, + Closure* done) { if (!_is_inited) { - done->status().set_error(EINVAL, "MergedMetaStorage of group %s not" - " init, path: %s", group.c_str(), _path.c_str()); + done->status().set_error(EINVAL, + "MergedMetaStorage of group %s not" + " init, path: %s", + group.c_str(), _path.c_str()); return run_closure_in_bthread(done); } - + WriteTask task; task.term = term; task.votedfor = peer_id; @@ -764,15 +789,16 @@ void KVBasedMergedMetaStorageImpl::set_term_and_votedfor( } } -butil::Status KVBasedMergedMetaStorageImpl::get_term_and_votedfor(int64_t* term, - PeerId* peer_id, const VersionedGroupId& group) { +butil::Status KVBasedMergedMetaStorageImpl::get_term_and_votedfor( + int64_t* term, PeerId* peer_id, const VersionedGroupId& group) { butil::Status status; if (!_is_inited) { - status.set_error(EINVAL, "MergedMetaStorage of group %s not init, path: %s", + status.set_error(EINVAL, + "MergedMetaStorage of group %s not init, path: %s", group.c_str(), _path.c_str()); return status; } - + butil::Timer timer; timer.start(); leveldb::Slice key(group.data(), group.size()); @@ -788,79 +814,82 @@ butil::Status KVBasedMergedMetaStorageImpl::get_term_and_votedfor(int64_t* term, status = done.status(); if (!status.ok()) { LOG(ERROR) << "node " << group - << " failed to set initial term and votedfor when first time init" - << ", path " << _path - << ", error " << status.error_cstr(); + << " failed to set initial term and votedfor when first " + "time init" + << ", path " << _path << ", error " + << status.error_cstr(); return status; } - LOG(INFO) << "node " << group - << " succeed to set initial term and votedfor when first time init" - << ", path " << _path; + LOG(INFO) + << "node " << group + << " succeed to set initial term and votedfor when first time init" + << ", path " << _path; return status; } else if (!st.ok()) { - LOG(ERROR) << "node " << group - << " failed to get value from db, path " << _path - << ", error " << st.ToString().c_str(); - status.set_error(EIO, "MergedMetaStorage of group %s failed to" - "get value from db, path: %s, error: %s", + LOG(ERROR) << "node " << group << " failed to get value from db, path " + << _path << ", error " << st.ToString().c_str(); + status.set_error(EIO, + "MergedMetaStorage of group %s failed to" + "get value from db, path: %s, error: %s", group.c_str(), _path.c_str(), st.ToString().c_str()); return status; } - + // TODO replace pb StablePBMeta meta; meta.ParseFromString(value); *term = meta.term(); if (0 != peer_id->parse(meta.votedfor())) { - LOG(ERROR) << "node " << group - << " failed to parse votedfor when loading meta from db, path " - << _path; - status.set_error(EINVAL, "MergedMetaStorage of group %s failed to" - " parse votedfor when loading meta from db, path: %s", + LOG(ERROR) + << "node " << group + << " failed to parse votedfor when loading meta from db, path " + << _path; + status.set_error(EINVAL, + "MergedMetaStorage of group %s failed to" + " parse votedfor when loading meta from db, path: %s", group.c_str(), _path.c_str()); return status; } timer.stop(); - g_load_kv_raft_meta << timer.u_elapsed(); - LOG(INFO) << "Loaded merged stable meta, path " << _path - << " group " << group - << " term " << *term - << " votedfor " << *peer_id + g_load_kv_raft_meta << timer.u_elapsed(); + LOG(INFO) << "Loaded merged stable meta, path " << _path << " group " + << group << " term " << *term << " votedfor " << *peer_id << " time: " << timer.u_elapsed(); return status; } butil::Status KVBasedMergedMetaStorageImpl::delete_meta( - const VersionedGroupId& group) { + const VersionedGroupId& group) { butil::Status status; if (!_is_inited) { - status.set_error(EINVAL, "MergedMetaStorage of group %s not init, path: %s", + status.set_error(EINVAL, + "MergedMetaStorage of group %s not init, path: %s", group.c_str(), _path.c_str()); return status; } - + butil::Timer timer; timer.start(); leveldb::WriteOptions options; options.sync = raft_sync_meta(); - - leveldb::Slice key(group.data(), group.size()); + + leveldb::Slice key(group.data(), group.size()); leveldb::Status st = _db->Delete(options, key); if (!st.ok()) { LOG(ERROR) << "Fail to delete meta info from db, group " << group; - status.set_error(EIO, "MergedMetaStorage failed to delete group %s" + status.set_error(EIO, + "MergedMetaStorage failed to delete group %s" ", path: %s, error: %s", group.c_str(), _path.c_str(), st.ToString().c_str()); return status; } timer.stop(); - g_delete_kv_raft_meta << timer.u_elapsed(); - LOG(INFO) << "Deleted merged stable meta, path " << _path - << " group " << group - << " time: " << timer.u_elapsed(); + g_delete_kv_raft_meta << timer.u_elapsed(); + LOG(INFO) << "Deleted merged stable meta, path " << _path << " group " + << group << " time: " << timer.u_elapsed(); return status; } -} +} // namespace braft diff --git a/src/braft/raft_meta.h b/src/braft/raft_meta.h index 2d4200a..4988f13 100644 --- a/src/braft/raft_meta.h +++ b/src/braft/raft_meta.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,10 +18,11 @@ #ifndef BRAFT_RAFT_META_H #define BRAFT_RAFT_META_H +#include #include #include #include -#include + #include "braft/storage.h" namespace braft { @@ -29,8 +30,8 @@ namespace braft { class FileBasedSingleMetaStorage; class KVBasedMergedMetaStorageImpl; -class MixedMetaStorage : public RaftMetaStorage { -public: +class MixedMetaStorage : public RaftMetaStorage { + public: explicit MixedMetaStorage(const std::string& path); MixedMetaStorage() {} virtual ~MixedMetaStorage(); @@ -39,37 +40,38 @@ class MixedMetaStorage : public RaftMetaStorage { virtual butil::Status init(); // set term and votedfor information - virtual butil::Status set_term_and_votedfor(const int64_t term, - const PeerId& peer_id, const VersionedGroupId& group); + virtual butil::Status set_term_and_votedfor(const int64_t term, + const PeerId& peer_id, + const VersionedGroupId& group); // get term and votedfor information - virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, + virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, const VersionedGroupId& group); RaftMetaStorage* new_instance(const std::string& uri) const; - - butil::Status gc_instance(const std::string& uri, + + butil::Status gc_instance(const std::string& uri, const VersionedGroupId& vgid) const; - + bool is_bad() { return _is_bad; } - -private: - static int parse_mixed_path(const std::string& uri, std::string& merged_path, - std::string& single_path); + private: + static int parse_mixed_path(const std::string& uri, + std::string& merged_path, + std::string& single_path); bool _is_inited; bool _is_bad; std::string _path; - // Origin stable storage for each raft node + // Origin stable storage for each raft node FileBasedSingleMetaStorage* _single_impl; // Merged stable storage for raft nodes on the same disk scoped_refptr _merged_impl; }; // Manage meta info of ONLY ONE raft instance -class FileBasedSingleMetaStorage : public RaftMetaStorage { -public: +class FileBasedSingleMetaStorage : public RaftMetaStorage { + public: explicit FileBasedSingleMetaStorage(const std::string& path) : _is_inited(false), _path(path), _term(1) {} FileBasedSingleMetaStorage() {} @@ -77,13 +79,14 @@ class FileBasedSingleMetaStorage : public RaftMetaStorage { // init stable storage virtual butil::Status init(); - + // set term and votedfor information - virtual butil::Status set_term_and_votedfor(const int64_t term, const PeerId& peer_id, - const VersionedGroupId& group); - + virtual butil::Status set_term_and_votedfor(const int64_t term, + const PeerId& peer_id, + const VersionedGroupId& group); + // get term and votedfor information - virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, + virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, const VersionedGroupId& group); RaftMetaStorage* new_instance(const std::string& uri) const; @@ -91,7 +94,7 @@ class FileBasedSingleMetaStorage : public RaftMetaStorage { butil::Status gc_instance(const std::string& uri, const VersionedGroupId& vgid) const; -private: + private: static const char* _s_raft_meta; int load(); int save(); @@ -102,10 +105,10 @@ class FileBasedSingleMetaStorage : public RaftMetaStorage { PeerId _votedfor; }; -// Manage meta info of A BATCH of raft instances who share the same disk_path prefix -class KVBasedMergedMetaStorage : public RaftMetaStorage { - -public: +// Manage meta info of A BATCH of raft instances who share the same disk_path +// prefix +class KVBasedMergedMetaStorage : public RaftMetaStorage { + public: explicit KVBasedMergedMetaStorage(const std::string& path); KVBasedMergedMetaStorage() {} @@ -113,35 +116,34 @@ class KVBasedMergedMetaStorage : public RaftMetaStorage { // init stable storage virtual butil::Status init(); - + // set term and votedfor information - virtual butil::Status set_term_and_votedfor(const int64_t term, - const PeerId& peer_id, + virtual butil::Status set_term_and_votedfor(const int64_t term, + const PeerId& peer_id, const VersionedGroupId& group); // get term and votedfor information - virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, + virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, const VersionedGroupId& group); RaftMetaStorage* new_instance(const std::string& uri) const; - + butil::Status gc_instance(const std::string& uri, const VersionedGroupId& vgid) const; - + // GC meta info of a raft instance indicated by |group| virtual butil::Status delete_meta(const VersionedGroupId& group); -private: - + private: scoped_refptr _merged_impl; }; // Inner class of KVBasedMergedMetaStorage -class KVBasedMergedMetaStorageImpl : - public butil::RefCountedThreadSafe { -friend class scoped_refptr; +class KVBasedMergedMetaStorageImpl + : public butil::RefCountedThreadSafe { + friend class scoped_refptr; -public: + public: explicit KVBasedMergedMetaStorageImpl(const std::string& path) : _is_inited(false), _path(path), _db(nullptr) {} KVBasedMergedMetaStorageImpl() {} @@ -151,9 +153,9 @@ friend class scoped_refptr; _db = nullptr; } } - + struct WriteTask { - //TaskType type; + // TaskType type; int64_t term; PeerId votedfor; VersionedGroupId vgid; @@ -162,25 +164,27 @@ friend class scoped_refptr; // init stable storage virtual butil::Status init(); - + // set term and votedfor information - virtual void set_term_and_votedfor(const int64_t term, const PeerId& peer_id, - const VersionedGroupId& group, Closure* done); - + virtual void set_term_and_votedfor(const int64_t term, + const PeerId& peer_id, + const VersionedGroupId& group, + Closure* done); + // get term and votedfor information // [NOTICE] If some new instance init stable storage for the first time, // no record would be found from db, in which case initial term and votedfor // will be set. // Initial term: 1 Initial votedfor: ANY_PEER - virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, + virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, const VersionedGroupId& group); // GC meta info of a raft instance indicated by |group| virtual butil::Status delete_meta(const VersionedGroupId& group); -private: + private: friend class butil::RefCountedThreadSafe; - + static int run(void* meta, bthread::TaskIterator& iter); void run_tasks(leveldb::WriteBatch& updates, Closure* dones[], size_t size); @@ -193,6 +197,6 @@ friend class scoped_refptr; leveldb::DB* _db; }; -} +} // namespace braft -#endif //~BRAFT_RAFT_META_H +#endif //~BRAFT_RAFT_META_H diff --git a/src/braft/raft_service.cpp b/src/braft/raft_service.cpp index 0bdd6bf..e2ff9ea 100644 --- a/src/braft/raft_service.cpp +++ b/src/braft/raft_service.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,12 +15,14 @@ // Authors: Wang,Yao(wangyao02@baidu.com) // Zhangyi Chen(chenzhangyi01@baidu.com) -#include -#include #include "braft/raft_service.h" -#include "braft/raft.h" + +#include +#include + #include "braft/node.h" #include "braft/node_manager.h" +#include "braft/raft.h" namespace braft { @@ -29,12 +31,11 @@ RaftServiceImpl::~RaftServiceImpl() { } void RaftServiceImpl::pre_vote(google::protobuf::RpcController* cntl_base, - const RequestVoteRequest* request, - RequestVoteResponse* response, - google::protobuf::Closure* done) { + const RequestVoteRequest* request, + RequestVoteResponse* response, + google::protobuf::Closure* done) { brpc::ClosureGuard done_guard(done); - brpc::Controller* cntl = - static_cast(cntl_base); + brpc::Controller* cntl = static_cast(cntl_base); PeerId peer_id; if (0 != peer_id.parse(request->peer_id())) { @@ -42,8 +43,8 @@ void RaftServiceImpl::pre_vote(google::protobuf::RpcController* cntl_base, return; } - scoped_refptr node_ptr = - global_node_manager->get(request->group_id(), peer_id); + scoped_refptr node_ptr = + global_node_manager->get(request->group_id(), peer_id); NodeImpl* node = node_ptr.get(); if (!node) { cntl->SetFailed(ENOENT, "peer_id not exist"); @@ -59,12 +60,11 @@ void RaftServiceImpl::pre_vote(google::protobuf::RpcController* cntl_base, } void RaftServiceImpl::request_vote(google::protobuf::RpcController* cntl_base, - const RequestVoteRequest* request, - RequestVoteResponse* response, - google::protobuf::Closure* done) { + const RequestVoteRequest* request, + RequestVoteResponse* response, + google::protobuf::Closure* done) { brpc::ClosureGuard done_guard(done); - brpc::Controller* cntl = - static_cast(cntl_base); + brpc::Controller* cntl = static_cast(cntl_base); PeerId peer_id; if (0 != peer_id.parse(request->peer_id())) { @@ -72,8 +72,8 @@ void RaftServiceImpl::request_vote(google::protobuf::RpcController* cntl_base, return; } - scoped_refptr node_ptr = - global_node_manager->get(request->group_id(), peer_id); + scoped_refptr node_ptr = + global_node_manager->get(request->group_id(), peer_id); NodeImpl* node = node_ptr.get(); if (!node) { cntl->SetFailed(ENOENT, "peer_id not exist"); @@ -88,12 +88,11 @@ void RaftServiceImpl::request_vote(google::protobuf::RpcController* cntl_base, } void RaftServiceImpl::append_entries(google::protobuf::RpcController* cntl_base, - const AppendEntriesRequest* request, - AppendEntriesResponse* response, - google::protobuf::Closure* done) { + const AppendEntriesRequest* request, + AppendEntriesResponse* response, + google::protobuf::Closure* done) { brpc::ClosureGuard done_guard(done); - brpc::Controller* cntl = - static_cast(cntl_base); + brpc::Controller* cntl = static_cast(cntl_base); PeerId peer_id; if (0 != peer_id.parse(request->peer_id())) { @@ -101,24 +100,23 @@ void RaftServiceImpl::append_entries(google::protobuf::RpcController* cntl_base, return; } - scoped_refptr node_ptr = - global_node_manager->get(request->group_id(), peer_id); + scoped_refptr node_ptr = + global_node_manager->get(request->group_id(), peer_id); NodeImpl* node = node_ptr.get(); if (!node) { cntl->SetFailed(ENOENT, "peer_id not exist"); return; } - return node->handle_append_entries_request(cntl, request, response, + return node->handle_append_entries_request(cntl, request, response, done_guard.release()); } -void RaftServiceImpl::install_snapshot(google::protobuf::RpcController* cntl_base, - const InstallSnapshotRequest* request, - InstallSnapshotResponse* response, - google::protobuf::Closure* done) { - brpc::Controller* cntl = - static_cast(cntl_base); +void RaftServiceImpl::install_snapshot( + google::protobuf::RpcController* cntl_base, + const InstallSnapshotRequest* request, InstallSnapshotResponse* response, + google::protobuf::Closure* done) { + brpc::Controller* cntl = static_cast(cntl_base); PeerId peer_id; if (0 != peer_id.parse(request->peer_id())) { @@ -127,8 +125,8 @@ void RaftServiceImpl::install_snapshot(google::protobuf::RpcController* cntl_bas return; } - scoped_refptr node_ptr = - global_node_manager->get(request->group_id(), peer_id); + scoped_refptr node_ptr = + global_node_manager->get(request->group_id(), peer_id); NodeImpl* node = node_ptr.get(); if (!node) { cntl->SetFailed(ENOENT, "peer_id not exist"); @@ -144,8 +142,7 @@ void RaftServiceImpl::timeout_now(::google::protobuf::RpcController* controller, const ::braft::TimeoutNowRequest* request, ::braft::TimeoutNowResponse* response, ::google::protobuf::Closure* done) { - brpc::Controller* cntl = - static_cast(controller); + brpc::Controller* cntl = static_cast(controller); PeerId peer_id; if (0 != peer_id.parse(request->peer_id())) { @@ -154,8 +151,8 @@ void RaftServiceImpl::timeout_now(::google::protobuf::RpcController* controller, return; } - scoped_refptr node_ptr = - global_node_manager->get(request->group_id(), peer_id); + scoped_refptr node_ptr = + global_node_manager->get(request->group_id(), peer_id); NodeImpl* node = node_ptr.get(); if (!node) { cntl->SetFailed(ENOENT, "peer_id not exist"); @@ -166,4 +163,4 @@ void RaftServiceImpl::timeout_now(::google::protobuf::RpcController* controller, node->handle_timeout_now_request(cntl, request, response, done); } -} +} // namespace braft diff --git a/src/braft/raft_service.h b/src/braft/raft_service.h index 954f923..be60b00 100644 --- a/src/braft/raft_service.h +++ b/src/braft/raft_service.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,42 +18,43 @@ #define BRAFT_RAFT_SERVICE_H #include "braft/raft.pb.h" +#include "butil/endpoint.h" namespace braft { class RaftServiceImpl : public RaftService { -public: - explicit RaftServiceImpl(butil::EndPoint addr) - : _addr(addr) {} + public: + explicit RaftServiceImpl(butil::EndPoint addr) : _addr(addr) {} ~RaftServiceImpl(); void pre_vote(google::protobuf::RpcController* controller, - const RequestVoteRequest* request, - RequestVoteResponse* response, - google::protobuf::Closure* done); + const RequestVoteRequest* request, + RequestVoteResponse* response, + google::protobuf::Closure* done); void request_vote(google::protobuf::RpcController* controller, - const RequestVoteRequest* request, - RequestVoteResponse* response, - google::protobuf::Closure* done); + const RequestVoteRequest* request, + RequestVoteResponse* response, + google::protobuf::Closure* done); void append_entries(google::protobuf::RpcController* controller, - const AppendEntriesRequest* request, - AppendEntriesResponse* response, - google::protobuf::Closure* done); + const AppendEntriesRequest* request, + AppendEntriesResponse* response, + google::protobuf::Closure* done); void install_snapshot(google::protobuf::RpcController* controller, - const InstallSnapshotRequest* request, - InstallSnapshotResponse* response, - google::protobuf::Closure* done); + const InstallSnapshotRequest* request, + InstallSnapshotResponse* response, + google::protobuf::Closure* done); void timeout_now(::google::protobuf::RpcController* controller, const ::braft::TimeoutNowRequest* request, ::braft::TimeoutNowResponse* response, ::google::protobuf::Closure* done); -private: + + private: butil::EndPoint _addr; }; -} +} // namespace braft -#endif //~BRAFT_RAFT_SERVICE_H +#endif //~BRAFT_RAFT_SERVICE_H diff --git a/src/braft/remote_file_copier.cpp b/src/braft/remote_file_copier.cpp index 534e7bf..b365abd 100644 --- a/src/braft/remote_file_copier.cpp +++ b/src/braft/remote_file_copier.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,15 +18,16 @@ #include "braft/remote_file_copier.h" -#include -#include -#include -#include -#include -#include #include -#include "braft/util.h" +#include +#include +#include +#include +#include +#include + #include "braft/snapshot.h" +#include "braft/util.h" namespace braft { @@ -37,20 +38,18 @@ DEFINE_bool(raft_allow_read_partly_when_install_snapshot, true, "Whether allowing read snapshot data partly"); BRPC_VALIDATE_GFLAG(raft_allow_read_partly_when_install_snapshot, ::brpc::PassValidate); -DEFINE_bool(raft_enable_throttle_when_install_snapshot, true, - "enable throttle when install snapshot, for both leader and follower"); +DEFINE_bool( + raft_enable_throttle_when_install_snapshot, true, + "enable throttle when install snapshot, for both leader and follower"); BRPC_VALIDATE_GFLAG(raft_enable_throttle_when_install_snapshot, ::brpc::PassValidate); DECLARE_int32(raft_rpc_channel_connect_timeout_ms); -RemoteFileCopier::RemoteFileCopier() - : _reader_id(0) - , _throttle(NULL) -{} +RemoteFileCopier::RemoteFileCopier() : _reader_id(0), _throttle(NULL) {} -int RemoteFileCopier::init(const std::string& uri, FileSystemAdaptor* fs, - SnapshotThrottle* throttle) { +int RemoteFileCopier::init(const std::string& uri, FileSystemAdaptor* fs, + SnapshotThrottle* throttle) { // Parse uri format: remote://ip:port/reader_id static const size_t prefix_size = strlen("remote://"); butil::StringPiece uri_str(uri); @@ -63,8 +62,7 @@ int RemoteFileCopier::init(const std::string& uri, FileSystemAdaptor* fs, butil::StringPiece ip_and_port = uri_str.substr(0, slash_pos); uri_str.remove_prefix(slash_pos + 1); if (!butil::StringToInt64(uri_str, &_reader_id)) { - LOG(ERROR) << "Invalid reader_id_format=" << uri_str - << " in " << uri; + LOG(ERROR) << "Invalid reader_id_format=" << uri_str << " in " << uri; return -1; } brpc::ChannelOptions channel_opt; @@ -78,13 +76,10 @@ int RemoteFileCopier::init(const std::string& uri, FileSystemAdaptor* fs, return 0; } -int RemoteFileCopier::read_piece_of_file( - butil::IOBuf* buf, - const std::string& source, - off_t offset, - size_t max_count, - long timeout_ms, - bool* is_eof) { +int RemoteFileCopier::read_piece_of_file(butil::IOBuf* buf, + const std::string& source, + off_t offset, size_t max_count, + long timeout_ms, bool* is_eof) { brpc::Controller cntl; GetFileRequest request; request.set_reader_id(_reader_id); @@ -107,8 +102,8 @@ int RemoteFileCopier::read_piece_of_file( int RemoteFileCopier::copy_to_file(const std::string& source, const std::string& dest_path, const CopyOptions* options) { - scoped_refptr session = start_to_copy_to_file( - source, dest_path, options); + scoped_refptr session = + start_to_copy_to_file(source, dest_path, options); if (session == NULL) { return -1; } @@ -117,10 +112,10 @@ int RemoteFileCopier::copy_to_file(const std::string& source, } int RemoteFileCopier::copy_to_iobuf(const std::string& source, - butil::IOBuf* dest_buf, + butil::IOBuf* dest_buf, const CopyOptions* options) { - scoped_refptr session = start_to_copy_to_iobuf( - source, dest_buf, options); + scoped_refptr session = + start_to_copy_to_iobuf(source, dest_buf, options); if (session == NULL) { return -1; } @@ -128,17 +123,17 @@ int RemoteFileCopier::copy_to_iobuf(const std::string& source, return session->status().error_code(); } -scoped_refptr -RemoteFileCopier::start_to_copy_to_file( - const std::string& source, - const std::string& dest_path, - const CopyOptions* options) { +scoped_refptr +RemoteFileCopier::start_to_copy_to_file(const std::string& source, + const std::string& dest_path, + const CopyOptions* options) { butil::File::Error e; - FileAdaptor* file = _fs->open(dest_path, O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, NULL, &e); - + FileAdaptor* file = _fs->open( + dest_path, O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, NULL, &e); + if (!file) { - LOG(ERROR) << "Fail to open " << dest_path - << ", " << butil::File::ErrorToString(e); + LOG(ERROR) << "Fail to open " << dest_path << ", " + << butil::File::ErrorToString(e); return NULL; } @@ -159,11 +154,10 @@ RemoteFileCopier::start_to_copy_to_file( return session; } -scoped_refptr -RemoteFileCopier::start_to_copy_to_iobuf( - const std::string& source, - butil::IOBuf* dest_buf, - const CopyOptions* options) { +scoped_refptr +RemoteFileCopier::start_to_copy_to_iobuf(const std::string& source, + butil::IOBuf* dest_buf, + const CopyOptions* options) { dest_buf->clear(); scoped_refptr session(new Session()); session->_file = NULL; @@ -178,16 +172,15 @@ RemoteFileCopier::start_to_copy_to_iobuf( return session; } -RemoteFileCopier::Session::Session() - : _channel(NULL) - , _file(NULL) - , _retry_times(0) - , _finished(false) - , _buf(NULL) - , _timer() - , _throttle(NULL) - , _throttle_token_acquire_time_us(1) -{ +RemoteFileCopier::Session::Session() + : _channel(NULL), + _file(NULL), + _retry_times(0), + _finished(false), + _buf(NULL), + _timer(), + _throttle(NULL), + _throttle_token_acquire_time_us(1) { _done.owner = this; } @@ -204,12 +197,13 @@ void RemoteFileCopier::Session::send_next_rpc() { _response.Clear(); // Not clear request as we need some fields of the previous RPC off_t offset = _request.offset() + _request.count(); - const size_t max_count = - (!_buf) ? FLAGS_raft_max_byte_count_per_rpc : UINT_MAX; + const size_t max_count = + (!_buf) ? FLAGS_raft_max_byte_count_per_rpc : UINT_MAX; _cntl.set_timeout_ms(_options.timeout_ms); _request.set_offset(offset); // Read partly when throttled - _request.set_read_partly(FLAGS_raft_allow_read_partly_when_install_snapshot); + _request.set_read_partly( + FLAGS_raft_allow_read_partly_when_install_snapshot); std::unique_lock lck(_mutex); if (_finished) { return; @@ -224,12 +218,12 @@ void RemoteFileCopier::Session::send_next_rpc() { BRAFT_VLOG << "Copy file throttled, path: " << _dest_path; _request.set_count(0); AddRef(); - int64_t retry_interval_ms_when_throttled = - _throttle->get_retry_interval_ms(); - if (bthread_timer_add( - &_timer, - butil::milliseconds_from_now(retry_interval_ms_when_throttled), - on_timer, this) != 0) { + int64_t retry_interval_ms_when_throttled = + _throttle->get_retry_interval_ms(); + if (bthread_timer_add(&_timer, + butil::milliseconds_from_now( + retry_interval_ms_when_throttled), + on_timer, this) != 0) { lck.unlock(); LOG(ERROR) << "Fail to add timer"; return on_timer(this); @@ -263,28 +257,29 @@ void RemoteFileCopier::Session::on_rpc_returned() { } } // Throttled reading failure does not increase _retry_times - if (_cntl.ErrorCode() != EAGAIN && _retry_times++ >= _options.max_retry) { + if (_cntl.ErrorCode() != EAGAIN && + _retry_times++ >= _options.max_retry) { if (_st.ok()) { _st.set_error(_cntl.ErrorCode(), _cntl.ErrorText()); return on_finished(); } } // set retry time interval - int64_t retry_interval_ms = _options.retry_interval_ms; + int64_t retry_interval_ms = _options.retry_interval_ms; if (_cntl.ErrorCode() == EAGAIN && _throttle) { retry_interval_ms = _throttle->get_retry_interval_ms(); - // No token consumed, just return back, other nodes maybe able to use them + // No token consumed, just return back, other nodes maybe able to + // use them if (FLAGS_raft_enable_throttle_when_install_snapshot) { _throttle->return_unused_throughput( - request_count, 0, - butil::cpuwide_time_us() - _throttle_token_acquire_time_us); + request_count, 0, + butil::cpuwide_time_us() - _throttle_token_acquire_time_us); } } AddRef(); - if (bthread_timer_add( - &_timer, - butil::milliseconds_from_now(retry_interval_ms), - on_timer, this) != 0) { + if (bthread_timer_add(&_timer, + butil::milliseconds_from_now(retry_interval_ms), + on_timer, this) != 0) { lck.unlock(); LOG(ERROR) << "Fail to add timer"; return on_timer(this); @@ -294,13 +289,13 @@ void RemoteFileCopier::Session::on_rpc_returned() { if (_throttle && FLAGS_raft_enable_throttle_when_install_snapshot && _request.count() > (int64_t)_cntl.response_attachment().size()) { _throttle->return_unused_throughput( - _request.count(), _cntl.response_attachment().size(), - butil::cpuwide_time_us() - _throttle_token_acquire_time_us); + _request.count(), _cntl.response_attachment().size(), + butil::cpuwide_time_us() - _throttle_token_acquire_time_us); } _retry_times = 0; // Reset count to |real_read_size| to make next rpc get the right offset - if (_response.has_read_size() && (_response.read_size() != 0) - && FLAGS_raft_allow_read_partly_when_install_snapshot) { + if (_response.has_read_size() && (_response.read_size() != 0) && + FLAGS_raft_allow_read_partly_when_install_snapshot) { _request.set_count(_response.read_size()); } if (_file) { @@ -343,8 +338,8 @@ void* RemoteFileCopier::Session::send_next_rpc_on_timedout(void* arg) { void RemoteFileCopier::Session::on_timer(void* arg) { bthread_t tid; - if (bthread_start_background( - &tid, NULL, send_next_rpc_on_timedout, arg) != 0) { + if (bthread_start_background(&tid, NULL, send_next_rpc_on_timedout, arg) != + 0) { PLOG(ERROR) << "Fail to start bthread"; send_next_rpc_on_timedout(arg); } @@ -367,7 +362,7 @@ void RemoteFileCopier::Session::on_finished() { void RemoteFileCopier::Session::cancel() { BAIDU_SCOPED_LOCK(_mutex); if (_finished) { - return; + return; } brpc::StartCancel(_rpc_call); if (bthread_timer_del(_timer) == 0) { @@ -380,8 +375,6 @@ void RemoteFileCopier::Session::cancel() { on_finished(); } -void RemoteFileCopier::Session::join() { - _finish_event.wait(); -} +void RemoteFileCopier::Session::join() { _finish_event.wait(); } -} // namespace braft +} // namespace braft diff --git a/src/braft/remote_file_copier.h b/src/braft/remote_file_copier.h index a15bf71..a270b3f 100644 --- a/src/braft/remote_file_copier.h +++ b/src/braft/remote_file_copier.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,14 +16,15 @@ // Zheng,Pengfei(zhengpengfei@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#ifndef BRAFT_REMOTE_FILE_COPIER_H -#define BRAFT_REMOTE_FILE_COPIER_H +#ifndef BRAFT_REMOTE_FILE_COPIER_H +#define BRAFT_REMOTE_FILE_COPIER_H #include #include + #include "braft/file_service.pb.h" -#include "braft/util.h" #include "braft/snapshot_throttle.h" +#include "braft/util.h" namespace braft { @@ -37,9 +38,10 @@ struct CopyOptions { }; inline CopyOptions::CopyOptions() - : max_retry(3) - , retry_interval_ms(1000) // 1s - , timeout_ms(10L * 1000) // 10s + : max_retry(3), + retry_interval_ms(1000) // 1s + , + timeout_ms(10L * 1000) // 10s {} class FileAdaptor; @@ -47,10 +49,10 @@ class FileSystemAdaptor; class LocalSnapshotWriter; class RemoteFileCopier { -public: + public: // Stands for a copying session class Session : public butil::RefCountedThreadSafe { - public: + public: Session(); ~Session(); // Cancel the copy process @@ -59,13 +61,12 @@ class RemoteFileCopier { void join(); const butil::Status& status() const { return _st; } - private: - friend class RemoteFileCopier; - friend class Closure; + + private: + friend class RemoteFileCopier; + friend class Closure; struct Closure : google::protobuf::Closure { - void Run() { - owner->on_rpc_returned(); - } + void Run() { owner->on_rpc_returned(); } Session* owner; }; void on_rpc_returned(); @@ -90,33 +91,30 @@ class RemoteFileCopier { GetFileRequest _request; GetFileResponse _response; bthread::CountdownEvent _finish_event; - scoped_refptr _throttle; + scoped_refptr _throttle; int64_t _throttle_token_acquire_time_us; }; RemoteFileCopier(); - int init(const std::string& uri, FileSystemAdaptor* fs, - SnapshotThrottle* throttle); + int init(const std::string& uri, FileSystemAdaptor* fs, + SnapshotThrottle* throttle); // Copy `source' from remote to dest - int copy_to_file(const std::string& source, - const std::string& dest_path, + int copy_to_file(const std::string& source, const std::string& dest_path, const CopyOptions* options); - int copy_to_iobuf(const std::string& source, - butil::IOBuf* dest_buf, + int copy_to_iobuf(const std::string& source, butil::IOBuf* dest_buf, const CopyOptions* options); - scoped_refptr start_to_copy_to_file( - const std::string& source, - const std::string& dest_path, - const CopyOptions* options); - scoped_refptr start_to_copy_to_iobuf( - const std::string& source, - butil::IOBuf* dest_buf, - const CopyOptions* options); -private: + scoped_refptr start_to_copy_to_file(const std::string& source, + const std::string& dest_path, + const CopyOptions* options); + scoped_refptr start_to_copy_to_iobuf(const std::string& source, + butil::IOBuf* dest_buf, + const CopyOptions* options); + + private: int read_piece_of_file(butil::IOBuf* buf, const std::string& source, - off_t offset, size_t max_count, - long timeout_ms, bool* is_eof); + off_t offset, size_t max_count, long timeout_ms, + bool* is_eof); DISALLOW_COPY_AND_ASSIGN(RemoteFileCopier); brpc::Channel _channel; int64_t _reader_id; @@ -126,4 +124,4 @@ class RemoteFileCopier { } // namespace braft -#endif //BRAFT_REMOTE_FILE_COPIER_H +#endif // BRAFT_REMOTE_FILE_COPIER_H diff --git a/src/braft/repeated_timer_task.cpp b/src/braft/repeated_timer_task.cpp index c72e279..7b764b0 100644 --- a/src/braft/repeated_timer_task.cpp +++ b/src/braft/repeated_timer_task.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,20 +16,19 @@ // Ma,Jingwei(majingwei@baidu.com) #include "braft/repeated_timer_task.h" + #include "braft/util.h" namespace braft { RepeatedTimerTask::RepeatedTimerTask() - : _timeout_ms(0) - , _stopped(true) - , _running(false) - , _destroyed(true) - , _invoking(false) -{} - -RepeatedTimerTask::~RepeatedTimerTask() -{ + : _timeout_ms(0), + _stopped(true), + _running(false), + _destroyed(true), + _invoking(false) {} + +RepeatedTimerTask::~RepeatedTimerTask() { CHECK(!_running) << "Is still running"; CHECK(_destroyed) << "destroy() must be invoked before descrution"; } @@ -91,9 +90,9 @@ void RepeatedTimerTask::start() { _stopped = false; BRAFT_RETURN_IF(_running); - // ^^^ _timer was not successfully deleted and the former task - // is still running, in which case on_timedout would invoke - // schedule as it would not see _stopped + // ^^^ _timer was not successfully deleted and the former task + // is still running, in which case on_timedout would invoke + // schedule as it would not see _stopped _running = true; schedule(lck); } @@ -117,8 +116,8 @@ void RepeatedTimerTask::on_timedout(void* arg) { // as run() might access the disk so the time it takes is probably beyond // expection bthread_t tid; - if (bthread_start_background( - &tid, NULL, run_on_timedout_in_new_thread, arg) != 0) { + if (bthread_start_background(&tid, NULL, run_on_timedout_in_new_thread, + arg) != 0) { PLOG(ERROR) << "Fail to start bthread"; run_on_timedout_in_new_thread(arg); } @@ -126,7 +125,7 @@ void RepeatedTimerTask::on_timedout(void* arg) { void RepeatedTimerTask::schedule(std::unique_lock& lck) { _next_duetime = - butil::milliseconds_from_now(adjust_timeout_ms(_timeout_ms)); + butil::milliseconds_from_now(adjust_timeout_ms(_timeout_ms)); if (bthread_timer_add(&_timer, _next_duetime, on_timedout, this) != 0) { lck.unlock(); LOG(ERROR) << "Fail to add timer"; @@ -201,11 +200,11 @@ void RepeatedTimerTask::describe(std::ostream& os, bool use_html) { os << " INVOKING"; } else { os << " SCHEDULING(in " - << butil::timespec_to_milliseconds(duetime) - butil::gettimeofday_ms() + << butil::timespec_to_milliseconds(duetime) - + butil::gettimeofday_ms() << "ms)"; } } } } // namespace braft - diff --git a/src/braft/repeated_timer_task.h b/src/braft/repeated_timer_task.h index d541e30..1b072c7 100644 --- a/src/braft/repeated_timer_task.h +++ b/src/braft/repeated_timer_task.h @@ -1,11 +1,11 @@ // Copyright (c) 2016 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,18 +15,20 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) // Ma,Jingwei(majingwei@baidu.com) -#ifndef BRAFT_REPEATED_TIMER_TASK_H -#define BRAFT_REPEATED_TIMER_TASK_H +#ifndef BRAFT_REPEATED_TIMER_TASK_H +#define BRAFT_REPEATED_TIMER_TASK_H #include + #include "braft/macros.h" namespace braft { // Repeated scheduled timer task -class RepeatedTimerTask{ -DISALLOW_COPY_AND_ASSIGN(RepeatedTimerTask); -public: +class RepeatedTimerTask { + DISALLOW_COPY_AND_ASSIGN(RepeatedTimerTask); + + public: RepeatedTimerTask(); virtual ~RepeatedTimerTask(); // Initialize timer task @@ -53,20 +55,16 @@ DISALLOW_COPY_AND_ASSIGN(RepeatedTimerTask); // Describe the current status of timer void describe(std::ostream& os, bool use_html); -protected: - + protected: // Invoked everytime when it reaches the timeout virtual void run() = 0; // Invoked when the timer is finally destroyed virtual void on_destroy() = 0; - virtual int adjust_timeout_ms(int timeout_ms) { - return timeout_ms; - } - -private: + virtual int adjust_timeout_ms(int timeout_ms) { return timeout_ms; } + private: static void on_timedout(void* arg); static void* run_on_timedout_in_new_thread(void* arg); void on_timedout(); @@ -75,7 +73,7 @@ DISALLOW_COPY_AND_ASSIGN(RepeatedTimerTask); raft_mutex_t _mutex; bthread_timer_t _timer; timespec _next_duetime; - int _timeout_ms; + int _timeout_ms; bool _stopped; bool _running; bool _destroyed; @@ -84,4 +82,4 @@ DISALLOW_COPY_AND_ASSIGN(RepeatedTimerTask); } // namespace braft -#endif //BRAFT_REPEATED_TIMER_TASK_H +#endif // BRAFT_REPEATED_TIMER_TASK_H diff --git a/src/braft/replicator.cpp b/src/braft/replicator.cpp index d639312..64bc1d0 100644 --- a/src/braft/replicator.cpp +++ b/src/braft/replicator.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,16 +16,18 @@ // Wang,Yao(wangyao02@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#include // DEFINE_int32 -#include // std::unique_ptr -#include // butil::gettimeofday_us -#include // brpc::Controller -#include // BRPC_VALIDATE_GFLAG #include "braft/replicator.h" -#include "braft/node.h" // NodeImpl -#include "braft/ballot_box.h" // BallotBox -#include "braft/log_entry.h" // LogEntry -#include "braft/snapshot_throttle.h" // SnapshotThrottle + +#include // brpc::Controller +#include // BRPC_VALIDATE_GFLAG +#include // butil::gettimeofday_us +#include // std::unique_ptr +#include // DEFINE_int32 + +#include "braft/ballot_box.h" // BallotBox +#include "braft/log_entry.h" // LogEntry +#include "braft/node.h" // NodeImpl +#include "braft/snapshot_throttle.h" // SnapshotThrottle namespace braft { @@ -44,8 +46,7 @@ BRPC_VALIDATE_GFLAG(raft_max_body_size, ::brpc::PositiveInteger); DEFINE_int32(raft_retry_replicate_interval_ms, 1000, "Interval of retry to append entries or install snapshot"); -BRPC_VALIDATE_GFLAG(raft_retry_replicate_interval_ms, - brpc::PositiveInteger); +BRPC_VALIDATE_GFLAG(raft_retry_replicate_interval_ms, brpc::PositiveInteger); DECLARE_bool(raft_enable_witness_to_leader); DECLARE_int64(raft_append_entry_high_lat_us); @@ -55,36 +56,33 @@ DECLARE_int32(raft_rpc_channel_connect_timeout_ms); static bvar::LatencyRecorder g_send_entries_latency("raft_send_entries"); static bvar::LatencyRecorder g_normalized_send_entries_latency( - "raft_send_entries_normalized"); + "raft_send_entries_normalized"); static bvar::CounterRecorder g_send_entries_batch_counter( - "raft_send_entries_batch_counter"); + "raft_send_entries_batch_counter"); ReplicatorOptions::ReplicatorOptions() - : dynamic_heartbeat_timeout_ms(NULL) - , log_manager(NULL) - , ballot_box(NULL) - , node(NULL) - , term(0) - , snapshot_storage(NULL) - , replicator_status(NULL) -{ -} - -Replicator::Replicator() - : _next_index(0) - , _flying_append_entries_size(0) - , _consecutive_error_times(0) - , _has_succeeded(false) - , _timeout_now_index(0) - , _heartbeat_counter(0) - , _append_entries_counter(0) - , _install_snapshot_counter(0) - , _readonly_index(0) - , _wait_id(0) - , _is_waiter_canceled(false) - , _reader(NULL) - , _catchup_closure(NULL) -{ + : dynamic_heartbeat_timeout_ms(NULL), + log_manager(NULL), + ballot_box(NULL), + node(NULL), + term(0), + snapshot_storage(NULL), + replicator_status(NULL) {} + +Replicator::Replicator() + : _next_index(0), + _flying_append_entries_size(0), + _consecutive_error_times(0), + _has_succeeded(false), + _timeout_now_index(0), + _heartbeat_counter(0), + _append_entries_counter(0), + _install_snapshot_counter(0), + _readonly_index(0), + _wait_id(0), + _is_waiter_canceled(false), + _reader(NULL), + _catchup_closure(NULL) { _install_snapshot_in_fly.value = 0; _heartbeat_in_fly.value = 0; _timeout_now_in_fly.value = 0; @@ -106,16 +104,16 @@ Replicator::~Replicator() { } } -int Replicator::start(const ReplicatorOptions& options, ReplicatorId *id) { - if (options.log_manager == NULL || options.ballot_box == NULL - || options.node == NULL) { +int Replicator::start(const ReplicatorOptions& options, ReplicatorId* id) { + if (options.log_manager == NULL || options.ballot_box == NULL || + options.node == NULL) { LOG(ERROR) << "Invalid arguments, group " << options.group_id; return -1; } Replicator* r = new Replicator(); brpc::ChannelOptions channel_opt; channel_opt.connect_timeout_ms = FLAGS_raft_rpc_channel_connect_timeout_ms; - channel_opt.timeout_ms = -1; // We don't need RPC timeout + channel_opt.timeout_ms = -1; // We don't need RPC timeout if (r->_sending_channel.Init(options.peer_id.addr, &channel_opt) != 0) { LOG(ERROR) << "Fail to init sending channel" << ", group " << options.group_id; @@ -136,12 +134,12 @@ int Replicator::start(const ReplicatorOptions& options, ReplicatorId *id) { return -1; } - bthread_id_lock(r->_id, NULL); if (id) { *id = r->_id.value; } - LOG(INFO) << "Replicator=" << r->_id << "@" << r->_options.peer_id << " is started" + LOG(INFO) << "Replicator=" << r->_id << "@" << r->_options.peer_id + << " is started" << ", group " << r->_options.group_id; r->_catchup_closure = NULL; r->_update_last_rpc_send_timestamp(butil::monotonic_time_ms()); @@ -152,7 +150,7 @@ int Replicator::start(const ReplicatorOptions& options, ReplicatorId *id) { } int Replicator::stop(ReplicatorId id) { - bthread_id_t dummy_id = { id }; + bthread_id_t dummy_id = {id}; Replicator* r = NULL; // already stopped if (bthread_id_lock(dummy_id, (void**)&r) != 0) { @@ -165,15 +163,14 @@ int Replicator::stop(ReplicatorId id) { } int Replicator::join(ReplicatorId id) { - bthread_id_t dummy_id = { id }; + bthread_id_t dummy_id = {id}; return bthread_id_join(dummy_id); } -void Replicator::wait_for_caught_up(ReplicatorId id, - int64_t max_margin, +void Replicator::wait_for_caught_up(ReplicatorId id, int64_t max_margin, const timespec* due_time, CatchupClosure* done) { - bthread_id_t dummy_id = { id }; + bthread_id_t dummy_id = {id}; Replicator* r = NULL; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { done->status().set_error(EINVAL, "No such replicator"); @@ -181,8 +178,8 @@ void Replicator::wait_for_caught_up(ReplicatorId id, return; } if (r->_catchup_closure != NULL) { - CHECK_EQ(0, bthread_id_unlock(dummy_id)) - << "Fail to unlock " << dummy_id; + CHECK_EQ(0, bthread_id_unlock(dummy_id)) + << "Fail to unlock " << dummy_id; LOG(ERROR) << "Previous wait_for_caught_up is not over" << ", group " << r->_options.group_id; done->status().set_error(EINVAL, "Duplicated call"); @@ -195,14 +192,12 @@ void Replicator::wait_for_caught_up(ReplicatorId id, << ", group " << r->_options.group_id; run_closure_in_bthread(done); CHECK_EQ(0, bthread_id_unlock(dummy_id)) - << "Fail to unlock" << dummy_id; + << "Fail to unlock" << dummy_id; return; } if (due_time != NULL) { done->_has_timer = true; - if (bthread_timer_add(&done->_timer, - *due_time, - _on_catch_up_timedout, + if (bthread_timer_add(&done->_timer, *due_time, _on_catch_up_timedout, (void*)id) != 0) { CHECK_EQ(0, bthread_id_unlock(dummy_id)); LOG(ERROR) << "Fail to add timer"; @@ -213,14 +208,13 @@ void Replicator::wait_for_caught_up(ReplicatorId id, } r->_catchup_closure = done; // success - CHECK_EQ(0, bthread_id_unlock(dummy_id)) - << "Fail to unlock " << dummy_id; + CHECK_EQ(0, bthread_id_unlock(dummy_id)) << "Fail to unlock " << dummy_id; return; } void* Replicator::_on_block_timedout_in_new_thread(void* arg) { Replicator* r = NULL; - bthread_id_t id = { (uint64_t)arg }; + bthread_id_t id = {(uint64_t)arg}; if (bthread_id_lock(id, (void**)&r) != 0) { return NULL; } @@ -230,17 +224,17 @@ void* Replicator::_on_block_timedout_in_new_thread(void* arg) { return NULL; } -void Replicator::_on_block_timedout(void *arg) { +void Replicator::_on_block_timedout(void* arg) { bthread_t tid; - if (bthread_start_background( - &tid, NULL, _on_block_timedout_in_new_thread, arg) != 0) { + if (bthread_start_background(&tid, NULL, _on_block_timedout_in_new_thread, + arg) != 0) { PLOG(ERROR) << "Fail to start bthread"; _on_block_timedout_in_new_thread(arg); } } void Replicator::_block(long start_time_us, int error_code) { - // mainly for pipeline case, to avoid too many block timer when this + // mainly for pipeline case, to avoid too many block timer when this // replicator is something wrong if (_st.st == BLOCKING) { CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; @@ -250,8 +244,8 @@ void Replicator::_block(long start_time_us, int error_code) { // TODO: Currently we don't care about error_code which indicates why the // very RPC fails. To make it better there should be different timeout for // each individual error (e.g. we don't need check every - // heartbeat_timeout_ms whether a dead follower has come back), but it's just - // fine now. + // heartbeat_timeout_ms whether a dead follower has come back), but it's + // just fine now. int blocking_time = 0; if (error_code == EBUSY || error_code == EINTR) { blocking_time = FLAGS_raft_retry_replicate_interval_ms; @@ -259,13 +253,14 @@ void Replicator::_block(long start_time_us, int error_code) { blocking_time = *_options.dynamic_heartbeat_timeout_ms; } const timespec due_time = butil::milliseconds_from( - butil::microseconds_to_timespec(start_time_us), blocking_time); + butil::microseconds_to_timespec(start_time_us), blocking_time); bthread_timer_t timer; - const int rc = bthread_timer_add(&timer, due_time, - _on_block_timedout, (void*)_id.value); + const int rc = bthread_timer_add(&timer, due_time, _on_block_timedout, + (void*)_id.value); if (rc == 0) { - BRAFT_VLOG << "Blocking " << _options.peer_id << " for " - << blocking_time << "ms" << ", group " << _options.group_id; + BRAFT_VLOG << "Blocking " << _options.peer_id << " for " + << blocking_time << "ms" + << ", group " << _options.group_id; _st.st = BLOCKING; CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; return; @@ -276,48 +271,48 @@ void Replicator::_block(long start_time_us, int error_code) { } } -void Replicator::_on_heartbeat_returned( - ReplicatorId id, brpc::Controller* cntl, - AppendEntriesRequest* request, - AppendEntriesResponse* response, - int64_t rpc_send_time) { +void Replicator::_on_heartbeat_returned(ReplicatorId id, brpc::Controller* cntl, + AppendEntriesRequest* request, + AppendEntriesResponse* response, + int64_t rpc_send_time) { std::unique_ptr cntl_guard(cntl); - std::unique_ptr req_guard(request); + std::unique_ptr req_guard(request); std::unique_ptr res_guard(response); - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; const long start_time_us = butil::gettimeofday_us(); if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return; } std::stringstream ss; - ss << "node " << r->_options.group_id << ":" << r->_options.server_id - << " received HeartbeatResponse from " - << r->_options.peer_id << " prev_log_index " << request->prev_log_index() - << " prev_log_term " << request->prev_log_term(); - + ss << "node " << r->_options.group_id << ":" << r->_options.server_id + << " received HeartbeatResponse from " << r->_options.peer_id + << " prev_log_index " << request->prev_log_index() << " prev_log_term " + << request->prev_log_term(); + if (cntl->Failed()) { ss << " fail, sleep."; BRAFT_VLOG << ss.str(); // TODO: Should it be VLOG? LOG_IF(WARNING, (r->_consecutive_error_times++) % 10 == 0) - << "Group " << r->_options.group_id - << " fail to issue RPC to " << r->_options.peer_id - << " _consecutive_error_times=" << r->_consecutive_error_times - << ", " << cntl->ErrorText(); + << "Group " << r->_options.group_id << " fail to issue RPC to " + << r->_options.peer_id + << " _consecutive_error_times=" << r->_consecutive_error_times + << ", " << cntl->ErrorText(); r->_start_heartbeat_timer(start_time_us); - CHECK_EQ(0, bthread_id_unlock(dummy_id)) << "Fail to unlock " << dummy_id; + CHECK_EQ(0, bthread_id_unlock(dummy_id)) + << "Fail to unlock " << dummy_id; return; } r->_consecutive_error_times = 0; if (response->term() > r->_options.term) { - ss << " fail, greater term " << response->term() - << " expect term " << r->_options.term; + ss << " fail, greater term " << response->term() << " expect term " + << r->_options.term; BRAFT_VLOG << ss.str(); - NodeImpl *node_impl = r->_options.node; + NodeImpl* node_impl = r->_options.node; // Acquire a reference of Node here in case that Node is detroyed // after _notify_on_caught_up. node_impl->AddRef(); @@ -325,8 +320,10 @@ void Replicator::_on_heartbeat_returned( LOG(INFO) << "Replicator=" << dummy_id << " is going to quit" << ", group " << r->_options.group_id; butil::Status status; - status.set_error(EHIGHERTERMRESPONSE, "Leader receives higher term " - "heartbeat_response from peer:%s", r->_options.peer_id.to_string().c_str()); + status.set_error(EHIGHERTERMRESPONSE, + "Leader receives higher term " + "heartbeat_response from peer:%s", + r->_options.peer_id.to_string().c_str()); r->_destroy(); node_impl->increase_term_to(response->term(), status); node_impl->Release(); @@ -345,7 +342,8 @@ void Replicator::_on_heartbeat_returned( node_impl->AddRef(); } if (!node_impl) { - CHECK_EQ(0, bthread_id_unlock(dummy_id)) << "Fail to unlock " << dummy_id; + CHECK_EQ(0, bthread_id_unlock(dummy_id)) + << "Fail to unlock " << dummy_id; return; } const PeerId peer_id = r->_options.peer_id; @@ -357,32 +355,33 @@ void Replicator::_on_heartbeat_returned( } void Replicator::_on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, - AppendEntriesRequest* request, - AppendEntriesResponse* response, - int64_t rpc_send_time) { + AppendEntriesRequest* request, + AppendEntriesResponse* response, + int64_t rpc_send_time) { std::unique_ptr cntl_guard(cntl); - std::unique_ptr req_guard(request); + std::unique_ptr req_guard(request); std::unique_ptr res_guard(response); - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; const long start_time_us = butil::gettimeofday_us(); if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return; } std::stringstream ss; - ss << "node " << r->_options.group_id << ":" << r->_options.server_id - << " received AppendEntriesResponse from " - << r->_options.peer_id << " prev_log_index " << request->prev_log_index() - << " prev_log_term " << request->prev_log_term() << " count " << request->entries_size(); + ss << "node " << r->_options.group_id << ":" << r->_options.server_id + << " received AppendEntriesResponse from " << r->_options.peer_id + << " prev_log_index " << request->prev_log_index() << " prev_log_term " + << request->prev_log_term() << " count " << request->entries_size(); bool valid_rpc = false; int64_t rpc_first_index = request->prev_log_index() + 1; int64_t min_flying_index = r->_min_flying_index(); CHECK_GT(min_flying_index, 0); - for (std::deque::iterator rpc_it = r->_append_entries_in_fly.begin(); - rpc_it != r->_append_entries_in_fly.end(); ++rpc_it) { + for (std::deque::iterator rpc_it = + r->_append_entries_in_fly.begin(); + rpc_it != r->_append_entries_in_fly.end(); ++rpc_it) { if (rpc_it->log_index > rpc_first_index) { break; } @@ -403,10 +402,10 @@ void Replicator::_on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, // TODO: Should it be VLOG? LOG_IF(WARNING, (r->_consecutive_error_times++) % 10 == 0) - << "Group " << r->_options.group_id - << " fail to issue RPC to " << r->_options.peer_id - << " _consecutive_error_times=" << r->_consecutive_error_times - << ", " << cntl->ErrorText(); + << "Group " << r->_options.group_id << " fail to issue RPC to " + << r->_options.peer_id + << " _consecutive_error_times=" << r->_consecutive_error_times + << ", " << cntl->ErrorText(); // If the follower crashes, any RPC to the follower fails immediately, // so we need to block the follower for a while instead of looping until // it comes back or be removed @@ -421,29 +420,33 @@ void Replicator::_on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, << " expect term " << r->_options.term; r->_reset_next_index(); - NodeImpl *node_impl = r->_options.node; + NodeImpl* node_impl = r->_options.node; // Acquire a reference of Node here in case that Node is destroyed // after _notify_on_caught_up. node_impl->AddRef(); r->_notify_on_caught_up(EPERM, true); butil::Status status; - status.set_error(EHIGHERTERMRESPONSE, "Leader receives higher term " - "%s from peer:%s", response->GetTypeName().c_str(), r->_options.peer_id.to_string().c_str()); + status.set_error(EHIGHERTERMRESPONSE, + "Leader receives higher term " + "%s from peer:%s", + response->GetTypeName().c_str(), + r->_options.peer_id.to_string().c_str()); r->_destroy(); node_impl->increase_term_to(response->term(), status); node_impl->Release(); return; } - ss << " fail, find next_index remote last_log_index " << response->last_log_index() - << " local next_index " << r->_next_index - << " rpc prev_log_index " << request->prev_log_index(); + ss << " fail, find next_index remote last_log_index " + << response->last_log_index() << " local next_index " + << r->_next_index << " rpc prev_log_index " + << request->prev_log_index(); BRAFT_VLOG << ss.str(); r->_update_last_rpc_send_timestamp(rpc_send_time); // prev_log_index and prev_log_term doesn't match r->_reset_next_index(); if (response->last_log_index() + 1 < r->_next_index) { BRAFT_VLOG << "Group " << r->_options.group_id - << " last_log_index at peer=" << r->_options.peer_id + << " last_log_index at peer=" << r->_options.peer_id << " is " << response->last_log_index(); // The peer contains less logs than leader r->_next_index = response->last_log_index() + 1; @@ -453,12 +456,11 @@ void Replicator::_on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, if (BAIDU_LIKELY(r->_next_index > 1)) { BRAFT_VLOG << "Group " << r->_options.group_id << " log_index=" << r->_next_index << " mismatch"; - // Decrease the |_next_index| when the request with the minimum log - // index mismatch. - // Because when we enable pipeline replication and disable cache, - // the request with larger log index would be handled before the - // smaller request, we should ignore it. - // See https://github.com/baidu/braft/issues/421 + // Decrease the |_next_index| when the request with the minimum + // log index mismatch. Because when we enable pipeline + // replication and disable cache, the request with larger log + // index would be handled before the smaller request, we should + // ignore it. See https://github.com/baidu/braft/issues/421 if (request->prev_log_index() == r->_next_index - 1) { --r->_next_index; } @@ -476,7 +478,7 @@ void Replicator::_on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, ss << " success"; BRAFT_VLOG << ss.str(); - + if (response->term() != r->_options.term) { LOG(ERROR) << "Group " << r->_options.group_id << " fail, response term " << response->term() @@ -488,54 +490,54 @@ void Replicator::_on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, r->_update_last_rpc_send_timestamp(rpc_send_time); const int entries_size = request->entries_size(); const int64_t rpc_last_log_index = request->prev_log_index() + entries_size; - BRAFT_VLOG_IF(entries_size > 0) << "Group " << r->_options.group_id - << " replicated logs in [" - << min_flying_index << ", " - << rpc_last_log_index - << "] to peer " << r->_options.peer_id; + BRAFT_VLOG_IF(entries_size > 0) + << "Group " << r->_options.group_id << " replicated logs in [" + << min_flying_index << ", " << rpc_last_log_index << "] to peer " + << r->_options.peer_id; if (entries_size > 0) { - r->_options.ballot_box->commit_at( - min_flying_index, rpc_last_log_index, - r->_options.peer_id); + r->_options.ballot_box->commit_at(min_flying_index, rpc_last_log_index, + r->_options.peer_id); int64_t rpc_latency_us = cntl->latency_us(); - if (FLAGS_raft_trace_append_entry_latency && + if (FLAGS_raft_trace_append_entry_latency && rpc_latency_us > FLAGS_raft_append_entry_high_lat_us) { LOG(WARNING) << "append entry rpc latency us " << rpc_latency_us - << " greater than " - << FLAGS_raft_append_entry_high_lat_us - << " Group " << r->_options.group_id - << " to peer " << r->_options.peer_id - << " request entry size " << entries_size - << " request data size " - << cntl->request_attachment().size(); + << " greater than " + << FLAGS_raft_append_entry_high_lat_us << " Group " + << r->_options.group_id << " to peer " + << r->_options.peer_id << " request entry size " + << entries_size << " request data size " + << cntl->request_attachment().size(); } g_send_entries_latency << cntl->latency_us(); if (cntl->request_attachment().size() > 0) { - g_normalized_send_entries_latency << - cntl->latency_us() * 1024 / cntl->request_attachment().size(); + g_normalized_send_entries_latency + << cntl->latency_us() * 1024 / + cntl->request_attachment().size(); } } // A rpc is marked as success, means all request before it are success, // erase them sequentially. while (!r->_append_entries_in_fly.empty() && r->_append_entries_in_fly.front().log_index <= rpc_first_index) { - r->_flying_append_entries_size -= r->_append_entries_in_fly.front().entries_size; + r->_flying_append_entries_size -= + r->_append_entries_in_fly.front().entries_size; r->_append_entries_in_fly.pop_front(); } r->_has_succeeded = true; r->_notify_on_caught_up(0, false); // dummy_id is unlock in _send_entries - if (r->_timeout_now_index > 0 && r->_timeout_now_index < r->_min_flying_index()) { + if (r->_timeout_now_index > 0 && + r->_timeout_now_index < r->_min_flying_index()) { r->_send_timeout_now(false, false); } r->_send_entries(); return; } -int Replicator::_fill_common_fields(AppendEntriesRequest* request, - int64_t prev_log_index, - bool is_heartbeat) { - const int64_t prev_log_term = _options.log_manager->get_term(prev_log_index); +int Replicator::_fill_common_fields(AppendEntriesRequest* request, + int64_t prev_log_index, bool is_heartbeat) { + const int64_t prev_log_term = + _options.log_manager->get_term(prev_log_index); if (prev_log_term == 0 && prev_log_index != 0) { if (!is_heartbeat) { CHECK_LT(prev_log_index, _options.log_manager->first_log_index()); @@ -543,10 +545,10 @@ int Replicator::_fill_common_fields(AppendEntriesRequest* request, << " log_index=" << prev_log_index << " was compacted"; return -1; } else { - // The log at prev_log_index has been compacted, which indicates - // we are or are going to install snapshot to the follower. So we let - // both prev_log_index and prev_log_term be 0 in the heartbeat - // request so that follower would do nothing besides updating its + // The log at prev_log_index has been compacted, which indicates + // we are or are going to install snapshot to the follower. So we + // let both prev_log_index and prev_log_term be 0 in the heartbeat + // request so that follower would do nothing besides updating its // leader timestamp. prev_log_index = 0; } @@ -565,8 +567,8 @@ void Replicator::_send_empty_entries(bool is_heartbeat) { std::unique_ptr cntl(new brpc::Controller); std::unique_ptr request(new AppendEntriesRequest); std::unique_ptr response(new AppendEntriesResponse); - if (_fill_common_fields( - request.get(), _next_index - 1, is_heartbeat) != 0) { + if (_fill_common_fields(request.get(), _next_index - 1, is_heartbeat) != + 0) { CHECK(!is_heartbeat); // _id is unlock in _install_snapshot return _install_snapshot(); @@ -574,7 +576,8 @@ void Replicator::_send_empty_entries(bool is_heartbeat) { if (is_heartbeat) { _heartbeat_in_fly = cntl->call_id(); _heartbeat_counter++; - // set RPC timeout for heartbeat, how long should timeout be is waiting to be optimized. + // set RPC timeout for heartbeat, how long should timeout be is waiting + // to be optimized. cntl->set_timeout_ms(*_options.election_timeout_ms / 2); } else { _st.st = APPENDING_ENTRIES; @@ -582,40 +585,40 @@ void Replicator::_send_empty_entries(bool is_heartbeat) { _st.last_log_index = _next_index - 1; CHECK(_append_entries_in_fly.empty()); CHECK_EQ(_flying_append_entries_size, 0); - _append_entries_in_fly.push_back(FlyingAppendEntriesRpc(_next_index, 0, cntl->call_id())); + _append_entries_in_fly.push_back( + FlyingAppendEntriesRpc(_next_index, 0, cntl->call_id())); _append_entries_counter++; } BRAFT_VLOG << "node " << _options.group_id << ":" << _options.server_id - << " send HeartbeatRequest to " << _options.peer_id - << " term " << _options.term - << " prev_log_index " << request->prev_log_index() - << " last_committed_index " << request->committed_index(); + << " send HeartbeatRequest to " << _options.peer_id << " term " + << _options.term << " prev_log_index " + << request->prev_log_index() << " last_committed_index " + << request->committed_index(); google::protobuf::Closure* done = brpc::NewCallback( - is_heartbeat ? _on_heartbeat_returned : _on_rpc_returned, - _id.value, cntl.get(), request.get(), response.get(), - butil::monotonic_time_ms()); + is_heartbeat ? _on_heartbeat_returned : _on_rpc_returned, _id.value, + cntl.get(), request.get(), response.get(), butil::monotonic_time_ms()); RaftService_Stub stub(&_sending_channel); - stub.append_entries(cntl.release(), request.release(), - response.release(), done); + stub.append_entries(cntl.release(), request.release(), response.release(), + done); CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; } -int Replicator::_prepare_entry(int offset, EntryMeta* em, butil::IOBuf *data) { +int Replicator::_prepare_entry(int offset, EntryMeta* em, butil::IOBuf* data) { if (data->length() >= (size_t)FLAGS_raft_max_body_size) { return ERANGE; } const int64_t log_index = _next_index + offset; - LogEntry *entry = _options.log_manager->get_entry(log_index); + LogEntry* entry = _options.log_manager->get_entry(log_index); if (entry == NULL) { return ENOENT; } - // When leader become readonly, no new user logs can submit. On the other side, - // if any user log are accepted after this replicator become readonly, the leader - // still have enough followers to commit logs, we can safely stop waiting new logs - // until the replicator leave readonly mode. + // When leader become readonly, no new user logs can submit. On the other + // side, if any user log are accepted after this replicator become readonly, + // the leader still have enough followers to commit logs, we can safely stop + // waiting new logs until the replicator leave readonly mode. if (_readonly_index != 0 && log_index >= _readonly_index) { if (entry->type != ENTRY_TYPE_CONFIGURATION) { return EREADONLY; @@ -635,7 +638,8 @@ int Replicator::_prepare_entry(int offset, EntryMeta* em, butil::IOBuf *data) { } } } else { - CHECK(entry->type != ENTRY_TYPE_CONFIGURATION) << "log_index=" << log_index; + CHECK(entry->type != ENTRY_TYPE_CONFIGURATION) + << "log_index=" << log_index; } if (!is_witness() || FLAGS_raft_enable_witness_to_leader) { em->set_data_len(entry->data.length()); @@ -647,12 +651,15 @@ int Replicator::_prepare_entry(int offset, EntryMeta* em, butil::IOBuf *data) { void Replicator::_send_entries() { if (_flying_append_entries_size >= FLAGS_raft_max_entries_size || - _append_entries_in_fly.size() >= (size_t)FLAGS_raft_max_parallel_append_entries_rpc_num || + _append_entries_in_fly.size() >= + (size_t)FLAGS_raft_max_parallel_append_entries_rpc_num || _st.st == BLOCKING) { - BRAFT_VLOG << "node " << _options.group_id << ":" << _options.server_id + BRAFT_VLOG + << "node " << _options.group_id << ":" << _options.server_id << " skip sending AppendEntriesRequest to " << _options.peer_id << ", too many requests in flying, or the replicator is in block," - << " next_index " << _next_index << " flying_size " << _flying_append_entries_size; + << " next_index " << _next_index << " flying_size " + << _flying_append_entries_size; CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; return; } @@ -665,7 +672,8 @@ void Replicator::_send_entries() { return _install_snapshot(); } EntryMeta em; - const int max_entries_size = FLAGS_raft_max_entries_size - _flying_append_entries_size; + const int max_entries_size = + FLAGS_raft_max_entries_size - _flying_append_entries_size; int prepare_entry_rc = 0; CHECK_GT(max_entries_size, 0); for (int i = 0; i < max_entries_size; ++i) { @@ -682,8 +690,8 @@ void Replicator::_send_entries() { return _install_snapshot(); } // NOTICE: a follower's readonly mode does not prevent install_snapshot - // as we need followers to commit conf log(like add_node) when - // leader reaches readonly as well + // as we need followers to commit conf log(like add_node) when + // leader reaches readonly as well if (prepare_entry_rc == EREADONLY) { if (_flying_append_entries_size == 0) { _st.st = IDLE; @@ -694,78 +702,82 @@ void Replicator::_send_entries() { return _wait_more_entries(); } - _append_entries_in_fly.push_back(FlyingAppendEntriesRpc(_next_index, - request->entries_size(), cntl->call_id())); + _append_entries_in_fly.push_back(FlyingAppendEntriesRpc( + _next_index, request->entries_size(), cntl->call_id())); _append_entries_counter++; _next_index += request->entries_size(); _flying_append_entries_size += request->entries_size(); - + g_send_entries_batch_counter << request->entries_size(); BRAFT_VLOG << "node " << _options.group_id << ":" << _options.server_id - << " send AppendEntriesRequest to " << _options.peer_id << " term " << _options.term - << " last_committed_index " << request->committed_index() - << " prev_log_index " << request->prev_log_index() - << " prev_log_term " << request->prev_log_term() - << " next_index " << _next_index << " count " << request->entries_size(); + << " send AppendEntriesRequest to " << _options.peer_id + << " term " << _options.term << " last_committed_index " + << request->committed_index() << " prev_log_index " + << request->prev_log_index() << " prev_log_term " + << request->prev_log_term() << " next_index " << _next_index + << " count " << request->entries_size(); _st.st = APPENDING_ENTRIES; _st.first_log_index = _min_flying_index(); _st.last_log_index = _next_index - 1; google::protobuf::Closure* done = brpc::NewCallback( - _on_rpc_returned, _id.value, cntl.get(), - request.get(), response.get(), butil::monotonic_time_ms()); + _on_rpc_returned, _id.value, cntl.get(), request.get(), response.get(), + butil::monotonic_time_ms()); RaftService_Stub stub(&_sending_channel); - stub.append_entries(cntl.release(), request.release(), - response.release(), done); + stub.append_entries(cntl.release(), request.release(), response.release(), + done); _wait_more_entries(); } int Replicator::_continue_sending(void* arg, int error_code) { Replicator* r = NULL; - bthread_id_t id = { (uint64_t)arg }; + bthread_id_t id = {(uint64_t)arg}; if (bthread_id_lock(id, (void**)&r) != 0) { return -1; } if (error_code == ETIMEDOUT) { // Replication is in progress when block timeout, no need to start again - // this case can happen when - // 1. pipeline is enabled and + // this case can happen when + // 1. pipeline is enabled and // 2. disable readonly mode triggers another replication if (r->_wait_id != 0) { bthread_id_unlock(id); return 0; } - + // Send empty entries after block timeout to check the correct // _next_index otherwise the replictor is likely waits in - // _wait_more_entries and no further logs would be replicated even if the - // last_index of this followers is less than |next_index - 1| + // _wait_more_entries and no further logs would be replicated even if + // the last_index of this followers is less than |next_index - 1| r->_send_empty_entries(false); } else if (error_code != ESTOP && !r->_is_waiter_canceled) { // id is unlock in _send_entries r->_wait_id = 0; r->_send_entries(); } else if (r->_is_waiter_canceled) { - // The replicator is checking current next index by sending empty entries or - // install snapshot now. Although the registered waiter will be canceled - // before the operations, there is still a little chance that LogManger already - // waked up the waiter, and _continue_sending is waiting to execute. - BRAFT_VLOG << "Group " << r->_options.group_id - << " Replicator=" << id << " canceled waiter"; + // The replicator is checking current next index by sending empty + // entries or install snapshot now. Although the registered waiter will + // be canceled before the operations, there is still a little chance + // that LogManger already waked up the waiter, and _continue_sending is + // waiting to execute. + BRAFT_VLOG << "Group " << r->_options.group_id << " Replicator=" << id + << " canceled waiter"; bthread_id_unlock(id); } else { - LOG(WARNING) << "Group " << r->_options.group_id - << " Replicator=" << id << " stops sending entries"; + LOG(WARNING) << "Group " << r->_options.group_id << " Replicator=" << id + << " stops sending entries"; bthread_id_unlock(id); } return 0; } void Replicator::_wait_more_entries() { - if (_wait_id == 0 && FLAGS_raft_max_entries_size > _flying_append_entries_size && - (size_t)FLAGS_raft_max_parallel_append_entries_rpc_num > _append_entries_in_fly.size()) { + if (_wait_id == 0 && + FLAGS_raft_max_entries_size > _flying_append_entries_size && + (size_t)FLAGS_raft_max_parallel_append_entries_rpc_num > + _append_entries_in_fly.size()) { _wait_id = _options.log_manager->wait( - _next_index - 1, _continue_sending, (void*)_id.value); + _next_index - 1, _continue_sending, (void*)_id.value); _is_waiter_canceled = false; BRAFT_VLOG << "node " << _options.group_id << ":" << _options.peer_id << " wait more entries, wait_id " << _wait_id; @@ -777,29 +789,30 @@ void Replicator::_wait_more_entries() { } void Replicator::_install_snapshot() { - NodeImpl *node_impl = _options.node; + NodeImpl* node_impl = _options.node; if (node_impl->is_witness()) { return _block(butil::gettimeofday_us(), EBUSY); } if (_reader) { // follower's readonly mode change may cause two install_snapshot - // one possible case is: + // one possible case is: // enable -> install_snapshot -> disable -> wait_more_entries -> // install_snapshot again - LOG(WARNING) << "node " << _options.group_id << ":" << _options.server_id - << " refuse to send InstallSnapshotRequest to " << _options.peer_id - << " because there is an running one"; + LOG(WARNING) << "node " << _options.group_id << ":" + << _options.server_id + << " refuse to send InstallSnapshotRequest to " + << _options.peer_id << " because there is an running one"; CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; return; } - if (_options.snapshot_throttle && !_options.snapshot_throttle-> - add_one_more_task(true)) { + if (_options.snapshot_throttle && + !_options.snapshot_throttle->add_one_more_task(true)) { return _block(butil::gettimeofday_us(), EBUSY); } - + // pre-set replicator state to INSTALLING_SNAPSHOT, so replicator could be - // blocked if something is wrong, such as throttled for a period of time + // blocked if something is wrong, such as throttled for a period of time _st.st = INSTALLING_SNAPSHOT; _reader = _options.snapshot_storage->open(); @@ -807,7 +820,7 @@ void Replicator::_install_snapshot() { if (_options.snapshot_throttle) { _options.snapshot_throttle->finish_one_task(true); } - NodeImpl *node_impl = _options.node; + NodeImpl* node_impl = _options.node; node_impl->AddRef(); CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; braft::Error e; @@ -816,23 +829,24 @@ void Replicator::_install_snapshot() { node_impl->on_error(e); node_impl->Release(); return; - } + } std::string uri = _reader->generate_uri_for_copy(); // NOTICE: If uri is something wrong, retry later instead of reporting error - // immediately(making raft Node error), as FileSystemAdaptor layer of _reader is - // user defined and may need some control logic when opened + // immediately(making raft Node error), as FileSystemAdaptor layer of + // _reader is user defined and may need some control logic when opened if (uri.empty()) { - LOG(WARNING) << "node " << _options.group_id << ":" << _options.server_id - << " refuse to send InstallSnapshotRequest to " << _options.peer_id - << " because snapshot uri is empty"; + LOG(WARNING) << "node " << _options.group_id << ":" + << _options.server_id + << " refuse to send InstallSnapshotRequest to " + << _options.peer_id << " because snapshot uri is empty"; _close_reader(); - return _block(butil::gettimeofday_us(), EBUSY); + return _block(butil::gettimeofday_us(), EBUSY); } SnapshotMeta meta; // report error on failure if (_reader->load_meta(&meta) != 0) { std::string snapshot_path = _reader->get_path(); - NodeImpl *node_impl = _options.node; + NodeImpl* node_impl = _options.node; node_impl->AddRef(); CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; braft::Error e; @@ -841,7 +855,7 @@ void Replicator::_install_snapshot() { node_impl->on_error(e); node_impl->Release(); return; - } + } brpc::Controller* cntl = new brpc::Controller; cntl->set_max_retry(0); cntl->set_timeout_ms(-1); @@ -856,32 +870,31 @@ void Replicator::_install_snapshot() { LOG(INFO) << "node " << _options.group_id << ":" << _options.server_id << " send InstallSnapshotRequest to " << _options.peer_id - << " term " << _options.term << " last_included_term " << meta.last_included_term() - << " last_included_index " << meta.last_included_index() << " uri " << uri; + << " term " << _options.term << " last_included_term " + << meta.last_included_term() << " last_included_index " + << meta.last_included_index() << " uri " << uri; _install_snapshot_in_fly = cntl->call_id(); _install_snapshot_counter++; _st.last_log_included = meta.last_included_index(); _st.last_term_included = meta.last_included_term(); - google::protobuf::Closure* done = brpc::NewCallback< - ReplicatorId, brpc::Controller*, - InstallSnapshotRequest*, InstallSnapshotResponse*>( - _on_install_snapshot_returned, _id.value, - cntl, request, response); + google::protobuf::Closure* done = + brpc::NewCallback( + _on_install_snapshot_returned, _id.value, cntl, request, response); RaftService_Stub stub(&_sending_channel); stub.install_snapshot(cntl, request, response, done); CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; } void Replicator::_on_install_snapshot_returned( - ReplicatorId id, brpc::Controller* cntl, - InstallSnapshotRequest* request, - InstallSnapshotResponse* response) { + ReplicatorId id, brpc::Controller* cntl, InstallSnapshotRequest* request, + InstallSnapshotResponse* response) { std::unique_ptr cntl_guard(cntl); std::unique_ptr request_guard(request); std::unique_ptr response_guard(response); - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; bool succ = true; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return; @@ -894,20 +907,19 @@ void Replicator::_on_install_snapshot_returned( } } std::stringstream ss; - ss << "received InstallSnapshotResponse from " - << r->_options.group_id << ":" << r->_options.peer_id - << " last_included_index " << request->meta().last_included_index() - << " last_included_term " << request->meta().last_included_term(); + ss << "received InstallSnapshotResponse from " << r->_options.group_id + << ":" << r->_options.peer_id << " last_included_index " + << request->meta().last_included_index() << " last_included_term " + << request->meta().last_included_term(); do { if (cntl->Failed()) { ss << " error: " << cntl->ErrorText(); LOG(INFO) << ss.str(); - LOG_IF(WARNING, (r->_consecutive_error_times++) % 10 == 0) - << "Group " << r->_options.group_id - << " Fail to install snapshot at peer=" - << r->_options.peer_id - <<", " << cntl->ErrorText(); + LOG_IF(WARNING, (r->_consecutive_error_times++) % 10 == 0) + << "Group " << r->_options.group_id + << " Fail to install snapshot at peer=" << r->_options.peer_id + << ", " << cntl->ErrorText(); succ = false; break; } @@ -918,20 +930,21 @@ void Replicator::_on_install_snapshot_returned( // Let heartbeat do step down break; } - // Success + // Success r->_next_index = request->meta().last_included_index() + 1; ss << " success."; LOG(INFO) << ss.str(); } while (0); - // We don't retry installing the snapshot explicitly. + // We don't retry installing the snapshot explicitly. // dummy_id is unlock in _send_entries if (!succ) { return r->_block(butil::gettimeofday_us(), cntl->ErrorCode()); } r->_has_succeeded = true; r->_notify_on_caught_up(0, false); - if (r->_timeout_now_index > 0 && r->_timeout_now_index < r->_min_flying_index()) { + if (r->_timeout_now_index > 0 && + r->_timeout_now_index < r->_min_flying_index()) { r->_send_timeout_now(false, false); } // dummy_id is unlock in _send_entries @@ -951,18 +964,21 @@ void Replicator::_notify_on_caught_up(int error_code, bool before_destroy) { } _catchup_closure->_error_was_set = true; if (error_code) { - _catchup_closure->status().set_error(error_code, "%s", berror(error_code)); + _catchup_closure->status().set_error(error_code, "%s", + berror(error_code)); } if (_catchup_closure->_has_timer) { - if (!before_destroy && bthread_timer_del(_catchup_closure->_timer) == 1) { + if (!before_destroy && + bthread_timer_del(_catchup_closure->_timer) == 1) { // There's running timer task, let timer task trigger // on_caught_up to void ABA problem return; } } - } else { // Timed out or leader step_down + } else { // Timed out or leader step_down if (!_catchup_closure->_error_was_set) { - _catchup_closure->status().set_error(error_code, "%s", berror(error_code)); + _catchup_closure->status().set_error(error_code, "%s", + berror(error_code)); } } Closure* saved_catchup_closure = _catchup_closure; @@ -971,23 +987,23 @@ void Replicator::_notify_on_caught_up(int error_code, bool before_destroy) { } void Replicator::_on_timedout(void* arg) { - bthread_id_t id = { (uint64_t)arg }; + bthread_id_t id = {(uint64_t)arg}; bthread_id_error(id, ETIMEDOUT); } void Replicator::_start_heartbeat_timer(long start_time_us) { - const timespec due_time = butil::milliseconds_from( - butil::microseconds_to_timespec(start_time_us), - *_options.dynamic_heartbeat_timeout_ms); - if (bthread_timer_add(&_heartbeat_timer, due_time, - _on_timedout, (void*)_id.value) != 0) { + const timespec due_time = + butil::milliseconds_from(butil::microseconds_to_timespec(start_time_us), + *_options.dynamic_heartbeat_timeout_ms); + if (bthread_timer_add(&_heartbeat_timer, due_time, _on_timedout, + (void*)_id.value) != 0) { _on_timedout((void*)_id.value); } } void* Replicator::_send_heartbeat(void* arg) { Replicator* r = NULL; - bthread_id_t id = { (uint64_t)arg }; + bthread_id_t id = {(uint64_t)arg}; if (bthread_id_lock(id, (void**)&r) != 0) { // This replicator is stopped return NULL; @@ -1008,8 +1024,8 @@ int Replicator::_on_error(bthread_id_t id, void* arg, int error_code) { r->_options.log_manager->remove_waiter(r->_wait_id); r->_notify_on_caught_up(error_code, true); r->_wait_id = 0; - LOG(INFO) << "Group " << r->_options.group_id - << " Replicator=" << id << " is going to quit"; + LOG(INFO) << "Group " << r->_options.group_id << " Replicator=" << id + << " is going to quit"; r->_destroy(); return 0; } else if (error_code == ETIMEDOUT) { @@ -1025,7 +1041,7 @@ int Replicator::_on_error(bthread_id_t id, void* arg, int error_code) { } return 0; } else { - CHECK(false) << "Group " << r->_options.group_id + CHECK(false) << "Group " << r->_options.group_id << " Unknown error_code=" << error_code; CHECK_EQ(0, bthread_id_unlock(id)) << "Fail to unlock " << id; return -1; @@ -1033,20 +1049,19 @@ int Replicator::_on_error(bthread_id_t id, void* arg, int error_code) { } void Replicator::_on_catch_up_timedout(void* arg) { - bthread_id_t id = { (uint64_t)arg }; + bthread_id_t id = {(uint64_t)arg}; Replicator* r = NULL; if (bthread_id_lock(id, (void**)&r) != 0) { LOG(WARNING) << "Replicator is destroyed when catch_up_timedout."; return; } r->_notify_on_caught_up(ETIMEDOUT, false); - CHECK_EQ(0, bthread_id_unlock(id)) - << "Fail to unlock" << id; + CHECK_EQ(0, bthread_id_unlock(id)) << "Fail to unlock" << id; } int Replicator::transfer_leadership(ReplicatorId id, int64_t log_index) { Replicator* r = NULL; - bthread_id_t dummy = { id }; + bthread_id_t dummy = {id}; const int rc = bthread_id_lock(dummy, (void**)&r); if (rc != 0) { return rc; @@ -1057,7 +1072,7 @@ int Replicator::transfer_leadership(ReplicatorId id, int64_t log_index) { int Replicator::stop_transfer_leadership(ReplicatorId id) { Replicator* r = NULL; - bthread_id_t dummy = { id }; + bthread_id_t dummy = {id}; const int rc = bthread_id_lock(dummy, (void**)&r); if (rc != 0) { return rc; @@ -1083,8 +1098,8 @@ int Replicator::_transfer_leadership(int64_t log_index) { void Replicator::_cancel_append_entries_rpcs() { for (std::deque::iterator rpc_it = - _append_entries_in_fly.begin(); - rpc_it != _append_entries_in_fly.end(); ++rpc_it) { + _append_entries_in_fly.begin(); + rpc_it != _append_entries_in_fly.end(); ++rpc_it) { brpc::StartCancel(rpc_it->call_id); } _append_entries_in_fly.clear(); @@ -1121,33 +1136,32 @@ void Replicator::_send_timeout_now(bool unlock_id, bool old_leader_stepped_down, cntl->set_timeout_ms(timeout_ms); } RaftService_Stub stub(&_sending_channel); - ::google::protobuf::Closure* done = brpc::NewCallback( - _on_timeout_now_returned, _id.value, cntl, request, response, - old_leader_stepped_down); + ::google::protobuf::Closure* done = + brpc::NewCallback(_on_timeout_now_returned, _id.value, cntl, request, + response, old_leader_stepped_down); stub.timeout_now(cntl, request, response, done); if (unlock_id) { CHECK_EQ(0, bthread_id_unlock(_id)); } } -void Replicator::_on_timeout_now_returned( - ReplicatorId id, brpc::Controller* cntl, - TimeoutNowRequest* request, - TimeoutNowResponse* response, - bool old_leader_stepped_down) { +void Replicator::_on_timeout_now_returned(ReplicatorId id, + brpc::Controller* cntl, + TimeoutNowRequest* request, + TimeoutNowResponse* response, + bool old_leader_stepped_down) { std::unique_ptr cntl_guard(cntl); - std::unique_ptr req_guard(request); + std::unique_ptr req_guard(request); std::unique_ptr res_guard(response); - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return; } std::stringstream ss; - ss << "node " << r->_options.group_id << ":" << r->_options.server_id - << " received TimeoutNowResponse from " - << r->_options.peer_id; + ss << "node " << r->_options.group_id << ":" << r->_options.server_id + << " received TimeoutNowResponse from " << r->_options.peer_id; if (cntl->Failed()) { ss << " fail : " << cntl->ErrorText(); @@ -1165,14 +1179,16 @@ void Replicator::_on_timeout_now_returned( BRAFT_VLOG << ss.str(); if (response->term() > r->_options.term) { - NodeImpl *node_impl = r->_options.node; + NodeImpl* node_impl = r->_options.node; // Acquire a reference of Node here in case that Node is detroyed // after _notify_on_caught_up. node_impl->AddRef(); r->_notify_on_caught_up(EPERM, true); butil::Status status; - status.set_error(EHIGHERTERMRESPONSE, "Leader receives higher term " - "timeout_now_response from peer:%s", r->_options.peer_id.to_string().c_str()); + status.set_error(EHIGHERTERMRESPONSE, + "Leader receives higher term " + "timeout_now_response from peer:%s", + r->_options.peer_id.to_string().c_str()); r->_destroy(); node_impl->increase_term_to(response->term(), status); node_impl->Release(); @@ -1187,8 +1203,8 @@ void Replicator::_on_timeout_now_returned( } int Replicator::send_timeout_now_and_stop(ReplicatorId id, int timeout_ms) { - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return -1; } @@ -1198,8 +1214,8 @@ int Replicator::send_timeout_now_and_stop(ReplicatorId id, int timeout_ms) { } int64_t Replicator::get_next_index(ReplicatorId id) { - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return 0; } @@ -1212,8 +1228,8 @@ int64_t Replicator::get_next_index(ReplicatorId id) { } int Replicator::get_consecutive_error_times(ReplicatorId id) { - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return -1; } @@ -1223,8 +1239,8 @@ int Replicator::get_consecutive_error_times(ReplicatorId id) { } int Replicator::change_readonly_config(ReplicatorId id, bool readonly) { - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return 0; } @@ -1237,12 +1253,14 @@ int Replicator::_change_readonly_config(bool readonly) { // Check if readonly already set BRAFT_VLOG << "node " << _options.group_id << ":" << _options.server_id << " ignore change readonly config of " << _options.peer_id - << " to " << readonly << ", readonly_index " << _readonly_index; + << " to " << readonly << ", readonly_index " + << _readonly_index; CHECK_EQ(0, bthread_id_unlock(_id)) << "Fail to unlock " << _id; return 0; } if (readonly) { - // Keep a readonly index here to make sure the pending logs can be committed. + // Keep a readonly index here to make sure the pending logs can be + // committed. _readonly_index = _options.log_manager->last_log_index() + 1; LOG(INFO) << "node " << _options.group_id << ":" << _options.server_id << " enable readonly for " << _options.peer_id @@ -1258,8 +1276,8 @@ int Replicator::_change_readonly_config(bool readonly) { } bool Replicator::readonly(ReplicatorId id) { - Replicator *r = NULL; - bthread_id_t dummy_id = { id }; + Replicator* r = NULL; + bthread_id_t dummy_id = {id}; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return 0; } @@ -1297,24 +1315,26 @@ void Replicator::_describe(std::ostream& os, bool use_html) { os << " readonly_index=" << readonly_index << ' '; } switch (st.st) { - case IDLE: - os << "idle"; - break; - case BLOCKING: - os << "blocking"; - break; - case APPENDING_ENTRIES: - os << "appending [" << st.first_log_index << ", " << st.last_log_index << ']'; - break; - case INSTALLING_SNAPSHOT: - os << "installing snapshot {" << st.last_log_included - << ", " << st.last_term_included << '}'; - break; + case IDLE: + os << "idle"; + break; + case BLOCKING: + os << "blocking"; + break; + case APPENDING_ENTRIES: + os << "appending [" << st.first_log_index << ", " + << st.last_log_index << ']'; + break; + case INSTALLING_SNAPSHOT: + os << "installing snapshot {" << st.last_log_included << ", " + << st.last_term_included << '}'; + break; } if (consecutive_error_times != 0) { os << " consecutive_error_times=" << consecutive_error_times; } - os << " hc=" << heartbeat_counter << " ac=" << append_entries_counter << " ic=" << install_snapshot_counter << new_line; + os << " hc=" << heartbeat_counter << " ac=" << append_entries_counter + << " ic=" << install_snapshot_counter << new_line; } void Replicator::_get_status(PeerStatus* status) { @@ -1330,7 +1350,7 @@ void Replicator::_get_status(PeerStatus* status) { } void Replicator::describe(ReplicatorId id, std::ostream& os, bool use_html) { - bthread_id_t dummy_id = { id }; + bthread_id_t dummy_id = {id}; Replicator* r = NULL; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return; @@ -1343,7 +1363,7 @@ void Replicator::get_status(ReplicatorId id, PeerStatus* status) { if (!status) { return; } - bthread_id_t dummy_id = { id }; + bthread_id_t dummy_id = {id}; Replicator* r = NULL; if (bthread_id_lock(dummy_id, (void**)&r) != 0) { return; @@ -1364,27 +1384,23 @@ void Replicator::_close_reader() { // ==================== ReplicatorGroup ========================== ReplicatorGroupOptions::ReplicatorGroupOptions() - : heartbeat_timeout_ms(-1) - , election_timeout_ms(-1) - , log_manager(NULL) - , ballot_box(NULL) - , node(NULL) - , snapshot_storage(NULL) -{} - -ReplicatorGroup::ReplicatorGroup() - : _dynamic_timeout_ms(-1) - , _election_timeout_ms(-1) -{ + : heartbeat_timeout_ms(-1), + election_timeout_ms(-1), + log_manager(NULL), + ballot_box(NULL), + node(NULL), + snapshot_storage(NULL) {} + +ReplicatorGroup::ReplicatorGroup() + : _dynamic_timeout_ms(-1), _election_timeout_ms(-1) { _common_options.dynamic_heartbeat_timeout_ms = &_dynamic_timeout_ms; _common_options.election_timeout_ms = &_election_timeout_ms; } -ReplicatorGroup::~ReplicatorGroup() { - stop_all(); -} +ReplicatorGroup::~ReplicatorGroup() { stop_all(); } -int ReplicatorGroup::init(const NodeId& node_id, const ReplicatorGroupOptions& options) { +int ReplicatorGroup::init(const NodeId& node_id, + const ReplicatorGroupOptions& options) { _dynamic_timeout_ms = options.heartbeat_timeout_ms; _election_timeout_ms = options.election_timeout_ms; _common_options.log_manager = options.log_manager; @@ -1414,12 +1430,12 @@ int ReplicatorGroup::add_replicator(const PeerId& peer) { delete options.replicator_status; return -1; } - _rmap[peer] = { rid, options.replicator_status }; + _rmap[peer] = {rid, options.replicator_status}; return 0; } -int ReplicatorGroup::wait_caughtup(const PeerId& peer, - int64_t max_margin, const timespec* due_time, +int ReplicatorGroup::wait_caughtup(const PeerId& peer, int64_t max_margin, + const timespec* due_time, CatchupClosure* done) { std::map::iterator iter = _rmap.find(peer); if (iter == _rmap.end()) { @@ -1435,16 +1451,17 @@ int64_t ReplicatorGroup::last_rpc_send_timestamp(const PeerId& peer) { if (iter == _rmap.end()) { return 0; } - return iter->second.status->last_rpc_send_timestamp.load(butil::memory_order_relaxed); + return iter->second.status->last_rpc_send_timestamp.load( + butil::memory_order_relaxed); } -int ReplicatorGroup::stop_replicator(const PeerId &peer) { +int ReplicatorGroup::stop_replicator(const PeerId& peer) { std::map::iterator iter = _rmap.find(peer); if (iter == _rmap.end()) { return -1; } ReplicatorId rid = iter->second.id; - // Calling ReplicatorId::stop might lead to calling stop_replicator again, + // Calling ReplicatorId::stop might lead to calling stop_replicator again, // erase iter first to avoid race condition _rmap.erase(iter); return Replicator::stop(rid); @@ -1453,8 +1470,9 @@ int ReplicatorGroup::stop_replicator(const PeerId &peer) { int ReplicatorGroup::stop_all() { std::vector rids; rids.reserve(_rmap.size()); - for (std::map::const_iterator - iter = _rmap.begin(); iter != _rmap.end(); ++iter) { + for (std::map::const_iterator iter = + _rmap.begin(); + iter != _rmap.end(); ++iter) { rids.push_back(iter->second.id); } _rmap.clear(); @@ -1487,14 +1505,16 @@ int ReplicatorGroup::reset_election_timeout_interval(int new_interval_ms) { return 0; } -int ReplicatorGroup::transfer_leadership_to( - const PeerId& peer, int64_t log_index) { - std::map::const_iterator iter = _rmap.find(peer); +int ReplicatorGroup::transfer_leadership_to(const PeerId& peer, + int64_t log_index) { + std::map::const_iterator iter = + _rmap.find(peer); if (iter == _rmap.end()) { return EINVAL; } ReplicatorId rid = iter->second.id; - const int consecutive_error_times = Replicator::get_consecutive_error_times(rid); + const int consecutive_error_times = + Replicator::get_consecutive_error_times(rid); if (consecutive_error_times > 0) { return EHOSTUNREACH; } @@ -1502,7 +1522,8 @@ int ReplicatorGroup::transfer_leadership_to( } int ReplicatorGroup::stop_transfer_leadership(const PeerId& peer) { - std::map::const_iterator iter = _rmap.find(peer); + std::map::const_iterator iter = + _rmap.find(peer); if (iter == _rmap.end()) { return -1; } @@ -1511,20 +1532,21 @@ int ReplicatorGroup::stop_transfer_leadership(const PeerId& peer) { } int ReplicatorGroup::stop_all_and_find_the_next_candidate( - ReplicatorId* candidate, const ConfigurationEntry& conf) { + ReplicatorId* candidate, const ConfigurationEntry& conf) { *candidate = INVALID_BTHREAD_ID.value; PeerId candidate_id; const int rc = find_the_next_candidate(&candidate_id, conf); if (rc == 0) { - LOG(INFO) << "Group " << _common_options.group_id - << " Found " << candidate_id << " as the next candidate"; + LOG(INFO) << "Group " << _common_options.group_id << " Found " + << candidate_id << " as the next candidate"; *candidate = _rmap[candidate_id].id; } else { LOG(INFO) << "Group " << _common_options.group_id << " Fail to find the next candidate"; } - for (std::map::const_iterator - iter = _rmap.begin(); iter != _rmap.end(); ++iter) { + for (std::map::const_iterator iter = + _rmap.begin(); + iter != _rmap.end(); ++iter) { if (iter->second.id != *candidate) { Replicator::stop(iter->second.id); } @@ -1533,9 +1555,9 @@ int ReplicatorGroup::stop_all_and_find_the_next_candidate( return 0; } -int ReplicatorGroup::find_the_next_candidate( - PeerId* peer_id, const ConfigurationEntry& conf) { - int64_t max_index = 0; +int ReplicatorGroup::find_the_next_candidate(PeerId* peer_id, + const ConfigurationEntry& conf) { + int64_t max_index = 0; struct peerInfo { peerInfo(const PeerId& id, const ReplicatorIdAndStatus& status) { peer_id = id; @@ -1545,18 +1567,22 @@ int ReplicatorGroup::find_the_next_candidate( ReplicatorIdAndStatus id_and_status; }; std::vector peers; - for (std::map::const_iterator - iter = _rmap.begin(); iter != _rmap.end(); ++iter) { - peers.emplace_back(peerInfo(iter->first,iter->second)); + for (std::map::const_iterator iter = + _rmap.begin(); + iter != _rmap.end(); ++iter) { + peers.emplace_back(peerInfo(iter->first, iter->second)); } std::random_shuffle(peers.begin(), peers.end()); - for (auto iter = peers.begin(); iter != peers.end(); ++iter) { + for (auto iter = peers.begin(); iter != peers.end(); ++iter) { if (!conf.contains(iter->peer_id)) { continue; } - const int64_t next_index = Replicator::get_next_index(iter->id_and_status.id); - const int consecutive_error_times = Replicator::get_consecutive_error_times(iter->id_and_status.id); - if (consecutive_error_times == 0 && next_index > max_index && !iter->peer_id.is_witness()) { + const int64_t next_index = + Replicator::get_next_index(iter->id_and_status.id); + const int consecutive_error_times = + Replicator::get_consecutive_error_times(iter->id_and_status.id); + if (consecutive_error_times == 0 && next_index > max_index && + !iter->peer_id.is_witness()) { max_index = next_index; if (peer_id) { *peer_id = iter->peer_id; @@ -1572,24 +1598,27 @@ int ReplicatorGroup::find_the_next_candidate( void ReplicatorGroup::list_replicators(std::vector* out) const { out->clear(); out->reserve(_rmap.size()); - for (std::map::const_iterator - iter = _rmap.begin(); iter != _rmap.end(); ++iter) { + for (std::map::const_iterator iter = + _rmap.begin(); + iter != _rmap.end(); ++iter) { out->push_back(iter->second.id); } } void ReplicatorGroup::list_replicators( - std::vector >* out) const { + std::vector >* out) const { out->clear(); out->reserve(_rmap.size()); - for (std::map::const_iterator - iter = _rmap.begin(); iter != _rmap.end(); ++iter) { + for (std::map::const_iterator iter = + _rmap.begin(); + iter != _rmap.end(); ++iter) { out->push_back(std::make_pair(iter->first, iter->second.id)); } } - + int ReplicatorGroup::change_readonly_config(const PeerId& peer, bool readonly) { - std::map::const_iterator iter = _rmap.find(peer); + std::map::const_iterator iter = + _rmap.find(peer); if (iter == _rmap.end()) { return -1; } @@ -1598,7 +1627,8 @@ int ReplicatorGroup::change_readonly_config(const PeerId& peer, bool readonly) { } bool ReplicatorGroup::readonly(const PeerId& peer) const { - std::map::const_iterator iter = _rmap.find(peer); + std::map::const_iterator iter = + _rmap.find(peer); if (iter == _rmap.end()) { return false; } @@ -1606,4 +1636,4 @@ bool ReplicatorGroup::readonly(const PeerId& peer) const { return Replicator::readonly(rid); } -} // namespace braft +} // namespace braft diff --git a/src/braft/replicator.h b/src/braft/replicator.h index 6ffd212..9608183 100644 --- a/src/braft/replicator.h +++ b/src/braft/replicator.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,17 +16,17 @@ // Wang,Yao(wangyao02@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#ifndef BRAFT_REPLICATOR_H -#define BRAFT_REPLICATOR_H +#ifndef BRAFT_REPLICATOR_H +#define BRAFT_REPLICATOR_H -#include // bthread_id -#include // brpc::Channel +#include // brpc::Channel +#include // bthread_id -#include "braft/storage.h" // SnapshotStorage -#include "braft/raft.h" // Closure -#include "braft/configuration.h" // Configuration -#include "braft/raft.pb.h" // AppendEntriesRequest -#include "braft/log_manager.h" // LogManager +#include "braft/configuration.h" // Configuration +#include "braft/log_manager.h" // LogManager +#include "braft/raft.h" // Closure +#include "braft/raft.pb.h" // AppendEntriesRequest +#include "braft/storage.h" // SnapshotStorage namespace braft { @@ -35,8 +35,8 @@ class BallotBox; class NodeImpl; class SnapshotThrottle; -// A shared structure to store some high-frequency replicator statuses, for reducing -// the lock contention between Replicator and NodeImpl. +// A shared structure to store some high-frequency replicator statuses, for +// reducing the lock contention between Replicator and NodeImpl. struct ReplicatorStatus : public butil::RefCountedThreadSafe { butil::atomic last_rpc_send_timestamp; @@ -52,7 +52,7 @@ struct ReplicatorOptions { PeerId peer_id; LogManager* log_manager; BallotBox* ballot_box; - NodeImpl *node; + NodeImpl* node; int64_t term; SnapshotStorage* snapshot_storage; SnapshotThrottle* snapshot_throttle; @@ -62,16 +62,15 @@ struct ReplicatorOptions { typedef uint64_t ReplicatorId; class CatchupClosure : public Closure { -public: + public: virtual void Run() = 0; -protected: + + protected: CatchupClosure() - : _max_margin(0) - , _has_timer(false) - , _error_was_set(false) - {} -private: -friend class Replicator; + : _max_margin(0), _has_timer(false), _error_was_set(false) {} + + private: + friend class Replicator; int64_t _max_margin; bthread_timer_t _timer; bool _has_timer; @@ -80,7 +79,7 @@ friend class Replicator; }; class BAIDU_CACHELINE_ALIGNMENT Replicator { -public: + public: // Called by the leader, otherwise the behavior is undefined // Start to replicate the log to the given follower static int start(const ReplicatorOptions&, ReplicatorId* id); @@ -92,7 +91,7 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { static int join(ReplicatorId); // Wait until the margin between |last_log_index| from leader and the peer - // is less than |max_margin| or error occurs. + // is less than |max_margin| or error occurs. // |done| can't be NULL and it is called after waiting fnishies. static void wait_for_caught_up(ReplicatorId, int64_t max_margin, const timespec* due_time, @@ -127,8 +126,8 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { // Check if a replicator is readonly static bool readonly(ReplicatorId id); - -private: + + private: enum St { IDLE, BLOCKING, @@ -155,8 +154,8 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { void _send_empty_entries(bool is_heartbeat); void _send_entries(); void _notify_on_caught_up(int error_code, bool); - int _fill_common_fields(AppendEntriesRequest* request, int64_t prev_log_index, - bool is_heartbeat); + int _fill_common_fields(AppendEntriesRequest* request, + int64_t prev_log_index, bool is_heartbeat); void _block(long start_time_us, int error_code); void _install_snapshot(); void _start_heartbeat_timer(long start_time_us); @@ -170,23 +169,20 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { } int _change_readonly_config(bool readonly); - static void _on_rpc_returned( - ReplicatorId id, brpc::Controller* cntl, - AppendEntriesRequest* request, - AppendEntriesResponse* response, - int64_t); + static void _on_rpc_returned(ReplicatorId id, brpc::Controller* cntl, + AppendEntriesRequest* request, + AppendEntriesResponse* response, int64_t); - static void _on_heartbeat_returned( - ReplicatorId id, brpc::Controller* cntl, - AppendEntriesRequest* request, - AppendEntriesResponse* response, - int64_t); + static void _on_heartbeat_returned(ReplicatorId id, brpc::Controller* cntl, + AppendEntriesRequest* request, + AppendEntriesResponse* response, + int64_t); - static void _on_timeout_now_returned( - ReplicatorId id, brpc::Controller* cntl, - TimeoutNowRequest* request, - TimeoutNowResponse* response, - bool old_leader_stepped_down); + static void _on_timeout_now_returned(ReplicatorId id, + brpc::Controller* cntl, + TimeoutNowRequest* request, + TimeoutNowResponse* response, + bool old_leader_stepped_down); static void _on_timedout(void* arg); static void* _send_heartbeat(void* arg); @@ -195,42 +191,40 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { static int _continue_sending(void* arg, int error_code); static void* _run_on_caught_up(void*); static void _on_catch_up_timedout(void*); - static void _on_block_timedout(void *arg); - static void* _on_block_timedout_in_new_thread(void *arg); + static void _on_block_timedout(void* arg); + static void* _on_block_timedout_in_new_thread(void* arg); static void _on_install_snapshot_returned( - ReplicatorId id, brpc::Controller* cntl, - InstallSnapshotRequest* request, - InstallSnapshotResponse* response); + ReplicatorId id, brpc::Controller* cntl, + InstallSnapshotRequest* request, InstallSnapshotResponse* response); void _destroy(); void _describe(std::ostream& os, bool use_html); void _get_status(PeerStatus* status); bool _is_catchup(int64_t max_margin) { - // We should wait until install snapshot finish. If the process is throttled, - // it maybe very slow. + // We should wait until install snapshot finish. If the process is + // throttled, it maybe very slow. if (_next_index < _options.log_manager->first_log_index()) { return false; } - if (_min_flying_index() - 1 + max_margin - < _options.log_manager->last_log_index()) { + if (_min_flying_index() - 1 + max_margin < + _options.log_manager->last_log_index()) { return false; } return true; } - bool is_witness() const { - return _options.peer_id.is_witness(); - } + bool is_witness() const { return _options.peer_id.is_witness(); } void _close_reader(); int64_t _last_rpc_send_timestamp() { - return _options.replicator_status->last_rpc_send_timestamp.load(butil::memory_order_relaxed); + return _options.replicator_status->last_rpc_send_timestamp.load( + butil::memory_order_relaxed); } void _update_last_rpc_send_timestamp(int64_t new_timestamp) { if (new_timestamp > _last_rpc_send_timestamp()) { - _options.replicator_status->last_rpc_send_timestamp - .store(new_timestamp, butil::memory_order_relaxed); + _options.replicator_status->last_rpc_send_timestamp.store( + new_timestamp, butil::memory_order_relaxed); } } -private: + private: struct FlyingAppendEntriesRpc { int64_t log_index; int entries_size; @@ -238,7 +232,7 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { FlyingAppendEntriesRpc(int64_t index, int size, brpc::CallId id) : log_index(index), entries_size(size), call_id(id) {} }; - + brpc::Channel _sending_channel; int64_t _next_index; int64_t _flying_append_entries_size; @@ -260,7 +254,7 @@ class BAIDU_CACHELINE_ALIGNMENT Replicator { ReplicatorOptions _options; bthread_timer_t _heartbeat_timer; SnapshotReader* _reader; - CatchupClosure *_catchup_closure; + CatchupClosure* _catchup_closure; }; struct ReplicatorGroupOptions { @@ -283,11 +277,11 @@ struct ReplicatorGroupOptions { // // Note: The methods of ReplicatorGroup are NOT thread-safe class ReplicatorGroup { -public: + public: ReplicatorGroup(); ~ReplicatorGroup(); int init(const NodeId& node_id, const ReplicatorGroupOptions&); - + // Add a replicator attached with |peer| // will be a notification when the replicator catches up according to the // arguments. @@ -295,7 +289,7 @@ class ReplicatorGroup { // immediately, annd might call node->step_down which might have race with // the caller, you should deal with this situation. int add_replicator(const PeerId& peer); - + // wait the very peer catchup int wait_caughtup(const PeerId& peer, int64_t max_margin, const timespec* due_time, CatchupClosure* done); @@ -305,7 +299,7 @@ class ReplicatorGroup { // Stop all the replicators int stop_all(); - int stop_replicator(const PeerId &peer); + int stop_replicator(const PeerId& peer); // Reset the term of all to-add replicators. // This method is supposed to be called when the very candidate becomes the @@ -315,11 +309,11 @@ class ReplicatorGroup { // Reset the interval of heartbeat // This method is supposed to be called when the very candidate becomes the - // leader, use new heartbeat_interval, maybe call vote() reset election_timeout - // Return 0 on success, -1 otherwise + // leader, use new heartbeat_interval, maybe call vote() reset + // election_timeout Return 0 on success, -1 otherwise int reset_heartbeat_interval(int new_interval_ms); - - // Reset the interval of election_timeout for replicator, + + // Reset the interval of election_timeout for replicator, // used in rpc's set_timeout_ms int reset_election_timeout_interval(int new_interval_ms); @@ -334,14 +328,14 @@ class ReplicatorGroup { // Stop all the replicators except for the one that we think can be the // candidate of the next leader, which has the largest `last_log_id' among - // peers in |current_conf|. + // peers in |current_conf|. // |candidate| would be assigned to a valid ReplicatorId if we found one and // the caller is responsible for stopping it, or an invalid value if we // found none. // Returns 0 on success and -1 otherwise. int stop_all_and_find_the_next_candidate(ReplicatorId* candidate, const ConfigurationEntry& conf); - + // Find the follower with the most log entries in this group, which is // likely becomes the leader according to the election algorithm of raft. // Returns 0 on success and |peer_id| is assigned with the very peer. @@ -353,7 +347,8 @@ class ReplicatorGroup { void list_replicators(std::vector* out) const; // List all the existing replicators with PeerId - void list_replicators(std::vector >* out) const; + void list_replicators( + std::vector >* out) const; // Change the readonly config for a peer int change_readonly_config(const PeerId& peer, bool readonly); @@ -361,9 +356,8 @@ class ReplicatorGroup { // Check if a replicator is in readonly bool readonly(const PeerId& peer) const; -private: - - int _add_replicator(const PeerId& peer, ReplicatorId *rid); + private: + int _add_replicator(const PeerId& peer, ReplicatorId* rid); struct ReplicatorIdAndStatus { ReplicatorId id; @@ -378,4 +372,4 @@ class ReplicatorGroup { } // namespace braft -#endif //BRAFT_REPLICATOR_H +#endif // BRAFT_REPLICATOR_H diff --git a/src/braft/route_table.cpp b/src/braft/route_table.cpp index 000abd5..8a760dd 100644 --- a/src/braft/route_table.cpp +++ b/src/braft/route_table.cpp @@ -3,9 +3,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,12 +16,13 @@ #include "braft/route_table.h" -#include -#include +#include +#include #include #include -#include -#include +#include +#include + #include "braft/cli.pb.h" namespace braft { @@ -30,11 +31,10 @@ namespace rtb { DEFINE_int32(initial_route_table_cap, 128, "Initial capacity of RouteTable"); class RouteTable { -DISALLOW_COPY_AND_ASSIGN(RouteTable); -public: - static RouteTable* GetInstance() { - return Singleton::get(); - } + DISALLOW_COPY_AND_ASSIGN(RouteTable); + + public: + static RouteTable* GetInstance() { return Singleton::get(); } void update_conf(const GroupId& group, const Configuration& conf) { _map.Modify(modify_conf, group, conf); } @@ -78,11 +78,9 @@ DISALLOW_COPY_AND_ASSIGN(RouteTable); return 0; } -private: -friend struct DefaultSingletonTraits; - RouteTable() { - _map.Modify(init); - } + private: + friend struct DefaultSingletonTraits; + RouteTable() { _map.Modify(init); } ~RouteTable() {} struct GroupConf { @@ -119,7 +117,8 @@ friend struct DefaultSingletonTraits; static size_t delete_group(GroupMap& m, const GroupId& group) { GroupConf* gc = m.seek(group); if (gc != NULL) { - return (size_t)m.erase(group);; + return (size_t)m.erase(group); + ; } return 0; } @@ -168,21 +167,20 @@ butil::Status refresh_leader(const GroupId& group, int timeout_ms) { Configuration conf; if (rtb->list_conf(group, &conf) != 0) { return butil::Status(ENOENT, "group %s is not reistered in RouteTable", - group.c_str()); + group.c_str()); } butil::Status error; - for (Configuration::const_iterator - iter = conf.begin(); iter != conf.end(); ++iter) { + for (Configuration::const_iterator iter = conf.begin(); iter != conf.end(); + ++iter) { brpc::Channel channel; if (channel.Init(iter->addr, NULL) != 0) { if (error.ok()) { error.set_error(-1, "Fail to init channel to %s", - iter->to_string().c_str()); + iter->to_string().c_str()); } else { std::string saved_et = error.error_str(); error.set_error(-1, "%s, Fail to init channel to %s", - saved_et.c_str(), - iter->to_string().c_str()); + saved_et.c_str(), iter->to_string().c_str()); } continue; } @@ -199,14 +197,13 @@ butil::Status refresh_leader(const GroupId& group, int timeout_ms) { } if (error.ok()) { error.set_error(cntl.ErrorCode(), "[%s] %s", - iter->to_string().c_str(), - cntl.ErrorText().c_str()); + iter->to_string().c_str(), + cntl.ErrorText().c_str()); } else { std::string saved_et = error.error_str(); - error.set_error(cntl.ErrorCode(), "%s, [%s] %s", - saved_et.c_str(), - iter->to_string().c_str(), - cntl.ErrorText().c_str()); + error.set_error(cntl.ErrorCode(), "%s, [%s] %s", saved_et.c_str(), + iter->to_string().c_str(), + cntl.ErrorText().c_str()); } } return error; diff --git a/src/braft/route_table.h b/src/braft/route_table.h index 10dc6f8..6dbf0ca 100644 --- a/src/braft/route_table.h +++ b/src/braft/route_table.h @@ -3,9 +3,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,10 +14,10 @@ // Authors: Zhangyi Chen (chenzhangyi01@baidu.com) -#ifndef BRAFT_ROUTE_TABLE_H -#define BRAFT_ROUTE_TABLE_H +#ifndef BRAFT_ROUTE_TABLE_H +#define BRAFT_ROUTE_TABLE_H -#include "braft/configuration.h" // Configuration +#include "braft/configuration.h" // Configuration #include "braft/raft.h" // Maintain routes to raft groups @@ -47,6 +47,6 @@ butil::Status refresh_leader(const GroupId& group, int timeout_ms); int remove_group(const GroupId& group); } // namespace rtb -} // namespace braft +} // namespace braft -#endif //BRAFT_ROUTE_TABLE_H +#endif // BRAFT_ROUTE_TABLE_H diff --git a/src/braft/snapshot.cpp b/src/braft/snapshot.cpp index 91a8385..1632aef 100644 --- a/src/braft/snapshot.cpp +++ b/src/braft/snapshot.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,16 +17,18 @@ // Zheng,Pengfei(zhengpengfei@baidu.com) // Xiong,Kai(xiongkai@baidu.com) -#include -#include // butil::string_appendf +#include "braft/snapshot.h" + #include -#include "braft/util.h" -#include "braft/protobuf_file.h" +#include // butil::string_appendf +#include + +#include "braft/file_service.h" #include "braft/local_storage.pb.h" -#include "braft/remote_file_copier.h" -#include "braft/snapshot.h" #include "braft/node.h" -#include "braft/file_service.h" +#include "braft/protobuf_file.h" +#include "braft/remote_file_copier.h" +#include "braft/util.h" //#define BRAFT_SNAPSHOT_PATTERN "snapshot_%020ld" #define BRAFT_SNAPSHOT_PATTERN "snapshot_%020" PRId64 @@ -40,12 +42,12 @@ LocalSnapshotMetaTable::LocalSnapshotMetaTable() {} LocalSnapshotMetaTable::~LocalSnapshotMetaTable() {} -int LocalSnapshotMetaTable::add_file(const std::string& filename, - const LocalFileMeta& meta) { +int LocalSnapshotMetaTable::add_file(const std::string& filename, + const LocalFileMeta& meta) { Map::value_type value(filename, meta); std::pair ret = _file_map.insert(value); LOG_IF(WARNING, !ret.second) - << "file=" << filename << " already exists in snapshot"; + << "file=" << filename << " already exists in snapshot"; return ret.second ? 0 : -1; } @@ -58,14 +60,15 @@ int LocalSnapshotMetaTable::remove_file(const std::string& filename) { return 0; } -int LocalSnapshotMetaTable::save_to_file(FileSystemAdaptor* fs, const std::string& path) const { +int LocalSnapshotMetaTable::save_to_file(FileSystemAdaptor* fs, + const std::string& path) const { LocalSnapshotPbMeta pb_meta; if (_meta.IsInitialized()) { *pb_meta.mutable_meta() = _meta; } - for (Map::const_iterator - iter = _file_map.begin(); iter != _file_map.end(); ++iter) { - LocalSnapshotPbMeta::File *f = pb_meta.add_files(); + for (Map::const_iterator iter = _file_map.begin(); iter != _file_map.end(); + ++iter) { + LocalSnapshotPbMeta::File* f = pb_meta.add_files(); f->set_name(iter->first); *f->mutable_meta() = iter->second; } @@ -75,7 +78,8 @@ int LocalSnapshotMetaTable::save_to_file(FileSystemAdaptor* fs, const std::strin return ret; } -int LocalSnapshotMetaTable::load_from_file(FileSystemAdaptor* fs, const std::string& path) { +int LocalSnapshotMetaTable::load_from_file(FileSystemAdaptor* fs, + const std::string& path) { ProtoBufFile pb_file(path, fs); LocalSnapshotPbMeta pb_meta; if (pb_file.load(&pb_meta) != 0) { @@ -100,9 +104,9 @@ int LocalSnapshotMetaTable::save_to_iobuf_as_remote(butil::IOBuf* buf) const { if (_meta.IsInitialized()) { *pb_meta.mutable_meta() = _meta; } - for (Map::const_iterator - iter = _file_map.begin(); iter != _file_map.end(); ++iter) { - LocalSnapshotPbMeta::File *f = pb_meta.add_files(); + for (Map::const_iterator iter = _file_map.begin(); iter != _file_map.end(); + ++iter) { + LocalSnapshotPbMeta::File* f = pb_meta.add_files(); f->set_name(iter->first); *f->mutable_meta() = iter->second; f->mutable_meta()->clear_source(); @@ -138,13 +142,13 @@ void LocalSnapshotMetaTable::list_files(std::vector* files) const { } files->clear(); files->reserve(_file_map.size()); - for (Map::const_iterator - iter = _file_map.begin(); iter != _file_map.end(); ++iter) { + for (Map::const_iterator iter = _file_map.begin(); iter != _file_map.end(); + ++iter) { files->push_back(iter->first); } } -int LocalSnapshotMetaTable::get_file_meta(const std::string& filename, +int LocalSnapshotMetaTable::get_file_meta(const std::string& filename, LocalFileMeta* file_meta) const { Map::const_iterator iter = _file_map.find(filename); if (iter == _file_map.end()) { @@ -158,12 +162,12 @@ int LocalSnapshotMetaTable::get_file_meta(const std::string& filename, std::string LocalSnapshot::get_path() { return std::string(); } -void LocalSnapshot::list_files(std::vector *files) { +void LocalSnapshot::list_files(std::vector* files) { return _meta_table.list_files(files); } -int LocalSnapshot::get_file_meta(const std::string& filename, - ::google::protobuf::Message* file_meta) { +int LocalSnapshot::get_file_meta(const std::string& filename, + ::google::protobuf::Message* file_meta) { LocalFileMeta* meta = NULL; if (file_meta) { meta = dynamic_cast(file_meta); @@ -176,11 +180,9 @@ int LocalSnapshot::get_file_meta(const std::string& filename, LocalSnapshotWriter::LocalSnapshotWriter(const std::string& path, FileSystemAdaptor* fs) - : _path(path), _fs(fs) { -} + : _path(path), _fs(fs) {} -LocalSnapshotWriter::~LocalSnapshotWriter() { -} +LocalSnapshotWriter::~LocalSnapshotWriter() {} int LocalSnapshotWriter::init() { butil::File::Error e; @@ -190,22 +192,24 @@ int LocalSnapshotWriter::init() { return EIO; } std::string meta_path = _path + "/" BRAFT_SNAPSHOT_META_FILE; - if (_fs->path_exists(meta_path) && - _meta_table.load_from_file(_fs, meta_path) != 0) { + if (_fs->path_exists(meta_path) && + _meta_table.load_from_file(_fs, meta_path) != 0) { LOG(ERROR) << "Fail to load meta from " << meta_path; set_error(EIO, "Fail to load metatable from %s", meta_path.c_str()); return EIO; } - // remove file if meta_path not exist or it's not in _meta_table + // remove file if meta_path not exist or it's not in _meta_table // to avoid dirty data { std::vector to_remove; DirReader* dir_reader = _fs->directory_reader(_path); if (!dir_reader->is_valid()) { - LOG(ERROR) << "Invalid directory reader, maybe NOEXIST or PERMISSION," - << " path: " << _path; - set_error(EIO, "Invalid directory reader in path: %s", _path.c_str()); + LOG(ERROR) + << "Invalid directory reader, maybe NOEXIST or PERMISSION," + << " path: " << _path; + set_error(EIO, "Invalid directory reader in path: %s", + _path.c_str()); delete dir_reader; return EIO; } @@ -221,7 +225,8 @@ int LocalSnapshotWriter::init() { for (size_t i = 0; i < to_remove.size(); ++i) { std::string file_path = _path + "/" + to_remove[i]; _fs->delete_file(file_path, false); - LOG(WARNING) << "Snapshot file exist but meta not found so delete it," + LOG(WARNING) + << "Snapshot file exist but meta not found so delete it," << " path: " << file_path; } } @@ -230,7 +235,8 @@ int LocalSnapshotWriter::init() { } int64_t LocalSnapshotWriter::snapshot_index() { - return _meta_table.has_meta() ? _meta_table.meta().last_included_index() : 0; + return _meta_table.has_meta() ? _meta_table.meta().last_included_index() + : 0; } int LocalSnapshotWriter::remove_file(const std::string& filename) { @@ -238,8 +244,7 @@ int LocalSnapshotWriter::remove_file(const std::string& filename) { } int LocalSnapshotWriter::add_file( - const std::string& filename, - const ::google::protobuf::Message* file_meta) { + const std::string& filename, const ::google::protobuf::Message* file_meta) { // TODO: normalize filename LocalFileMeta meta; if (file_meta) { @@ -249,11 +254,11 @@ int LocalSnapshotWriter::add_file( return _meta_table.add_file(filename, meta); } -void LocalSnapshotWriter::list_files(std::vector *files) { +void LocalSnapshotWriter::list_files(std::vector* files) { return _meta_table.list_files(files); } -int LocalSnapshotWriter::get_file_meta(const std::string& filename, +int LocalSnapshotWriter::get_file_meta(const std::string& filename, ::google::protobuf::Message* file_meta) { LocalFileMeta* meta = NULL; if (file_meta) { @@ -271,7 +276,8 @@ int LocalSnapshotWriter::save_meta(const SnapshotMeta& meta) { } int LocalSnapshotWriter::sync() { - const int rc = _meta_table.save_to_file(_fs, _path + "/" BRAFT_SNAPSHOT_META_FILE); + const int rc = + _meta_table.save_to_file(_fs, _path + "/" BRAFT_SNAPSHOT_META_FILE); if (rc != 0 && ok()) { LOG(ERROR) << "Fail to sync, path: " << _path; set_error(rc, "Fail to sync : %s", berror(rc)); @@ -283,12 +289,11 @@ LocalSnapshotReader::LocalSnapshotReader(const std::string& path, butil::EndPoint server_addr, FileSystemAdaptor* fs, SnapshotThrottle* snapshot_throttle) - : _path(path) - , _addr(server_addr) - , _reader_id(0) - , _fs(fs) - , _snapshot_throttle(snapshot_throttle) -{} + : _path(path), + _addr(server_addr), + _reader_id(0), + _fs(fs), + _snapshot_throttle(snapshot_throttle) {} LocalSnapshotReader::~LocalSnapshotReader() { destroy_reader_in_file_service(); @@ -318,16 +323,17 @@ int LocalSnapshotReader::load_meta(SnapshotMeta* meta) { int64_t LocalSnapshotReader::snapshot_index() { butil::FilePath path(_path); int64_t index = 0; - int ret = sscanf(path.BaseName().value().c_str(), BRAFT_SNAPSHOT_PATTERN, &index); + int ret = + sscanf(path.BaseName().value().c_str(), BRAFT_SNAPSHOT_PATTERN, &index); CHECK_EQ(ret, 1); return index; } -void LocalSnapshotReader::list_files(std::vector *files) { +void LocalSnapshotReader::list_files(std::vector* files) { return _meta_table.list_files(files); } -int LocalSnapshotReader::get_file_meta(const std::string& filename, +int LocalSnapshotReader::get_file_meta(const std::string& filename, ::google::protobuf::Message* file_meta) { LocalFileMeta* meta = NULL; if (file_meta) { @@ -340,24 +346,17 @@ int LocalSnapshotReader::get_file_meta(const std::string& filename, } class SnapshotFileReader : public LocalDirReader { -public: - SnapshotFileReader(FileSystemAdaptor* fs, - const std::string& path, + public: + SnapshotFileReader(FileSystemAdaptor* fs, const std::string& path, SnapshotThrottle* snapshot_throttle) - : LocalDirReader(fs, path) - , _snapshot_throttle(snapshot_throttle) - {} + : LocalDirReader(fs, path), _snapshot_throttle(snapshot_throttle) {} - void set_meta_table(const LocalSnapshotMetaTable &meta_table) { + void set_meta_table(const LocalSnapshotMetaTable& meta_table) { _meta_table = meta_table; } - int read_file(butil::IOBuf* out, - const std::string &filename, - off_t offset, - size_t max_count, - bool read_partly, - size_t* read_count, + int read_file(butil::IOBuf* out, const std::string& filename, off_t offset, + size_t max_count, bool read_partly, size_t* read_count, bool* is_eof) const { if (filename == BRAFT_SNAPSHOT_META_FILE) { int ret = _meta_table.save_to_iobuf_as_remote(out); @@ -373,11 +372,13 @@ class SnapshotFileReader : public LocalDirReader { } // go through throttle size_t new_max_count = max_count; - if (_snapshot_throttle && FLAGS_raft_enable_throttle_when_install_snapshot) { + if (_snapshot_throttle && + FLAGS_raft_enable_throttle_when_install_snapshot) { int ret = 0; int64_t start = butil::cpuwide_time_us(); int64_t used_count = 0; - new_max_count = _snapshot_throttle->throttled_by_throughput(max_count); + new_max_count = + _snapshot_throttle->throttled_by_throughput(max_count); if (new_max_count < max_count) { // if it's not allowed to read partly or it's allowed but // throughput is throttled to 0, try again. @@ -388,20 +389,24 @@ class SnapshotFileReader : public LocalDirReader { } if (ret == 0) { ret = LocalDirReader::read_file_with_meta( - out, filename, &file_meta, offset, new_max_count, read_count, is_eof); + out, filename, &file_meta, offset, new_max_count, + read_count, is_eof); used_count = out->size(); } - if ((ret == 0 || ret == EAGAIN) && used_count < (int64_t)new_max_count) { + if ((ret == 0 || ret == EAGAIN) && + used_count < (int64_t)new_max_count) { _snapshot_throttle->return_unused_throughput( - new_max_count, used_count, butil::cpuwide_time_us() - start); + new_max_count, used_count, + butil::cpuwide_time_us() - start); } return ret; } - return LocalDirReader::read_file_with_meta( - out, filename, &file_meta, offset, new_max_count, read_count, is_eof); + return LocalDirReader::read_file_with_meta(out, filename, &file_meta, + offset, new_max_count, + read_count, is_eof); } - -private: + + private: LocalSnapshotMetaTable _meta_table; scoped_refptr _snapshot_throttle; }; @@ -414,7 +419,7 @@ std::string LocalSnapshotReader::generate_uri_for_copy() { if (_reader_id == 0) { // TODO: handler referenced files scoped_refptr reader( - new SnapshotFileReader(_fs.get(), _path, _snapshot_throttle.get())); + new SnapshotFileReader(_fs.get(), _path, _snapshot_throttle.get())); reader->set_meta_table(_meta_table); if (!reader->open()) { LOG(ERROR) << "Open snapshot=" << _path << " failed"; @@ -438,20 +443,17 @@ void LocalSnapshotReader::destroy_reader_in_file_service() { } LocalSnapshotStorage::LocalSnapshotStorage(const std::string& path) - : _path(path) - , _last_snapshot_index(0) -{} + : _path(path), _last_snapshot_index(0) {} -LocalSnapshotStorage::~LocalSnapshotStorage() { -} +LocalSnapshotStorage::~LocalSnapshotStorage() {} int LocalSnapshotStorage::init() { butil::File::Error e; if (_fs == NULL) { _fs = default_file_system(); } - if (!_fs->create_directory( - _path, &e, FLAGS_raft_create_parent_directories)) { + if (!_fs->create_directory(_path, &e, + FLAGS_raft_create_parent_directories)) { LOG(ERROR) << "Fail to create " << _path << " : " << e; return -1; } @@ -462,7 +464,8 @@ int LocalSnapshotStorage::init() { temp_snapshot_path.append(_s_temp_path); LOG(INFO) << "Deleting " << temp_snapshot_path; if (!_fs->delete_file(temp_snapshot_path, true)) { - LOG(WARNING) << "delete temp snapshot path failed, path " << temp_snapshot_path; + LOG(WARNING) << "delete temp snapshot path failed, path " + << temp_snapshot_path; return EIO; } } @@ -470,7 +473,9 @@ int LocalSnapshotStorage::init() { // delete old snapshot DirReader* dir_reader = _fs->directory_reader(_path); if (!dir_reader->is_valid()) { - LOG(WARNING) << "directory reader failed, maybe NOEXIST or PERMISSION. path: " << _path; + LOG(WARNING) + << "directory reader failed, maybe NOEXIST or PERMISSION. path: " + << _path; delete dir_reader; return EIO; } @@ -494,11 +499,13 @@ int LocalSnapshotStorage::init() { snapshots.erase(index); std::string snapshot_path(_path); - butil::string_appendf(&snapshot_path, "/" BRAFT_SNAPSHOT_PATTERN, index); + butil::string_appendf(&snapshot_path, "/" BRAFT_SNAPSHOT_PATTERN, + index); LOG(INFO) << "Deleting snapshot `" << snapshot_path << "'"; // TODO: Notify Watcher before delete directories. if (!_fs->delete_file(snapshot_path, true)) { - LOG(WARNING) << "delete old snapshot path failed, path " << snapshot_path; + LOG(WARNING) << "delete old snapshot path failed, path " + << snapshot_path; return EIO; } } @@ -516,7 +523,7 @@ void LocalSnapshotStorage::ref(const int64_t index) { } int LocalSnapshotStorage::destroy_snapshot(const std::string& path) { - LOG(INFO) << "Deleting " << path; + LOG(INFO) << "Deleting " << path; if (!_fs->delete_file(path, true)) { LOG(WARNING) << "delete old snapshot path failed, path " << path; return -1; @@ -540,9 +547,7 @@ void LocalSnapshotStorage::unref(const int64_t index) { } } -SnapshotWriter* LocalSnapshotStorage::create() { - return create(true); -} +SnapshotWriter* LocalSnapshotStorage::create() { return create(true); } SnapshotWriter* LocalSnapshotStorage::create(bool from_empty) { LocalSnapshotWriter* writer = NULL; @@ -562,7 +567,7 @@ SnapshotWriter* LocalSnapshotStorage::create(bool from_empty) { writer = new LocalSnapshotWriter(snapshot_path, _fs.get()); if (writer->init() != 0) { - LOG(ERROR) << "Fail to init writer in path " << snapshot_path + LOG(ERROR) << "Fail to init writer in path " << snapshot_path << ", " << *writer; delete writer; writer = NULL; @@ -574,15 +579,15 @@ SnapshotWriter* LocalSnapshotStorage::create(bool from_empty) { return writer; } -SnapshotCopier* LocalSnapshotStorage::start_to_copy_from(const std::string& uri) { +SnapshotCopier* LocalSnapshotStorage::start_to_copy_from( + const std::string& uri) { LocalSnapshotCopier* copier = new LocalSnapshotCopier(_copy_file); copier->_storage = this; copier->_filter_before_copy_remote = _filter_before_copy_remote; copier->_fs = _fs.get(); copier->_throttle = _snapshot_throttle.get(); if (copier->init(uri) != 0) { - LOG(ERROR) << "Fail to init copier from " << uri - << " path: " << _path; + LOG(ERROR) << "Fail to init copier from " << uri << " path: " << _path; delete copier; return NULL; } @@ -612,7 +617,8 @@ int LocalSnapshotStorage::close(SnapshotWriter* writer) { int LocalSnapshotStorage::close(SnapshotWriter* writer_base, bool keep_data_on_error) { - LocalSnapshotWriter* writer = dynamic_cast(writer_base); + LocalSnapshotWriter* writer = + dynamic_cast(writer_base); int ret = writer->error_code(); do { if (0 != ret) { @@ -641,14 +647,15 @@ int LocalSnapshotStorage::close(SnapshotWriter* writer_base, butil::string_appendf(&new_path, "/" BRAFT_SNAPSHOT_PATTERN, new_index); LOG(INFO) << "Deleting " << new_path; if (!_fs->delete_file(new_path, true)) { - LOG(WARNING) << "delete new snapshot path failed, path " << new_path; + LOG(WARNING) << "delete new snapshot path failed, path " + << new_path; ret = EIO; break; } LOG(INFO) << "Renaming " << temp_path << " to " << new_path; if (!_fs->rename(temp_path, new_path)) { - LOG(WARNING) << "rename temp snapshot failed, from_path " << temp_path - << " to_path " << new_path; + LOG(WARNING) << "rename temp snapshot failed, from_path " + << temp_path << " to_path " << new_path; ret = EIO; break; } @@ -677,9 +684,10 @@ SnapshotReader* LocalSnapshotStorage::open() { ++_ref_map[last_snapshot_index]; lck.unlock(); std::string snapshot_path(_path); - butil::string_appendf(&snapshot_path, "/" BRAFT_SNAPSHOT_PATTERN, last_snapshot_index); - LocalSnapshotReader* reader = new LocalSnapshotReader(snapshot_path, _addr, - _fs.get(), _snapshot_throttle.get()); + butil::string_appendf(&snapshot_path, "/" BRAFT_SNAPSHOT_PATTERN, + last_snapshot_index); + LocalSnapshotReader* reader = new LocalSnapshotReader( + snapshot_path, _addr, _fs.get(), _snapshot_throttle.get()); if (reader->init() != 0) { CHECK(!lck.owns_lock()); unref(last_snapshot_index); @@ -714,12 +722,14 @@ int LocalSnapshotStorage::set_file_system_adaptor(FileSystemAdaptor* fs) { return 0; } -int LocalSnapshotStorage::set_snapshot_throttle(SnapshotThrottle* snapshot_throttle) { +int LocalSnapshotStorage::set_snapshot_throttle( + SnapshotThrottle* snapshot_throttle) { _snapshot_throttle = snapshot_throttle; return 0; } -SnapshotStorage* LocalSnapshotStorage::new_instance(const std::string& uri) const { +SnapshotStorage* LocalSnapshotStorage::new_instance( + const std::string& uri) const { return new LocalSnapshotStorage(uri); } @@ -727,7 +737,7 @@ butil::Status LocalSnapshotStorage::gc_instance(const std::string& uri) const { butil::Status status; if (gc_dir(uri) != 0) { LOG(WARNING) << "Failed to gc snapshot storage from path " << _path; - status.set_error(EINVAL, "Failed to gc snapshot storage from path %s", + status.set_error(EINVAL, "Failed to gc snapshot storage from path %s", uri.c_str()); return status; } @@ -737,26 +747,23 @@ butil::Status LocalSnapshotStorage::gc_instance(const std::string& uri) const { // LocalSnapshotCopier -LocalSnapshotCopier::LocalSnapshotCopier() - : LocalSnapshotCopier(true){} - -LocalSnapshotCopier::LocalSnapshotCopier(bool copy_file): - _tid(INVALID_BTHREAD) - , _cancelled(false) - , _filter_before_copy_remote(false) - , _copy_file(copy_file) - , _fs(NULL) - , _throttle(NULL) - , _writer(NULL) - , _storage(NULL) - , _reader(NULL) - , _cur_session(NULL){} - -LocalSnapshotCopier::~LocalSnapshotCopier() { - CHECK(!_writer); -} +LocalSnapshotCopier::LocalSnapshotCopier() : LocalSnapshotCopier(true) {} -void *LocalSnapshotCopier::start_copy(void* arg) { +LocalSnapshotCopier::LocalSnapshotCopier(bool copy_file) + : _tid(INVALID_BTHREAD), + _cancelled(false), + _filter_before_copy_remote(false), + _copy_file(copy_file), + _fs(NULL), + _throttle(NULL), + _writer(NULL), + _storage(NULL), + _reader(NULL), + _cur_session(NULL) {} + +LocalSnapshotCopier::~LocalSnapshotCopier() { CHECK(!_writer); } + +void* LocalSnapshotCopier::start_copy(void* arg) { LocalSnapshotCopier* c = (LocalSnapshotCopier*)arg; c->copy(); return NULL; @@ -773,7 +780,7 @@ void LocalSnapshotCopier::copy() { break; } if (!_copy_file) { - break; + break; } std::vector files; _remote_snapshot.list_files(&files); @@ -783,13 +790,13 @@ void LocalSnapshotCopier::copy() { } while (0); if (!ok() && _writer && _writer->ok()) { LOG(WARNING) << "Fail to copy, error_code " << error_code() - << " error_msg " << error_cstr() - << " writer path " << _writer->get_path(); + << " error_msg " << error_cstr() << " writer path " + << _writer->get_path(); _writer->set_error(error_code(), "%s", error_cstr()); } if (_writer) { - // set_error for copier only when failed to close writer and copier was - // ok before this moment + // set_error for copier only when failed to close writer and copier was + // ok before this moment if (_storage->close(_writer, _filter_before_copy_remote) != 0 && ok()) { set_error(EIO, "Fail to close writer"); } @@ -807,9 +814,9 @@ void LocalSnapshotCopier::load_meta_table() { set_error(ECANCELED, "%s", berror(ECANCELED)); return; } - scoped_refptr session - = _copier.start_to_copy_to_iobuf(BRAFT_SNAPSHOT_META_FILE, - &meta_buf, NULL); + scoped_refptr session = + _copier.start_to_copy_to_iobuf(BRAFT_SNAPSHOT_META_FILE, &meta_buf, + NULL); _cur_session = session.get(); lck.unlock(); session->join(); @@ -830,7 +837,7 @@ void LocalSnapshotCopier::load_meta_table() { CHECK(_remote_snapshot._meta_table.has_meta()); } -int LocalSnapshotCopier::filter_before_copy(LocalSnapshotWriter* writer, +int LocalSnapshotCopier::filter_before_copy(LocalSnapshotWriter* writer, SnapshotReader* last_snapshot) { std::vector existing_files; writer->list_files(&existing_files); @@ -848,8 +855,7 @@ int LocalSnapshotCopier::filter_before_copy(LocalSnapshotWriter* writer, for (size_t i = 0; i < remote_files.size(); ++i) { const std::string& filename = remote_files[i]; LocalFileMeta remote_meta; - CHECK_EQ(0, _remote_snapshot.get_file_meta( - filename, &remote_meta)); + CHECK_EQ(0, _remote_snapshot.get_file_meta(filename, &remote_meta)); if (!remote_meta.has_checksum()) { // Redownload file if this file doen't have checksum writer->remove_file(filename); @@ -862,8 +868,8 @@ int LocalSnapshotCopier::filter_before_copy(LocalSnapshotWriter* writer, if (local_meta.has_checksum() && local_meta.checksum() == remote_meta.checksum()) { LOG(INFO) << "Keep file=" << filename - << " checksum=" << remote_meta.checksum() - << " in " << writer->get_path(); + << " checksum=" << remote_meta.checksum() << " in " + << writer->get_path(); continue; } // Remove files from writer so that the file is to be copied from @@ -879,21 +885,21 @@ int LocalSnapshotCopier::filter_before_copy(LocalSnapshotWriter* writer, if (last_snapshot->get_file_meta(filename, &local_meta) != 0) { continue; } - if (!local_meta.has_checksum() || local_meta.checksum() != remote_meta.checksum()) { + if (!local_meta.has_checksum() || + local_meta.checksum() != remote_meta.checksum()) { continue; } LOG(INFO) << "Found the same file=" << filename << " checksum=" << remote_meta.checksum() << " in last_snapshot=" << last_snapshot->get_path(); if (local_meta.source() == braft::FILE_SOURCE_LOCAL) { - std::string source_path = last_snapshot->get_path() + '/' - + filename; - std::string dest_path = writer->get_path() + '/' - + filename; + std::string source_path = + last_snapshot->get_path() + '/' + filename; + std::string dest_path = writer->get_path() + '/' + filename; _fs->delete_file(dest_path, false); if (!_fs->link(source_path, dest_path)) { - PLOG(ERROR) << "Fail to link " << source_path - << " to " << dest_path; + PLOG(ERROR) + << "Fail to link " << source_path << " to " << dest_path; continue; } // Don't delete linked file @@ -919,7 +925,8 @@ int LocalSnapshotCopier::filter_before_copy(LocalSnapshotWriter* writer, } void LocalSnapshotCopier::filter() { - _writer = (LocalSnapshotWriter*)_storage->create(!_filter_before_copy_remote); + _writer = + (LocalSnapshotWriter*)_storage->create(!_filter_before_copy_remote); if (_writer == NULL) { set_error(EIO, "Fail to create snapshot writer"); return; @@ -929,7 +936,8 @@ void LocalSnapshotCopier::filter() { SnapshotReader* reader = _storage->open(); if (filter_before_copy(_writer, reader) != 0) { LOG(WARNING) << "Fail to filter writer before copying" - ", path: " << _writer->get_path() + ", path: " + << _writer->get_path() << ", destroy and create a new writer"; _writer->set_error(-1, "Fail to filter"); _storage->close(_writer, false); @@ -963,17 +971,16 @@ void LocalSnapshotCopier::copy_file(const std::string& filename) { bool rc = false; if (FLAGS_raft_create_parent_directories) { butil::FilePath sub_dir = - butil::FilePath(_writer->get_path()).Append(sub_path.DirName()); + butil::FilePath(_writer->get_path()).Append(sub_path.DirName()); rc = _fs->create_directory(sub_dir.value(), &e, true); } else { - rc = create_sub_directory( - _writer->get_path(), sub_path.DirName().value(), _fs, &e); + rc = create_sub_directory(_writer->get_path(), + sub_path.DirName().value(), _fs, &e); } if (!rc) { - LOG(ERROR) << "Fail to create directory for " << file_path - << " : " << butil::File::ErrorToString(e); - set_error(file_error_to_os_error(e), - "Fail to create directory"); + LOG(ERROR) << "Fail to create directory for " << file_path << " : " + << butil::File::ErrorToString(e); + set_error(file_error_to_os_error(e), "Fail to create directory"); } } LocalFileMeta meta; @@ -983,8 +990,8 @@ void LocalSnapshotCopier::copy_file(const std::string& filename) { set_error(ECANCELED, "%s", berror(ECANCELED)); return; } - scoped_refptr session - = _copier.start_to_copy_to_file(filename, file_path, NULL); + scoped_refptr session = + _copier.start_to_copy_to_file(filename, file_path, NULL); if (session == NULL) { LOG(WARNING) << "Fail to copy " << filename << " path: " << _writer->get_path(); @@ -998,9 +1005,9 @@ void LocalSnapshotCopier::copy_file(const std::string& filename) { _cur_session = NULL; lck.unlock(); if (!session->status().ok()) { - set_error(session->status().error_code(), "%s", - session->status().error_cstr()); - return; + set_error(session->status().error_code(), "%s", + session->status().error_cstr()); + return; } if (_writer->add_file(filename, &meta) != 0) { set_error(EIO, "Fail to add file to writer"); @@ -1013,16 +1020,13 @@ void LocalSnapshotCopier::copy_file(const std::string& filename) { } void LocalSnapshotCopier::start() { - if (bthread_start_background( - &_tid, NULL, start_copy, this) != 0) { + if (bthread_start_background(&_tid, NULL, start_copy, this) != 0) { PLOG(ERROR) << "Fail to start bthread"; copy(); } } -void LocalSnapshotCopier::join() { - bthread_join(_tid, NULL); -} +void LocalSnapshotCopier::join() { bthread_join(_tid, NULL); } void LocalSnapshotCopier::cancel() { BAIDU_SCOPED_LOCK(_mutex); diff --git a/src/braft/snapshot.h b/src/braft/snapshot.h index 8d617d9..d7cf356 100644 --- a/src/braft/snapshot.h +++ b/src/braft/snapshot.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,27 +21,28 @@ #define BRAFT_RAFT_SNAPSHOT_H #include -#include "braft/storage.h" -#include "braft/macros.h" -#include "braft/local_file_meta.pb.h" + #include "braft/file_system_adaptor.h" +#include "braft/local_file_meta.pb.h" +#include "braft/macros.h" #include "braft/remote_file_copier.h" #include "braft/snapshot_throttle.h" +#include "braft/storage.h" namespace braft { class LocalSnapshotMetaTable { -public: + public: LocalSnapshotMetaTable(); ~LocalSnapshotMetaTable(); // Add file to the meta - int add_file(const std::string& filename, - const LocalFileMeta& file_meta); + int add_file(const std::string& filename, const LocalFileMeta& file_meta); int remove_file(const std::string& filename); int save_to_file(FileSystemAdaptor* fs, const std::string& path) const; int load_from_file(FileSystemAdaptor* fs, const std::string& path); - int get_file_meta(const std::string& filename, LocalFileMeta* file_meta) const; - void list_files(std::vector *files) const; + int get_file_meta(const std::string& filename, + LocalFileMeta* file_meta) const; + void list_files(std::vector* files) const; bool has_meta() { return _meta.IsInitialized(); } const SnapshotMeta& meta() { return _meta; } void set_meta(const SnapshotMeta& meta) { _meta = meta; } @@ -51,16 +52,18 @@ class LocalSnapshotMetaTable { _file_map.swap(rhs._file_map); _meta.Swap(&rhs._meta); } -private: + + private: // Intentionally copyable typedef std::map Map; - Map _file_map; + Map _file_map; SnapshotMeta _meta; }; class LocalSnapshotWriter : public SnapshotWriter { -friend class LocalSnapshotStorage; -public: + friend class LocalSnapshotStorage; + + public: int64_t snapshot_index(); virtual int init(); virtual int save_meta(const SnapshotMeta& meta); @@ -68,24 +71,24 @@ friend class LocalSnapshotStorage; // Add file to the snapshot. It would fail it the file doesn't exist nor // references to any other file. // Returns 0 on success, -1 otherwise. - virtual int add_file(const std::string& filename, + virtual int add_file(const std::string& filename, const ::google::protobuf::Message* file_meta); // Remove a file from the snapshot, it doesn't guarantees that the real file // would be removed from the storage. virtual int remove_file(const std::string& filename); // List all the existing files in the Snapshot currently - virtual void list_files(std::vector *files); + virtual void list_files(std::vector* files); // Get the implementation-defined file_meta - virtual int get_file_meta(const std::string& filename, + virtual int get_file_meta(const std::string& filename, ::google::protobuf::Message* file_meta); // Sync meta table to disk int sync(); FileSystemAdaptor* file_system() { return _fs.get(); } -private: + + private: // Users shouldn't create LocalSnapshotWriter Directly - LocalSnapshotWriter(const std::string& path, - FileSystemAdaptor* fs); + LocalSnapshotWriter(const std::string& path, FileSystemAdaptor* fs); virtual ~LocalSnapshotWriter(); std::string _path; @@ -93,9 +96,10 @@ friend class LocalSnapshotStorage; scoped_refptr _fs; }; -class LocalSnapshotReader: public SnapshotReader { -friend class LocalSnapshotStorage; -public: +class LocalSnapshotReader : public SnapshotReader { + friend class LocalSnapshotStorage; + + public: int64_t snapshot_index(); virtual int init(); virtual int load_meta(SnapshotMeta* meta); @@ -105,15 +109,15 @@ friend class LocalSnapshotStorage; // Return an empty string if some error has occcured virtual std::string generate_uri_for_copy(); // List all the existing files in the Snapshot currently - virtual void list_files(std::vector *files); + virtual void list_files(std::vector* files); // Get the implementation-defined file_meta - virtual int get_file_meta(const std::string& filename, + virtual int get_file_meta(const std::string& filename, ::google::protobuf::Message* file_meta); -private: + + private: // Users shouldn't create LocalSnapshotReader Directly - LocalSnapshotReader(const std::string& path, - butil::EndPoint server_addr, + LocalSnapshotReader(const std::string& path, butil::EndPoint server_addr, FileSystemAdaptor* fs, SnapshotThrottle* snapshot_throttle); virtual ~LocalSnapshotReader(); @@ -129,23 +133,26 @@ friend class LocalSnapshotStorage; // Describe the Snapshot on another machine class LocalSnapshot : public Snapshot { -friend class LocalSnapshotCopier; -public: + friend class LocalSnapshotCopier; + + public: // Get the path of the Snapshot virtual std::string get_path(); // List all the existing files in the Snapshot currently - virtual void list_files(std::vector *files); + virtual void list_files(std::vector* files); // Get the implementation-defined file_meta - virtual int get_file_meta(const std::string& filename, + virtual int get_file_meta(const std::string& filename, ::google::protobuf::Message* file_meta); -private: + + private: LocalSnapshotMetaTable _meta_table; }; class LocalSnapshotStorage; class LocalSnapshotCopier : public SnapshotCopier { -friend class LocalSnapshotStorage; -public: + friend class LocalSnapshotStorage; + + public: LocalSnapshotCopier(); LocalSnapshotCopier(bool copy_file); ~LocalSnapshotCopier(); @@ -153,12 +160,13 @@ friend class LocalSnapshotStorage; virtual void join(); virtual SnapshotReader* get_reader() { return _reader; } int init(const std::string& uri); -private: + + private: static void* start_copy(void* arg); void start(); void copy(); void load_meta_table(); - int filter_before_copy(LocalSnapshotWriter* writer, + int filter_before_copy(LocalSnapshotWriter* writer, SnapshotReader* last_snapshot); void filter(); void copy_file(const std::string& filename); @@ -179,10 +187,11 @@ friend class LocalSnapshotStorage; }; class LocalSnapshotStorage : public SnapshotStorage { -friend class LocalSnapshotCopier; -public: + friend class LocalSnapshotCopier; + + public: explicit LocalSnapshotStorage(const std::string& path); - + LocalSnapshotStorage() {} virtual ~LocalSnapshotStorage(); @@ -194,7 +203,8 @@ friend class LocalSnapshotCopier; virtual SnapshotReader* open() WARN_UNUSED_RESULT; virtual int close(SnapshotReader* reader); - virtual SnapshotReader* copy_from(const std::string& uri) WARN_UNUSED_RESULT; + virtual SnapshotReader* copy_from(const std::string& uri) + WARN_UNUSED_RESULT; virtual SnapshotCopier* start_to_copy_from(const std::string& uri); virtual int close(SnapshotCopier* copier); virtual int set_filter_before_copy_remote(); @@ -203,11 +213,12 @@ friend class LocalSnapshotCopier; SnapshotStorage* new_instance(const std::string& uri) const; butil::Status gc_instance(const std::string& uri) const; - + void set_server_addr(butil::EndPoint server_addr) { _addr = server_addr; } bool has_server_addr() { return _addr != butil::EndPoint(); } void set_copy_file(bool copy_file) { _copy_file = copy_file; } -private: + + private: SnapshotWriter* create(bool from_empty) WARN_UNUSED_RESULT; int destroy_snapshot(const std::string& path); int close(SnapshotWriter* writer, bool keep_data_on_error); @@ -227,4 +238,4 @@ friend class LocalSnapshotCopier; } // namespace braft -#endif //~BRAFT_RAFT_SNAPSHOT_H +#endif //~BRAFT_RAFT_SNAPSHOT_H diff --git a/src/braft/snapshot_executor.cpp b/src/braft/snapshot_executor.cpp index d0227ce..a687f0b 100644 --- a/src/braft/snapshot_executor.cpp +++ b/src/braft/snapshot_executor.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,89 +15,84 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) #include "braft/snapshot_executor.h" -#include "braft/util.h" + #include "braft/node.h" -#include "braft/storage.h" #include "braft/snapshot.h" +#include "braft/storage.h" +#include "braft/util.h" namespace braft { -DEFINE_int32(raft_do_snapshot_min_index_gap, 1, +DEFINE_int32(raft_do_snapshot_min_index_gap, 1, "Will do snapshot only when actual gap between applied_index and" " last_snapshot_index is equal to or larger than this value"); BRPC_VALIDATE_GFLAG(raft_do_snapshot_min_index_gap, brpc::PositiveInteger); class SaveSnapshotDone : public SaveSnapshotClosure { -public: - SaveSnapshotDone(SnapshotExecutor* node, SnapshotWriter* writer, Closure* done); + public: + SaveSnapshotDone(SnapshotExecutor* node, SnapshotWriter* writer, + Closure* done); virtual ~SaveSnapshotDone(); SnapshotWriter* start(const SnapshotMeta& meta); virtual void Run(); - -private: + + private: static void* continue_run(void* arg); SnapshotExecutor* _se; SnapshotWriter* _writer; - Closure* _done; // user done + Closure* _done; // user done SnapshotMeta _meta; }; class InstallSnapshotDone : public LoadSnapshotClosure { -public: - InstallSnapshotDone(SnapshotExecutor* se, - SnapshotReader* reader); + public: + InstallSnapshotDone(SnapshotExecutor* se, SnapshotReader* reader); virtual ~InstallSnapshotDone(); SnapshotReader* start(); virtual void Run(); -private: + + private: SnapshotExecutor* _se; SnapshotReader* _reader; }; class FirstSnapshotLoadDone : public LoadSnapshotClosure { -public: - FirstSnapshotLoadDone(SnapshotExecutor* se, - SnapshotReader* reader) - : _se(se) - , _reader(reader) { - } - ~FirstSnapshotLoadDone() { - } + public: + FirstSnapshotLoadDone(SnapshotExecutor* se, SnapshotReader* reader) + : _se(se), _reader(reader) {} + ~FirstSnapshotLoadDone() {} SnapshotReader* start() { return _reader; } void Run() { _se->on_snapshot_load_done(status()); _event.signal(); } - void wait_for_run() { - _event.wait(); - } -private: + void wait_for_run() { _event.wait(); } + + private: SnapshotExecutor* _se; SnapshotReader* _reader; bthread::CountdownEvent _event; }; SnapshotExecutor::SnapshotExecutor() - : _last_snapshot_term(0) - , _last_snapshot_index(0) - , _term(0) - , _saving_snapshot(false) - , _loading_snapshot(false) - , _stopped(false) - , _snapshot_storage(NULL) - , _cur_copier(NULL) - , _fsm_caller(NULL) - , _node(NULL) - , _log_manager(NULL) - , _downloading_snapshot(NULL) - , _running_jobs(0) - , _snapshot_throttle(NULL) -{ -} + : _last_snapshot_term(0), + _last_snapshot_index(0), + _term(0), + _saving_snapshot(false), + _loading_snapshot(false), + _stopped(false), + _snapshot_storage(NULL), + _cur_copier(NULL), + _fsm_caller(NULL), + _node(NULL), + _log_manager(NULL), + _downloading_snapshot(NULL), + _running_jobs(0), + _snapshot_throttle(NULL) {} SnapshotExecutor::~SnapshotExecutor() { shutdown(); @@ -143,18 +138,19 @@ void SnapshotExecutor::do_snapshot(Closure* done) { return; } int64_t saved_fsm_applied_index = _fsm_caller->last_applied_index(); - if (saved_fsm_applied_index - _last_snapshot_index < - FLAGS_raft_do_snapshot_min_index_gap) { + if (saved_fsm_applied_index - _last_snapshot_index < + FLAGS_raft_do_snapshot_min_index_gap) { // There might be false positive as the last_applied_index() is being // updated. But it's fine since we will do next snapshot saving in a // predictable time. lck.unlock(); _log_manager->clear_bufferred_logs(); - LOG_IF(INFO, _node != NULL) << "node " << _node->node_id() + LOG_IF(INFO, _node != NULL) + << "node " << _node->node_id() << " the gap between fsm applied index " << saved_fsm_applied_index << " and last_snapshot_index " << saved_last_snapshot_index - << ", last_snapshot_term " << saved_last_snapshot_term + << ", last_snapshot_term " << saved_last_snapshot_term << " is less than " << FLAGS_raft_do_snapshot_min_index_gap << ", will clear bufferred logs and return success"; @@ -163,7 +159,7 @@ void SnapshotExecutor::do_snapshot(Closure* done) { } return; } - + SnapshotWriter* writer = _snapshot_storage->create(); if (!writer) { lck.unlock(); @@ -175,11 +171,13 @@ void SnapshotExecutor::do_snapshot(Closure* done) { return; } _saving_snapshot = true; - SaveSnapshotDone* snapshot_save_done = new SaveSnapshotDone(this, writer, done); + SaveSnapshotDone* snapshot_save_done = + new SaveSnapshotDone(this, writer, done); if (_fsm_caller->on_snapshot_save(snapshot_save_done) != 0) { lck.unlock(); if (done) { - snapshot_save_done->status().set_error(EHOSTDOWN, "The raft node is down"); + snapshot_save_done->status().set_error(EHOSTDOWN, + "The raft node is down"); run_closure_in_bthread(snapshot_save_done, _usercode_in_pthread); } return; @@ -187,27 +185,30 @@ void SnapshotExecutor::do_snapshot(Closure* done) { _running_jobs.add_count(1); } -int SnapshotExecutor::on_snapshot_save_done( - const butil::Status& st, const SnapshotMeta& meta, SnapshotWriter* writer) { +int SnapshotExecutor::on_snapshot_save_done(const butil::Status& st, + const SnapshotMeta& meta, + SnapshotWriter* writer) { std::unique_lock lck(_mutex); int ret = st.error_code(); - // InstallSnapshot can break SaveSnapshot, check InstallSnapshot when SaveSnapshot - // because upstream Snapshot maybe newer than local Snapshot. + // InstallSnapshot can break SaveSnapshot, check InstallSnapshot when + // SaveSnapshot because upstream Snapshot maybe newer than local Snapshot. if (st.ok()) { if (meta.last_included_index() <= _last_snapshot_index) { ret = ESTALE; - LOG_IF(WARNING, _node != NULL) << "node " << _node->node_id() - << " discards an stale snapshot " + LOG_IF(WARNING, _node != NULL) + << "node " << _node->node_id() << " discards an stale snapshot " << " last_included_index " << meta.last_included_index() << " last_snapshot_index " << _last_snapshot_index; - writer->set_error(ESTALE, "Installing snapshot is older than local snapshot"); + writer->set_error( + ESTALE, "Installing snapshot is older than local snapshot"); } } lck.unlock(); - + if (ret == 0) { if (writer->save_meta(meta)) { - LOG(WARNING) << "node " << _node->node_id() << " fail to save snapshot"; + LOG(WARNING) << "node " << _node->node_id() + << " fail to save snapshot"; ret = EIO; } } else { @@ -230,8 +231,9 @@ int SnapshotExecutor::on_snapshot_save_done( _last_snapshot_index = meta.last_included_index(); _last_snapshot_term = meta.last_included_term(); lck.unlock(); - ss << "snapshot_save_done, last_included_index=" << meta.last_included_index() - << " last_included_term=" << meta.last_included_term(); + ss << "snapshot_save_done, last_included_index=" + << meta.last_included_index() + << " last_included_term=" << meta.last_included_term(); LOG(INFO) << ss.str(); _log_manager->set_snapshot(&meta); lck.lock(); @@ -249,7 +251,8 @@ void SnapshotExecutor::on_snapshot_load_done(const butil::Status& st) { std::unique_lock lck(_mutex); CHECK(_loading_snapshot); - DownloadingSnapshot* m = _downloading_snapshot.load(butil::memory_order_relaxed); + DownloadingSnapshot* m = + _downloading_snapshot.load(butil::memory_order_relaxed); if (st.ok()) { _last_snapshot_index = _loading_snapshot_meta.last_included_index(); @@ -260,8 +263,7 @@ void SnapshotExecutor::on_snapshot_load_done(const butil::Status& st) { if (_node) { ss << "node " << _node->node_id() << ' '; } - ss << "snapshot_load_done, " - << _loading_snapshot_meta.ShortDebugString(); + ss << "snapshot_load_done, " << _loading_snapshot_meta.ShortDebugString(); LOG(INFO) << ss.str(); lck.unlock(); if (_node) { @@ -285,8 +287,7 @@ void SnapshotExecutor::on_snapshot_load_done(const butil::Status& st) { _running_jobs.signal(); } -SaveSnapshotDone::SaveSnapshotDone(SnapshotExecutor* se, - SnapshotWriter* writer, +SaveSnapshotDone::SaveSnapshotDone(SnapshotExecutor* se, SnapshotWriter* writer, Closure* done) : _se(se), _writer(writer), _done(done) { // here AddRef, SaveSnapshot maybe async @@ -310,12 +311,12 @@ void* SaveSnapshotDone::continue_run(void* arg) { SaveSnapshotDone* self = (SaveSnapshotDone*)arg; std::unique_ptr self_guard(self); // Must call on_snapshot_save_done to clear _saving_snapshot - int ret = self->_se->on_snapshot_save_done( - self->status(), self->_meta, self->_writer); + int ret = self->_se->on_snapshot_save_done(self->status(), self->_meta, + self->_writer); if (ret != 0 && self->status().ok()) { self->status().set_error(ret, "node call on_snapshot_save_done failed"); } - //user done, need set error + // user done, need set error if (self->_done) { self->_done->status() = self->status(); } @@ -351,8 +352,8 @@ int SnapshotExecutor::init(const SnapshotExecutorOptions& options) { _snapshot_storage = SnapshotStorage::create(options.uri); if (!_snapshot_storage) { - LOG(ERROR) << "node " << _node->node_id() - << " fail to find snapshot storage, uri " << options.uri; + LOG(ERROR) << "node " << _node->node_id() + << " fail to find snapshot storage, uri " << options.uri; return -1; } if (options.filter_before_copy_remote) { @@ -366,11 +367,12 @@ int SnapshotExecutor::init(const SnapshotExecutorOptions& options) { _snapshot_storage->set_snapshot_throttle(options.snapshot_throttle); } if (_snapshot_storage->init() != 0) { - LOG(ERROR) << "node " << _node->node_id() + LOG(ERROR) << "node " << _node->node_id() << " fail to init snapshot storage, uri " << options.uri; return -1; } - LocalSnapshotStorage* tmp = dynamic_cast(_snapshot_storage); + LocalSnapshotStorage* tmp = + dynamic_cast(_snapshot_storage); if (tmp != NULL && !tmp->has_server_addr()) { tmp->set_server_addr(options.addr); } @@ -408,7 +410,7 @@ void SnapshotExecutor::install_snapshot(brpc::Controller* cntl, brpc::ClosureGuard done_guard(done); SnapshotMeta meta = request->meta(); - // check if install_snapshot tasks num exceeds threshold + // check if install_snapshot tasks num exceeds threshold if (_snapshot_throttle && !_snapshot_throttle->add_one_more_task(false)) { LOG(WARNING) << "Fail to install snapshot"; cntl->SetFailed(EBUSY, "Fail to add install_snapshot tasks now"); @@ -443,7 +445,8 @@ void SnapshotExecutor::install_snapshot(brpc::Controller* cntl, done_guard.release(); CHECK(_cur_copier); _cur_copier->join(); - // when copying finished or canceled, more install_snapshot tasks are allowed + // when copying finished or canceled, more install_snapshot tasks are + // allowed if (_snapshot_throttle) { _snapshot_throttle->finish_one_task(false); } @@ -460,8 +463,8 @@ void SnapshotExecutor::load_downloading_snapshot(DownloadingSnapshot* ds, SnapshotReader* reader = _cur_copier->get_reader(); if (!_cur_copier->ok()) { if (_cur_copier->error_code() == EIO) { - report_error(_cur_copier->error_code(), - "%s", _cur_copier->error_cstr()); + report_error(_cur_copier->error_code(), "%s", + _cur_copier->error_cstr()); } if (reader) { _snapshot_storage->close(reader); @@ -484,8 +487,7 @@ void SnapshotExecutor::load_downloading_snapshot(DownloadingSnapshot* ds, } _downloading_snapshot.store(NULL, butil::memory_order_release); lck.unlock(); - ds->cntl->SetFailed(brpc::EINTERNAL, - "Fail to copy snapshot from %s", + ds->cntl->SetFailed(brpc::EINTERNAL, "Fail to copy snapshot from %s", ds->request->uri().c_str()); _running_jobs.signal(); return; @@ -498,11 +500,13 @@ void SnapshotExecutor::load_downloading_snapshot(DownloadingSnapshot* ds, _loading_snapshot_meta = meta; lck.unlock(); InstallSnapshotDone* install_snapshot_done = - new InstallSnapshotDone(this, reader); + new InstallSnapshotDone(this, reader); int ret = _fsm_caller->on_snapshot_load(install_snapshot_done); if (ret != 0) { - LOG(WARNING) << "node " << _node->node_id() << " fail to call on_snapshot_load"; - install_snapshot_done->status().set_error(EHOSTDOWN, "This raft node is down"); + LOG(WARNING) << "node " << _node->node_id() + << " fail to call on_snapshot_load"; + install_snapshot_done->status().set_error(EHOSTDOWN, + "This raft node is down"); return install_snapshot_done->Run(); } } @@ -532,8 +536,8 @@ int SnapshotExecutor::register_downloading_snapshot(DownloadingSnapshot* ds) { ds->cntl->SetFailed(EBUSY, "Is saving snapshot"); return -1; } - DownloadingSnapshot* m = _downloading_snapshot.load( - butil::memory_order_relaxed); + DownloadingSnapshot* m = + _downloading_snapshot.load(butil::memory_order_relaxed); if (!m) { _downloading_snapshot.store(ds, butil::memory_order_relaxed); // Now this session has the right to download the snapshot. @@ -556,8 +560,8 @@ int SnapshotExecutor::register_downloading_snapshot(DownloadingSnapshot* ds) { // A previouse snapshot is under installing, check if this is the same // snapshot and resume it, otherwise drop previous snapshot as this one is // newer - if (m->request->meta().last_included_index() - == ds->request->meta().last_included_index()) { + if (m->request->meta().last_included_index() == + ds->request->meta().last_included_index()) { // m is a retry has_saved = true; // Copy |*ds| to |*m| so that the former session would respond @@ -565,8 +569,8 @@ int SnapshotExecutor::register_downloading_snapshot(DownloadingSnapshot* ds) { saved = *m; *m = *ds; rc = 1; - } else if (m->request->meta().last_included_index() - > ds->request->meta().last_included_index()) { + } else if (m->request->meta().last_included_index() > + ds->request->meta().last_included_index()) { // |ds| is older LOG(WARNING) << "Register failed: is installing a newer one."; ds->cntl->SetFailed(EINVAL, "A newer snapshot is under installing"); @@ -581,18 +585,21 @@ int SnapshotExecutor::register_downloading_snapshot(DownloadingSnapshot* ds) { } CHECK(_cur_copier); _cur_copier->cancel(); - LOG(WARNING) << "Register failed: an older snapshot is under installing," - " cancle downloading."; - ds->cntl->SetFailed(EBUSY, "A former snapshot is under installing, " - " trying to cancel"); + LOG(WARNING) + << "Register failed: an older snapshot is under installing," + " cancle downloading."; + ds->cntl->SetFailed(EBUSY, + "A former snapshot is under installing, " + " trying to cancel"); return -1; } lck.unlock(); if (has_saved) { // Respond replaced session - LOG(WARNING) << "Register failed: interrupted by retry installing request."; + LOG(WARNING) + << "Register failed: interrupted by retry installing request."; saved.cntl->SetFailed( - EINTR, "Interrupted by the retry InstallSnapshotRequest"); + EINTR, "Interrupted by the retry InstallSnapshotRequest"); saved.done->Run(); } return rc; @@ -615,9 +622,9 @@ void SnapshotExecutor::interrupt_downloading_snapshot(int64_t new_term) { if (_node) { ss << "node " << _node->node_id() << ' '; } - ss << "Trying to cancel downloading snapshot : " + ss << "Trying to cancel downloading snapshot : " << _downloading_snapshot.load(butil::memory_order_relaxed) - ->request->ShortDebugString(); + ->request->ShortDebugString(); LOG(INFO) << ss.str(); } @@ -631,7 +638,7 @@ void SnapshotExecutor::report_error(int error_code, const char* fmt, ...) { _fsm_caller->on_error(e); } -void SnapshotExecutor::describe(std::ostream&os, bool use_html) { +void SnapshotExecutor::describe(std::ostream& os, bool use_html) { SnapshotMeta meta; InstallSnapshotRequest request; std::unique_lock lck(_mutex); @@ -641,19 +648,19 @@ void SnapshotExecutor::describe(std::ostream&os, bool use_html) { if (is_loading_snapshot) { meta = _loading_snapshot_meta; // ^ - // Cloning Configuration is expansive, since snapshot is not the hot spot, - // we think it's fine + // Cloning Configuration is expansive, since snapshot is not the hot + // spot, we think it's fine } - const DownloadingSnapshot* m = - _downloading_snapshot.load(butil::memory_order_acquire); + const DownloadingSnapshot* m = + _downloading_snapshot.load(butil::memory_order_acquire); if (m) { request.CopyFrom(*m->request); - // ^ It's also a little expansive, but fine + // ^ It's also a little expansive, but fine } const bool is_saving_snapshot = _saving_snapshot; // TODO: add timestamp of snapshot lck.unlock(); - const char *newline = use_html ? "
" : "\r\n"; + const char* newline = use_html ? "
" : "\r\n"; os << "last_snapshot_index: " << last_snapshot_index << newline; os << "last_snapshot_term: " << last_snapshot_term << newline; if (m && is_loading_snapshot) { @@ -665,7 +672,8 @@ void SnapshotExecutor::describe(std::ostream&os, bool use_html) { CHECK(!is_saving_snapshot); os << "snapshot_status: DOWNLOADING" << newline; os << "downloading_snapshot_from: " << request.uri() << newline; - os << "downloading_snapshot_meta: " << request.meta().ShortDebugString(); + os << "downloading_snapshot_meta: " + << request.meta().ShortDebugString(); } else if (is_saving_snapshot) { os << "snapshot_status: SAVING" << newline; } else { @@ -686,10 +694,11 @@ void SnapshotExecutor::join() { _running_jobs.wait(); } -InstallSnapshotDone::InstallSnapshotDone(SnapshotExecutor* se, +InstallSnapshotDone::InstallSnapshotDone(SnapshotExecutor* se, SnapshotReader* reader) - : _se(se) , _reader(reader) { - // node not need AddRef, FSMCaller::shutdown will flush running InstallSnapshot task + : _se(se), _reader(reader) { + // node not need AddRef, FSMCaller::shutdown will flush running + // InstallSnapshot task } InstallSnapshotDone::~InstallSnapshotDone() { @@ -698,9 +707,7 @@ InstallSnapshotDone::~InstallSnapshotDone() { } } -SnapshotReader* InstallSnapshotDone::start() { - return _reader; -} +SnapshotReader* InstallSnapshotDone::start() { return _reader; } void InstallSnapshotDone::Run() { _se->on_snapshot_load_done(status()); diff --git a/src/braft/snapshot_executor.h b/src/braft/snapshot_executor.h index d587950..51e6f29 100644 --- a/src/braft/snapshot_executor.h +++ b/src/braft/snapshot_executor.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,18 +14,18 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) -#ifndef BRAFT_SNAPSHOT_EXECUTOR_H -#define BRAFT_SNAPSHOT_EXECUTOR_H +#ifndef BRAFT_SNAPSHOT_EXECUTOR_H +#define BRAFT_SNAPSHOT_EXECUTOR_H #include +#include "braft/fsm_caller.h" +#include "braft/log_manager.h" #include "braft/raft.h" -#include "braft/util.h" +#include "braft/raft.pb.h" #include "braft/snapshot.h" #include "braft/storage.h" -#include "braft/raft.pb.h" -#include "braft/fsm_caller.h" -#include "braft/log_manager.h" +#include "braft/util.h" namespace braft { class NodeImpl; @@ -36,7 +36,7 @@ struct SnapshotExecutorOptions { SnapshotExecutorOptions(); // URI of SnapshotStorage std::string uri; - + FSMCaller* fsm_caller; NodeImpl* node; LogManager* log_manager; @@ -52,7 +52,8 @@ struct SnapshotExecutorOptions { // Executing Snapshot related stuff class BAIDU_CACHELINE_ALIGNMENT SnapshotExecutor { DISALLOW_COPY_AND_ASSIGN(SnapshotExecutor); -public: + + public: SnapshotExecutor(); ~SnapshotExecutor(); @@ -68,9 +69,9 @@ class BAIDU_CACHELINE_ALIGNMENT SnapshotExecutor { // Install snapshot according to the very RPC from leader // After the installing succeeds (StateMachine is reset with the snapshot) // or fails, done will be called to respond - // + // // Errors: - // - Term dismatches: which happens interrupt_downloading_snapshot was + // - Term dismatches: which happens interrupt_downloading_snapshot was // called before install_snapshot, indicating that this RPC was issued by // the old leader. // - Interrupted: happens when interrupt_downloading_snapshot is called or @@ -86,9 +87,9 @@ class BAIDU_CACHELINE_ALIGNMENT SnapshotExecutor { // happens when receiving RPC from new peer. In this case, it's hard to // determine whether to keep downloading snapshot as the new leader // possibly contains the missing logs and is going to send AppendEntries. To - // make things simplicity and leader changing during snapshot installing is + // make things simplicity and leader changing during snapshot installing is // very rare. So we interrupt snapshot downloading when leader changes, and - // let the new leader decide whether to install a new snapshot or continue + // let the new leader decide whether to install a new snapshot or continue // appending log entries. // // NOTE: we can't interrupt the snapshot insalling which has finsihed @@ -97,12 +98,12 @@ class BAIDU_CACHELINE_ALIGNMENT SnapshotExecutor { // Return true if this is currently installing a snapshot, either // downloading or loading. - bool is_installing_snapshot() const { + bool is_installing_snapshot() const { // 1: acquire fence makes this thread sees the latest change when seeing // the lastest _loading_snapshot - // _downloading_snapshot is NULL when then downloading was successfully + // _downloading_snapshot is NULL when then downloading was successfully // interrupted or installing has finished - return _downloading_snapshot.load(butil::memory_order_acquire/*1*/); + return _downloading_snapshot.load(butil::memory_order_acquire /*1*/); } // Return the backing snapshot storage @@ -117,14 +118,13 @@ class BAIDU_CACHELINE_ALIGNMENT SnapshotExecutor { // failure) void join(); -private: -friend class SaveSnapshotDone; -friend class FirstSnapshotLoadDone; -friend class InstallSnapshotDone; + private: + friend class SaveSnapshotDone; + friend class FirstSnapshotLoadDone; + friend class InstallSnapshotDone; void on_snapshot_load_done(const butil::Status& st); - int on_snapshot_save_done(const butil::Status& st, - const SnapshotMeta& meta, + int on_snapshot_save_done(const butil::Status& st, const SnapshotMeta& meta, SnapshotWriter* writer); struct DownloadingSnapshot { @@ -135,11 +135,10 @@ friend class InstallSnapshotDone; }; int register_downloading_snapshot(DownloadingSnapshot* ds); - int parse_install_snapshot_request( - const InstallSnapshotRequest* request, - SnapshotMeta* meta); + int parse_install_snapshot_request(const InstallSnapshotRequest* request, + SnapshotMeta* meta); void load_downloading_snapshot(DownloadingSnapshot* ds, - const SnapshotMeta& meta); + const SnapshotMeta& meta); void report_error(int error_code, const char* fmt, ...); raft_mutex_t _mutex; @@ -166,14 +165,13 @@ friend class InstallSnapshotDone; scoped_refptr _snapshot_throttle; }; -inline SnapshotExecutorOptions::SnapshotExecutorOptions() - : fsm_caller(NULL) - , node(NULL) - , log_manager(NULL) - , init_term(0) - , filter_before_copy_remote(false) - , usercode_in_pthread(false) -{} +inline SnapshotExecutorOptions::SnapshotExecutorOptions() + : fsm_caller(NULL), + node(NULL), + log_manager(NULL), + init_term(0), + filter_before_copy_remote(false), + usercode_in_pthread(false) {} } // namespace braft diff --git a/src/braft/snapshot_throttle.cpp b/src/braft/snapshot_throttle.cpp index 9920021..9569f50 100644 --- a/src/braft/snapshot_throttle.cpp +++ b/src/braft/snapshot_throttle.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,10 +14,12 @@ // Authors: Xiong,Kai(xiongkai@baidu.com) +#include "braft/snapshot_throttle.h" + +#include #include #include -#include -#include "braft/snapshot_throttle.h" + #include "braft/util.h" namespace braft { @@ -26,50 +28,51 @@ namespace braft { // threshold is too small in extreme cases. // notice that this flag does not distinguish disk types(sata or ssd, and so on) DEFINE_int64(raft_minimal_throttle_threshold_mb, 0, - "minimal throttle throughput threshold per second"); + "minimal throttle throughput threshold per second"); BRPC_VALIDATE_GFLAG(raft_minimal_throttle_threshold_mb, brpc::NonNegativeInteger); -DEFINE_int32(raft_max_install_snapshot_tasks_num, 1000, +DEFINE_int32(raft_max_install_snapshot_tasks_num, 1000, "Max num of install_snapshot tasks per disk at the same time"); -BRPC_VALIDATE_GFLAG(raft_max_install_snapshot_tasks_num, - brpc::PositiveInteger); +BRPC_VALIDATE_GFLAG(raft_max_install_snapshot_tasks_num, brpc::PositiveInteger); ThroughputSnapshotThrottle::ThroughputSnapshotThrottle( - int64_t throttle_throughput_bytes, int64_t check_cycle) - : _throttle_throughput_bytes(throttle_throughput_bytes) - , _check_cycle(check_cycle) - , _snapshot_task_num(0) - , _last_throughput_check_time_us( - caculate_check_time_us(butil::cpuwide_time_us(), check_cycle)) - , _cur_throughput_bytes(0) -{} + int64_t throttle_throughput_bytes, int64_t check_cycle) + : _throttle_throughput_bytes(throttle_throughput_bytes), + _check_cycle(check_cycle), + _snapshot_task_num(0), + _last_throughput_check_time_us( + caculate_check_time_us(butil::cpuwide_time_us(), check_cycle)), + _cur_throughput_bytes(0) {} ThroughputSnapshotThrottle::~ThroughputSnapshotThrottle() {} size_t ThroughputSnapshotThrottle::throttled_by_throughput(int64_t bytes) { size_t available_size = bytes; int64_t now = butil::cpuwide_time_us(); - int64_t limit_throughput_bytes_s = std::max(_throttle_throughput_bytes, - FLAGS_raft_minimal_throttle_threshold_mb * 1024 *1024); + int64_t limit_throughput_bytes_s = + std::max(_throttle_throughput_bytes, + FLAGS_raft_minimal_throttle_threshold_mb * 1024 * 1024); int64_t limit_per_cycle = limit_throughput_bytes_s / _check_cycle; std::unique_lock lck(_mutex); if (_cur_throughput_bytes + bytes > limit_per_cycle) { // reading another |bytes| excceds the limit - if (now - _last_throughput_check_time_us <= + if (now - _last_throughput_check_time_us <= 1 * 1000 * 1000 / _check_cycle) { // if time interval is less than or equal to a cycle, read more data // to make full use of the throughput of current cycle. - available_size = limit_per_cycle > _cur_throughput_bytes ? limit_per_cycle - _cur_throughput_bytes : 0; + available_size = limit_per_cycle > _cur_throughput_bytes + ? limit_per_cycle - _cur_throughput_bytes + : 0; _cur_throughput_bytes = limit_per_cycle; } else { // otherwise, read the data in the next cycle. available_size = bytes > limit_per_cycle ? limit_per_cycle : bytes; _cur_throughput_bytes = available_size; - _last_throughput_check_time_us = + _last_throughput_check_time_us = caculate_check_time_us(now, _check_cycle); } } else { - // reading another |bytes| doesn't excced limit(less than or equal to), + // reading another |bytes| doesn't excced limit(less than or equal to), // put it in current cycle available_size = bytes; _cur_throughput_bytes += available_size; @@ -78,7 +81,7 @@ size_t ThroughputSnapshotThrottle::throttled_by_throughput(int64_t bytes) { return available_size; } -bool ThroughputSnapshotThrottle::add_one_more_task(bool is_leader) { +bool ThroughputSnapshotThrottle::add_one_more_task(bool is_leader) { // Don't throttle leader, let follower do it if (is_leader) { return true; @@ -88,13 +91,15 @@ bool ThroughputSnapshotThrottle::add_one_more_task(bool is_leader) { int saved_task_num = _snapshot_task_num; if (_snapshot_task_num >= task_num_threshold) { lck.unlock(); - LOG(WARNING) << "Fail to add one more task when current task num is: " - << saved_task_num << ", task num threshold: " << task_num_threshold; + LOG(WARNING) << "Fail to add one more task when current task num is: " + << saved_task_num + << ", task num threshold: " << task_num_threshold; return false; } saved_task_num = ++_snapshot_task_num; lck.unlock(); - LOG(INFO) << "Succed to add one more task, new task num is: " << saved_task_num + LOG(INFO) << "Succed to add one more task, new task num is: " + << saved_task_num << ", task num threshold: " << task_num_threshold; return true; } @@ -106,23 +111,23 @@ void ThroughputSnapshotThrottle::finish_one_task(bool is_leader) { std::unique_lock lck(_mutex); int saved_task_num = --_snapshot_task_num; // _snapshot_task_num should not be negative - CHECK_GE(_snapshot_task_num, 0) << "Finishing task cause wrong task num: " - << saved_task_num; + CHECK_GE(_snapshot_task_num, 0) + << "Finishing task cause wrong task num: " << saved_task_num; lck.unlock(); LOG(INFO) << "Finish one task, new task num is: " << saved_task_num; return; } void ThroughputSnapshotThrottle::return_unused_throughput( - int64_t acquired, int64_t consumed, int64_t elaspe_time_us) { + int64_t acquired, int64_t consumed, int64_t elaspe_time_us) { int64_t now = butil::cpuwide_time_us(); std::unique_lock lck(_mutex); if (now - elaspe_time_us < _last_throughput_check_time_us) { // Tokens are aqured in last cycle, ignore return; } - _cur_throughput_bytes = std::max( - _cur_throughput_bytes - (acquired - consumed), int64_t(0)); + _cur_throughput_bytes = + std::max(_cur_throughput_bytes - (acquired - consumed), int64_t(0)); } } // namespace braft diff --git a/src/braft/snapshot_throttle.h b/src/braft/snapshot_throttle.h index ed2dd4e..85d02a5 100644 --- a/src/braft/snapshot_throttle.h +++ b/src/braft/snapshot_throttle.h @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,20 +14,22 @@ // Authors: Xiong,Kai(xiongkai@baidu.com) -#ifndef BRAFT_SNAPSHOT_THROTTLE_H -#define BRAFT_SNAPSHOT_THROTTLE_H +#ifndef BRAFT_SNAPSHOT_THROTTLE_H +#define BRAFT_SNAPSHOT_THROTTLE_H + +#include // butil::RefCountedThreadSafe -#include // butil::RefCountedThreadSafe #include "braft/util.h" namespace braft { -// Abstract class with the function of throttling during heavy disk reading/writing +// Abstract class with the function of throttling during heavy disk +// reading/writing class SnapshotThrottle : public butil::RefCountedThreadSafe { -public: + public: SnapshotThrottle() {} virtual ~SnapshotThrottle() {} - // Get available throughput after throttled + // Get available throughput after throttled // Must be thread-safe virtual size_t throttled_by_throughput(int64_t bytes) = 0; virtual bool add_one_more_task(bool is_leader) = 0; @@ -37,32 +39,36 @@ class SnapshotThrottle : public butil::RefCountedThreadSafe { // After a throttled request finish, |return_unused_throughput| is called to // return back unsed tokens (throghput). Default implementation do nothing. // There are two situations we can optimize: - // case 1: The follower and leader both try to throttle the same request, and - // only one of them permit the request. No actual IO and bandwidth consumed, - // the acquired tokens are wasted. - // case 2: We acquired some tokens, but only part of them are used, because of + // case 1: The follower and leader both try to throttle the same request, + // and + // only one of them permit the request. No actual IO and bandwidth + // consumed, the acquired tokens are wasted. + // case 2: We acquired some tokens, but only part of them are used, because + // of // the file reach the eof, or the file contains holes. - virtual void return_unused_throughput( - int64_t acquired, int64_t consumed, int64_t elaspe_time_us) {} -private: + virtual void return_unused_throughput(int64_t acquired, int64_t consumed, + int64_t elaspe_time_us) {} + + private: DISALLOW_COPY_AND_ASSIGN(SnapshotThrottle); friend class butil::RefCountedThreadSafe; }; // SnapshotThrottle with throughput threshold used in install_snapshot class ThroughputSnapshotThrottle : public SnapshotThrottle { -public: - ThroughputSnapshotThrottle(int64_t throttle_throughput_bytes, int64_t check_cycle); + public: + ThroughputSnapshotThrottle(int64_t throttle_throughput_bytes, + int64_t check_cycle); int64_t get_throughput() const { return _throttle_throughput_bytes; } int64_t get_cycle() const { return _check_cycle; } size_t throttled_by_throughput(int64_t bytes); bool add_one_more_task(bool is_leader); void finish_one_task(bool is_leader); - int64_t get_retry_interval_ms() { return 1000 / _check_cycle + 1;} - void return_unused_throughput( - int64_t acquired, int64_t consumed, int64_t elaspe_time_us); + int64_t get_retry_interval_ms() { return 1000 / _check_cycle + 1; } + void return_unused_throughput(int64_t acquired, int64_t consumed, + int64_t elaspe_time_us); -private: + private: ~ThroughputSnapshotThrottle(); // user defined throughput threshold for raft, bytes per second int64_t _throttle_throughput_bytes; @@ -75,12 +81,12 @@ class ThroughputSnapshotThrottle : public SnapshotThrottle { raft_mutex_t _mutex; }; -inline int64_t caculate_check_time_us(int64_t current_time_us, - int64_t check_cycle) { +inline int64_t caculate_check_time_us(int64_t current_time_us, + int64_t check_cycle) { int64_t base_aligning_time_us = 1000 * 1000 / check_cycle; return current_time_us / base_aligning_time_us * base_aligning_time_us; } -} // namespace braft +} // namespace braft #endif // BRAFT_SNAPSHOT_THROTTLE_H diff --git a/src/braft/storage.cpp b/src/braft/storage.cpp index 8630eb3..605b102 100644 --- a/src/braft/storage.cpp +++ b/src/braft/storage.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,13 +15,14 @@ // Authors: Zhangyi Chen(chenzhangyi01@baidu.com) // Wang,Yao(wangyao02@baidu.com) -#include +#include "braft/storage.h" + +#include +#include #include #include -#include -#include +#include -#include "braft/storage.h" #include "braft/log.h" #include "braft/raft_meta.h" #include "braft/snapshot.h" @@ -36,9 +37,11 @@ BRPC_VALIDATE_GFLAG(raft_sync_per_bytes, ::brpc::NonNegativeInteger); DEFINE_bool(raft_create_parent_directories, true, "Create parent directories of the path in local storage if true"); DEFINE_int32(raft_sync_policy, 0, - "raft sync policy when raft_sync set to true, 0 mean sync immediately, 1 mean sync by " + "raft sync policy when raft_sync set to true, 0 mean sync " + "immediately, 1 mean sync by " "writed bytes"); -DEFINE_bool(raft_sync_meta, false, "sync log meta, snapshot meta and raft meta"); +DEFINE_bool(raft_sync_meta, false, + "sync log meta, snapshot meta and raft meta"); BRPC_VALIDATE_GFLAG(raft_sync_meta, ::brpc::PassValidate); LogStorage* LogStorage::create(const std::string& uri) { @@ -49,8 +52,8 @@ LogStorage* LogStorage::create(const std::string& uri) { LOG(ERROR) << "Invalid log storage uri=`" << uri << '\''; return NULL; } - const LogStorage* type = log_storage_extension()->Find( - protocol.as_string().c_str()); + const LogStorage* type = + log_storage_extension()->Find(protocol.as_string().c_str()); if (type == NULL) { LOG(ERROR) << "Fail to find log storage type " << protocol << ", uri=" << uri; @@ -69,12 +72,12 @@ butil::Status LogStorage::destroy(const std::string& uri) { status.set_error(EINVAL, "Invalid log storage uri = %s", uri.c_str()); return status; } - const LogStorage* type = log_storage_extension()->Find( - protocol.as_string().c_str()); + const LogStorage* type = + log_storage_extension()->Find(protocol.as_string().c_str()); if (type == NULL) { LOG(ERROR) << "Fail to find log storage type " << protocol << ", uri=" << uri; - status.set_error(EINVAL, "Fail to find log storage type %s uri %s", + status.set_error(EINVAL, "Fail to find log storage type %s uri %s", protocol.as_string().c_str(), uri.c_str()); return status; } @@ -89,8 +92,8 @@ SnapshotStorage* SnapshotStorage::create(const std::string& uri) { LOG(ERROR) << "Invalid snapshot storage uri=`" << uri << '\''; return NULL; } - const SnapshotStorage* type = snapshot_storage_extension()->Find( - protocol.as_string().c_str()); + const SnapshotStorage* type = + snapshot_storage_extension()->Find(protocol.as_string().c_str()); if (type == NULL) { LOG(ERROR) << "Fail to find snapshot storage type " << protocol << ", uri=" << uri; @@ -109,12 +112,12 @@ butil::Status SnapshotStorage::destroy(const std::string& uri) { status.set_error(EINVAL, "Invalid log storage uri = %s", uri.c_str()); return status; } - const SnapshotStorage* type = snapshot_storage_extension()->Find( - protocol.as_string().c_str()); + const SnapshotStorage* type = + snapshot_storage_extension()->Find(protocol.as_string().c_str()); if (type == NULL) { LOG(ERROR) << "Fail to find snapshot storage type " << protocol << ", uri=" << uri; - status.set_error(EINVAL, "Fail to find snapshot storage type %s uri %s", + status.set_error(EINVAL, "Fail to find snapshot storage type %s uri %s", protocol.as_string().c_str(), uri.c_str()); return status; } @@ -129,8 +132,8 @@ RaftMetaStorage* RaftMetaStorage::create(const std::string& uri) { LOG(ERROR) << "Invalid meta storage uri=`" << uri << '\''; return NULL; } - const RaftMetaStorage* type = meta_storage_extension()->Find( - protocol.as_string().c_str()); + const RaftMetaStorage* type = + meta_storage_extension()->Find(protocol.as_string().c_str()); if (type == NULL) { LOG(ERROR) << "Fail to find meta storage type " << protocol << ", uri=" << uri; @@ -150,12 +153,12 @@ butil::Status RaftMetaStorage::destroy(const std::string& uri, status.set_error(EINVAL, "Invalid meta storage uri = %s", uri.c_str()); return status; } - const RaftMetaStorage* type = meta_storage_extension()->Find( - protocol.as_string().c_str()); + const RaftMetaStorage* type = + meta_storage_extension()->Find(protocol.as_string().c_str()); if (type == NULL) { LOG(ERROR) << "Fail to find meta storage type " << protocol << ", uri=" << uri; - status.set_error(EINVAL, "Fail to find meta storage type %s uri %s", + status.set_error(EINVAL, "Fail to find meta storage type %s uri %s", protocol.as_string().c_str(), uri.c_str()); return status; } diff --git a/src/braft/storage.h b/src/braft/storage.h index bdb5989..2749043 100644 --- a/src/braft/storage.h +++ b/src/braft/storage.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,13 +18,15 @@ #ifndef BRAFT_RAFT_STORAGE_H #define BRAFT_RAFT_STORAGE_H -#include -#include -#include -#include -#include #include +#include +#include #include +#include + +#include +#include + #include "braft/configuration.h" #include "braft/configuration_manager.h" @@ -45,13 +47,13 @@ DECLARE_bool(raft_create_parent_directories); struct LogEntry; struct IOMetric { -public: - IOMetric() - : start_time_us(butil::cpuwide_time_us()) - , bthread_queue_time_us(0) - , open_segment_time_us(0) - , append_entry_time_us(0) - , sync_segment_time_us(0) {} + public: + IOMetric() + : start_time_us(butil::cpuwide_time_us()), + bthread_queue_time_us(0), + open_segment_time_us(0), + append_entry_time_us(0), + sync_segment_time_us(0) {} int64_t start_time_us; int64_t bthread_queue_time_us; @@ -62,33 +64,34 @@ struct IOMetric { inline std::ostream& operator<<(std::ostream& os, const IOMetric& m) { return os << " bthread_queue_time_us: " << m.bthread_queue_time_us - << " open_segment_time_us: " << m.open_segment_time_us + << " open_segment_time_us: " << m.open_segment_time_us << " append_entry_time_us: " << m.append_entry_time_us << " sync_segment_time_us: " << m.sync_segment_time_us; } -inline butil::StringPiece parse_uri(butil::StringPiece* uri, std::string* parameter) { +inline butil::StringPiece parse_uri(butil::StringPiece* uri, + std::string* parameter) { // ${protocol}://${parameters} size_t pos = uri->find("://"); if (pos == butil::StringPiece::npos) { return butil::StringPiece(); } butil::StringPiece protocol = uri->substr(0, pos); - uri->remove_prefix(pos + 3/* length of '://' */); + uri->remove_prefix(pos + 3 /* length of '://' */); protocol.trim_spaces(); parameter->reserve(uri->size()); parameter->clear(); size_t removed_spaces = 0; - for (butil::StringPiece::const_iterator - iter = uri->begin(); iter != uri->end(); ++iter) { + for (butil::StringPiece::const_iterator iter = uri->begin(); + iter != uri->end(); ++iter) { if (!isspace(*iter)) { parameter->push_back(*iter); } else { ++removed_spaces; } } - LOG_IF(WARNING, removed_spaces) << "Removed " << removed_spaces - << " spaces from `" << *uri << '\''; + LOG_IF(WARNING, removed_spaces) + << "Removed " << removed_spaces << " spaces from `" << *uri << '\''; return protocol; } @@ -104,10 +107,10 @@ inline int gc_dir(const std::string& path) { if (butil::PathExists(target_path)) { const bool rc = butil::ReplaceFile(butil::FilePath(target_path), - butil::FilePath(tmp_path), &e); + butil::FilePath(tmp_path), &e); if (!rc) { - LOG(ERROR) << "Fail to rename `" << target_path.value() - << " to `" << tmp_path.value() << "' : " << e; + LOG(ERROR) << "Fail to rename `" << target_path.value() << " to `" + << tmp_path.value() << "' : " << e; return -1; } if (!butil::DeleteFile(tmp_path, true)) { @@ -115,14 +118,14 @@ inline int gc_dir(const std::string& path) { return -1; } } else { - LOG(INFO) << "Target path not exist, so no need to gc, path: " + LOG(INFO) << "Target path not exist, so no need to gc, path: " << target_path.value(); } - return 0; + return 0; } class LogStorage { -public: + public: virtual ~LogStorage() {} // init logstorage, check consistency and integrity @@ -143,82 +146,87 @@ class LogStorage { // append entries to log virtual int append_entry(const LogEntry* entry) = 0; - // append entries to log and update IOMetric, return append success number - virtual int append_entries(const std::vector& entries, IOMetric* metric) = 0; + // append entries to log and update IOMetric, return append success number + virtual int append_entries(const std::vector& entries, + IOMetric* metric) = 0; - // delete logs from storage's head, [first_log_index, first_index_kept) will be discarded + // delete logs from storage's head, [first_log_index, first_index_kept) will + // be discarded virtual int truncate_prefix(const int64_t first_index_kept) = 0; - // delete uncommitted logs from storage's tail, (last_index_kept, last_log_index] will be discarded + // delete uncommitted logs from storage's tail, (last_index_kept, + // last_log_index] will be discarded virtual int truncate_suffix(const int64_t last_index_kept) = 0; // Drop all the existing logs and reset next log index to |next_log_index|. // This function is called after installing snapshot from leader virtual int reset(const int64_t next_log_index) = 0; - // Create an instance of this kind of LogStorage with the parameters encoded + // Create an instance of this kind of LogStorage with the parameters encoded // in |uri| // Return the address referenced to the instance on success, NULL otherwise. virtual LogStorage* new_instance(const std::string& uri) const = 0; static LogStorage* create(const std::string& uri); - // GC an instance of this kind of LogStorage with the parameters encoded + // GC an instance of this kind of LogStorage with the parameters encoded // in |uri| virtual butil::Status gc_instance(const std::string& uri) const { CHECK(false) << butil::class_name_str(*this) << " didn't implement gc_instance interface while deleting" - " raft log in " << uri; + " raft log in " + << uri; butil::Status status; status.set_error(ENOSYS, "gc_instance interface is not implemented"); return status; } - + static butil::Status destroy(const std::string& uri); }; class RaftMetaStorage { -public: + public: virtual ~RaftMetaStorage() {} // init stable storage virtual butil::Status init() = 0; // set term and votedfor information - virtual butil::Status set_term_and_votedfor(const int64_t term, - const PeerId& peer_id, const VersionedGroupId& group) = 0; + virtual butil::Status set_term_and_votedfor( + const int64_t term, const PeerId& peer_id, + const VersionedGroupId& group) = 0; // get term and votedfor information - virtual butil::Status get_term_and_votedfor(int64_t* term, PeerId* peer_id, - const VersionedGroupId& group) = 0; + virtual butil::Status get_term_and_votedfor( + int64_t* term, PeerId* peer_id, const VersionedGroupId& group) = 0; - // Create an instance of this kind of RaftMetaStorage with the parameters encoded - // in |uri| - // Return the address referenced to the instance on success, NULL otherwise. + // Create an instance of this kind of RaftMetaStorage with the parameters + // encoded in |uri| Return the address referenced to the instance on + // success, NULL otherwise. virtual RaftMetaStorage* new_instance(const std::string& uri) const = 0; static RaftMetaStorage* create(const std::string& uri); - - // GC an instance of this kind of StableStorage with the parameters encoded + + // GC an instance of this kind of StableStorage with the parameters encoded // in |uri| - virtual butil::Status gc_instance(const std::string& uri, - const VersionedGroupId& vgid) const { + virtual butil::Status gc_instance(const std::string& uri, + const VersionedGroupId& vgid) const { CHECK(false) << butil::class_name_str(*this) << " didn't implement gc_instance interface while deleting" - " raft stable meta in " << uri; + " raft stable meta in " + << uri; butil::Status status; status.set_error(ENOSYS, "gc_instance interface is not implemented"); return status; } - - static butil::Status destroy(const std::string& uri, - const VersionedGroupId& vgid); + static butil::Status destroy(const std::string& uri, + const VersionedGroupId& vgid); }; -// Snapshot +// Snapshot class Snapshot : public butil::Status { -public: + public: Snapshot() {} virtual ~Snapshot() {} @@ -226,10 +234,10 @@ class Snapshot : public butil::Status { virtual std::string get_path() = 0; // List all the existing files in the Snapshot currently - virtual void list_files(std::vector *files) = 0; + virtual void list_files(std::vector* files) = 0; // Get the implementation-defined file_meta - virtual int get_file_meta(const std::string& filename, + virtual int get_file_meta(const std::string& filename, ::google::protobuf::Message* file_meta) { (void)filename; if (file_meta != NULL) { @@ -240,7 +248,7 @@ class Snapshot : public butil::Status { }; class SnapshotWriter : public Snapshot { -public: + public: SnapshotWriter() {} virtual ~SnapshotWriter() {} @@ -249,16 +257,16 @@ class SnapshotWriter : public Snapshot { virtual int save_meta(const SnapshotMeta& meta) = 0; // Add a file to the snapshot. - // |file_meta| is an implmentation-defined protobuf message + // |file_meta| is an implmentation-defined protobuf message // All the implementation must handle the case that |file_meta| is NULL and // no error can be raised. // Note that whether the file will be created onto the backing storage is // implementation-defined. - virtual int add_file(const std::string& filename) { + virtual int add_file(const std::string& filename) { return add_file(filename, NULL); } - virtual int add_file(const std::string& filename, + virtual int add_file(const std::string& filename, const ::google::protobuf::Message* file_meta) = 0; // Remove a file from the snapshot @@ -268,11 +276,11 @@ class SnapshotWriter : public Snapshot { }; class SnapshotReader : public Snapshot { -public: + public: SnapshotReader() {} virtual ~SnapshotReader() {} - // Load meta from + // Load meta from virtual int load_meta(SnapshotMeta* meta) = 0; // Generate uri for other peers to copy this snapshot. @@ -282,7 +290,7 @@ class SnapshotReader : public Snapshot { // Copy Snapshot from the given resource class SnapshotCopier : public butil::Status { -public: + public: virtual ~SnapshotCopier() {} // Cancel the copy job virtual void cancel() = 0; @@ -297,25 +305,25 @@ class FileSystemAdaptor; class SnapshotThrottle; class SnapshotStorage { -public: + public: virtual ~SnapshotStorage() {} virtual int set_filter_before_copy_remote() { - CHECK(false) << butil::class_name_str(*this) + CHECK(false) << butil::class_name_str(*this) << " doesn't support filter before copy remote"; return -1; } virtual int set_file_system_adaptor(FileSystemAdaptor* fs) { (void)fs; - CHECK(false) << butil::class_name_str(*this) + CHECK(false) << butil::class_name_str(*this) << " doesn't support file system adaptor"; return -1; } virtual int set_snapshot_throttle(SnapshotThrottle* st) { (void)st; - CHECK(false) << butil::class_name_str(*this) + CHECK(false) << butil::class_name_str(*this) << " doesn't support snapshot throttle"; return -1; } @@ -336,22 +344,25 @@ class SnapshotStorage { virtual int close(SnapshotReader* reader) = 0; // Copy snapshot from uri and open it as a SnapshotReader - virtual SnapshotReader* copy_from(const std::string& uri) WARN_UNUSED_RESULT = 0; + virtual SnapshotReader* copy_from(const std::string& uri) + WARN_UNUSED_RESULT = 0; virtual SnapshotCopier* start_to_copy_from(const std::string& uri) = 0; virtual int close(SnapshotCopier* copier) = 0; - // Create an instance of this kind of SnapshotStorage with the parameters encoded - // in |uri| - // Return the address referenced to the instance on success, NULL otherwise. - virtual SnapshotStorage* new_instance(const std::string& uri) const WARN_UNUSED_RESULT = 0; + // Create an instance of this kind of SnapshotStorage with the parameters + // encoded in |uri| Return the address referenced to the instance on + // success, NULL otherwise. + virtual SnapshotStorage* new_instance(const std::string& uri) const + WARN_UNUSED_RESULT = 0; static SnapshotStorage* create(const std::string& uri); - - // GC an instance of this kind of SnapshotStorage with the parameters encoded - // in |uri| + + // GC an instance of this kind of SnapshotStorage with the parameters + // encoded in |uri| virtual butil::Status gc_instance(const std::string& uri) const { CHECK(false) << butil::class_name_str(*this) << " didn't implement gc_instance interface while deleting" - " raft snapshot in " << uri; + " raft snapshot in " + << uri; butil::Status status; status.set_error(ENOSYS, "gc_instance interface is not implemented"); return status; @@ -372,6 +383,6 @@ inline brpc::Extension* snapshot_storage_extension() { return brpc::Extension::instance(); } -} +} // namespace braft -#endif //~BRAFT_RAFT_STORAGE_H +#endif //~BRAFT_RAFT_STORAGE_H diff --git a/src/braft/sync_point.cpp b/src/braft/sync_point.cpp index 518fc78..48ee3ce 100644 --- a/src/braft/sync_point.cpp +++ b/src/braft/sync_point.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,8 @@ #include "sync_point.h" +#include + #include #include #include @@ -28,81 +30,80 @@ #include #include -#include - #ifndef NDEBUG namespace braft { struct SyncPoint::Data { - Data() : enabled_(false) {} - // Enable proper deletion by subclasses - virtual ~Data() {} - // successor/predecessor map loaded from LoadDependency - std::unordered_map> successors_; - std::unordered_map> predecessors_; - std::unordered_map> callbacks_; - std::unordered_map> markers_; - std::unordered_map marked_thread_id_; - - std::mutex mutex_; - std::condition_variable cv_; - // sync points that have been passed through - std::unordered_set cleared_points_; - std::atomic enabled_; - int num_callbacks_running_ = 0; - - void LoadDependency(const std::vector &dependencies); - void LoadDependencyAndMarkers(const std::vector &dependencies, - const std::vector &markers); - bool PredecessorsAllCleared(const std::string &point); - void SetCallBack(const std::string &point, - const std::function &callback) { - std::lock_guard lock(mutex_); - callbacks_[point] = callback; - } - - void ClearCallBack(const std::string &point); - void ClearAllCallBacks(); - void EnableProcessing() { enabled_ = true; } - void DisableProcessing() { enabled_ = false; } - void ClearTrace() { - std::lock_guard lock(mutex_); - cleared_points_.clear(); - } - bool DisabledByMarker(const std::string &point, std::thread::id thread_id) { - auto marked_point_iter = marked_thread_id_.find(point); - return marked_point_iter != marked_thread_id_.end() && - thread_id != marked_point_iter->second; - } - void Process(const std::string &point, void *cb_arg); + Data() : enabled_(false) {} + // Enable proper deletion by subclasses + virtual ~Data() {} + // successor/predecessor map loaded from LoadDependency + std::unordered_map> successors_; + std::unordered_map> predecessors_; + std::unordered_map> callbacks_; + std::unordered_map> markers_; + std::unordered_map marked_thread_id_; + + std::mutex mutex_; + std::condition_variable cv_; + // sync points that have been passed through + std::unordered_set cleared_points_; + std::atomic enabled_; + int num_callbacks_running_ = 0; + + void LoadDependency(const std::vector& dependencies); + void LoadDependencyAndMarkers( + const std::vector& dependencies, + const std::vector& markers); + bool PredecessorsAllCleared(const std::string& point); + void SetCallBack(const std::string& point, + const std::function& callback) { + std::lock_guard lock(mutex_); + callbacks_[point] = callback; + } + + void ClearCallBack(const std::string& point); + void ClearAllCallBacks(); + void EnableProcessing() { enabled_ = true; } + void DisableProcessing() { enabled_ = false; } + void ClearTrace() { + std::lock_guard lock(mutex_); + cleared_points_.clear(); + } + bool DisabledByMarker(const std::string& point, std::thread::id thread_id) { + auto marked_point_iter = marked_thread_id_.find(point); + return marked_point_iter != marked_thread_id_.end() && + thread_id != marked_point_iter->second; + } + void Process(const std::string& point, void* cb_arg); }; -SyncPoint *SyncPoint::GetInstance() { - static SyncPoint sync_point; - return &sync_point; +SyncPoint* SyncPoint::GetInstance() { + static SyncPoint sync_point; + return &sync_point; } SyncPoint::SyncPoint() : impl_(new Data) {} SyncPoint::~SyncPoint() { delete impl_; } -void SyncPoint::LoadDependency(const std::vector &dependencies) { - impl_->LoadDependency(dependencies); +void SyncPoint::LoadDependency(const std::vector& dependencies) { + impl_->LoadDependency(dependencies); } void SyncPoint::LoadDependencyAndMarkers( - const std::vector &dependencies, - const std::vector &markers) { - impl_->LoadDependencyAndMarkers(dependencies, markers); + const std::vector& dependencies, + const std::vector& markers) { + impl_->LoadDependencyAndMarkers(dependencies, markers); } -void SyncPoint::SetCallBack(const std::string &point, - const std::function &callback) { - impl_->SetCallBack(point, callback); +void SyncPoint::SetCallBack(const std::string& point, + const std::function& callback) { + impl_->SetCallBack(point, callback); } -void SyncPoint::ClearCallBack(const std::string &point) { - impl_->ClearCallBack(point); +void SyncPoint::ClearCallBack(const std::string& point) { + impl_->ClearCallBack(point); } void SyncPoint::ClearAllCallBacks() { impl_->ClearAllCallBacks(); } @@ -113,105 +114,105 @@ void SyncPoint::DisableProcessing() { impl_->DisableProcessing(); } void SyncPoint::ClearTrace() { impl_->ClearTrace(); } -void SyncPoint::Process(const std::string &point, void *cb_arg) { - impl_->Process(point, cb_arg); +void SyncPoint::Process(const std::string& point, void* cb_arg) { + impl_->Process(point, cb_arg); } void SyncPoint::Data::LoadDependency( - const std::vector &dependencies) { - std::lock_guard lock(mutex_); - successors_.clear(); - predecessors_.clear(); - cleared_points_.clear(); - for (const auto &dependency : dependencies) { - successors_[dependency.predecessor].push_back(dependency.successor); - predecessors_[dependency.successor].push_back(dependency.predecessor); - } - cv_.notify_all(); + const std::vector& dependencies) { + std::lock_guard lock(mutex_); + successors_.clear(); + predecessors_.clear(); + cleared_points_.clear(); + for (const auto& dependency : dependencies) { + successors_[dependency.predecessor].push_back(dependency.successor); + predecessors_[dependency.successor].push_back(dependency.predecessor); + } + cv_.notify_all(); } void SyncPoint::Data::LoadDependencyAndMarkers( - const std::vector &dependencies, - const std::vector &markers) { - std::lock_guard lock(mutex_); - successors_.clear(); - predecessors_.clear(); - cleared_points_.clear(); - markers_.clear(); - marked_thread_id_.clear(); - for (const auto &dependency : dependencies) { - successors_[dependency.predecessor].push_back(dependency.successor); - predecessors_[dependency.successor].push_back(dependency.predecessor); - } - for (const auto &marker : markers) { - successors_[marker.predecessor].push_back(marker.successor); - predecessors_[marker.successor].push_back(marker.predecessor); - markers_[marker.predecessor].push_back(marker.successor); - } - cv_.notify_all(); + const std::vector& dependencies, + const std::vector& markers) { + std::lock_guard lock(mutex_); + successors_.clear(); + predecessors_.clear(); + cleared_points_.clear(); + markers_.clear(); + marked_thread_id_.clear(); + for (const auto& dependency : dependencies) { + successors_[dependency.predecessor].push_back(dependency.successor); + predecessors_[dependency.successor].push_back(dependency.predecessor); + } + for (const auto& marker : markers) { + successors_[marker.predecessor].push_back(marker.successor); + predecessors_[marker.successor].push_back(marker.predecessor); + markers_[marker.predecessor].push_back(marker.successor); + } + cv_.notify_all(); } -bool SyncPoint::Data::PredecessorsAllCleared(const std::string &point) { - for (const auto &pred : predecessors_[point]) { - if (cleared_points_.count(pred) == 0) { - return false; +bool SyncPoint::Data::PredecessorsAllCleared(const std::string& point) { + for (const auto& pred : predecessors_[point]) { + if (cleared_points_.count(pred) == 0) { + return false; + } } - } - return true; + return true; } -void SyncPoint::Data::ClearCallBack(const std::string &point) { - std::unique_lock lock(mutex_); - while (num_callbacks_running_ > 0) { - cv_.wait(lock); - } - callbacks_.erase(point); +void SyncPoint::Data::ClearCallBack(const std::string& point) { + std::unique_lock lock(mutex_); + while (num_callbacks_running_ > 0) { + cv_.wait(lock); + } + callbacks_.erase(point); } void SyncPoint::Data::ClearAllCallBacks() { - std::unique_lock lock(mutex_); - while (num_callbacks_running_ > 0) { - cv_.wait(lock); - } - callbacks_.clear(); + std::unique_lock lock(mutex_); + while (num_callbacks_running_ > 0) { + cv_.wait(lock); + } + callbacks_.clear(); } -void SyncPoint::Data::Process(const std::string &point, void *cb_arg) { - if (!enabled_) { - return; - } +void SyncPoint::Data::Process(const std::string& point, void* cb_arg) { + if (!enabled_) { + return; + } - std::unique_lock lock(mutex_); - auto thread_id = std::this_thread::get_id(); + std::unique_lock lock(mutex_); + auto thread_id = std::this_thread::get_id(); - auto marker_iter = markers_.find(point); - if (marker_iter != markers_.end()) { - for (auto &marked_point : marker_iter->second) { - marked_thread_id_.emplace(marked_point, thread_id); + auto marker_iter = markers_.find(point); + if (marker_iter != markers_.end()) { + for (auto& marked_point : marker_iter->second) { + marked_thread_id_.emplace(marked_point, thread_id); + } } - } - - if (DisabledByMarker(point, thread_id)) { - return; - } - while (!PredecessorsAllCleared(point)) { - cv_.wait(lock); if (DisabledByMarker(point, thread_id)) { - return; + return; + } + + while (!PredecessorsAllCleared(point)) { + cv_.wait(lock); + if (DisabledByMarker(point, thread_id)) { + return; + } + } + + auto callback_pair = callbacks_.find(point); + if (callback_pair != callbacks_.end()) { + num_callbacks_running_++; + mutex_.unlock(); + callback_pair->second(cb_arg); + mutex_.lock(); + num_callbacks_running_--; } - } - - auto callback_pair = callbacks_.find(point); - if (callback_pair != callbacks_.end()) { - num_callbacks_running_++; - mutex_.unlock(); - callback_pair->second(cb_arg); - mutex_.lock(); - num_callbacks_running_--; - } - cleared_points_.insert(point); - cv_.notify_all(); + cleared_points_.insert(point); + cv_.notify_all(); } -} // namespace braft -#endif // NDEBUG +} // namespace braft +#endif // NDEBUG diff --git a/src/braft/sync_point.h b/src/braft/sync_point.h index 0f8eb86..77f7bd1 100644 --- a/src/braft/sync_point.h +++ b/src/braft/sync_point.h @@ -1,11 +1,11 @@ // Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,77 +44,79 @@ namespace braft { // threads execution. class SyncPoint { - public: - static SyncPoint* GetInstance(); - - SyncPoint(const SyncPoint&) = delete; - SyncPoint& operator=(const SyncPoint&) = delete; - ~SyncPoint(); - - struct SyncPointPair { - std::string predecessor; - std::string successor; - }; - - // call once at the beginning of a test to setup the dependency between - // sync points. Specifically, execution will not be allowed to proceed past - // each successor until execution has reached the corresponding predecessor, - // in any thread. - void LoadDependency(const std::vector& dependencies); - - // call once at the beginning of a test to setup the dependency between - // sync points and setup markers indicating the successor is only enabled - // when it is processed on the same thread as the predecessor. - // When adding a marker, it implicitly adds a dependency for the marker pair. - void LoadDependencyAndMarkers(const std::vector& dependencies, - const std::vector& markers); - - // The argument to the callback is passed through from - // TEST_SYNC_POINT_CALLBACK(); nullptr if TEST_SYNC_POINT or - // TEST_IDX_SYNC_POINT was used. - void SetCallBack(const std::string& point, - const std::function& callback); - - // Clear callback function by point - void ClearCallBack(const std::string& point); - - // Clear all call back functions. - void ClearAllCallBacks(); - - // enable sync point processing (disabled on startup) - void EnableProcessing(); - - // disable sync point processing - void DisableProcessing(); - - // remove the execution trace of all sync points - void ClearTrace(); - - // triggered by TEST_SYNC_POINT, blocking execution until all predecessors - // are executed. - // And/or call registered callback function, with argument `cb_arg` - void Process(const std::string& point, void* cb_arg = nullptr); - - // template gets length of const string at compile time, - // avoiding strlen() at runtime - template - void Process(const char (&point)[kLen], void* cb_arg = nullptr) { - static_assert(kLen > 0, "Must not be empty"); - assert(point[kLen - 1] == '\0'); - Process(std::string(point, kLen - 1), cb_arg); - } - - // TODO: it might be useful to provide a function that blocks until all - // sync points are cleared. - - // We want this to be public so we can - // subclass the implementation - struct Data; - - private: - // Singleton - SyncPoint(); - Data* impl_; + public: + static SyncPoint* GetInstance(); + + SyncPoint(const SyncPoint&) = delete; + SyncPoint& operator=(const SyncPoint&) = delete; + ~SyncPoint(); + + struct SyncPointPair { + std::string predecessor; + std::string successor; + }; + + // call once at the beginning of a test to setup the dependency between + // sync points. Specifically, execution will not be allowed to proceed past + // each successor until execution has reached the corresponding predecessor, + // in any thread. + void LoadDependency(const std::vector& dependencies); + + // call once at the beginning of a test to setup the dependency between + // sync points and setup markers indicating the successor is only enabled + // when it is processed on the same thread as the predecessor. + // When adding a marker, it implicitly adds a dependency for the marker + // pair. + void LoadDependencyAndMarkers( + const std::vector& dependencies, + const std::vector& markers); + + // The argument to the callback is passed through from + // TEST_SYNC_POINT_CALLBACK(); nullptr if TEST_SYNC_POINT or + // TEST_IDX_SYNC_POINT was used. + void SetCallBack(const std::string& point, + const std::function& callback); + + // Clear callback function by point + void ClearCallBack(const std::string& point); + + // Clear all call back functions. + void ClearAllCallBacks(); + + // enable sync point processing (disabled on startup) + void EnableProcessing(); + + // disable sync point processing + void DisableProcessing(); + + // remove the execution trace of all sync points + void ClearTrace(); + + // triggered by TEST_SYNC_POINT, blocking execution until all predecessors + // are executed. + // And/or call registered callback function, with argument `cb_arg` + void Process(const std::string& point, void* cb_arg = nullptr); + + // template gets length of const string at compile time, + // avoiding strlen() at runtime + template + void Process(const char (&point)[kLen], void* cb_arg = nullptr) { + static_assert(kLen > 0, "Must not be empty"); + assert(point[kLen - 1] == '\0'); + Process(std::string(point, kLen - 1), cb_arg); + } + + // TODO: it might be useful to provide a function that blocks until all + // sync points are cleared. + + // We want this to be public so we can + // subclass the implementation + struct Data; + + private: + // Singleton + SyncPoint(); + Data* impl_; }; // Sets up sync points to mock direct IO instead of actually issuing direct IO @@ -128,15 +130,12 @@ void SetupSyncPointsToMockDirectIO(); // utilized to re-produce race conditions between threads. // See TransactionLogIteratorRace in db_test.cc for an example use case. // TEST_SYNC_POINT is no op in release build. -#define TEST_SYNC_POINT(x) \ - braft::SyncPoint::GetInstance()->Process(x) -#define TEST_IDX_SYNC_POINT(x, index) \ - braft::SyncPoint::GetInstance()->Process(x + \ - std::to_string(index)) +#define TEST_SYNC_POINT(x) braft::SyncPoint::GetInstance()->Process(x) +#define TEST_IDX_SYNC_POINT(x, index) \ + braft::SyncPoint::GetInstance()->Process(x + std::to_string(index)) #define TEST_SYNC_POINT_CALLBACK(x, y) \ - braft::SyncPoint::GetInstance()->Process(x, y) -#define INIT_SYNC_POINT_SINGLETONS() \ - (void)braft::SyncPoint::GetInstance(); + braft::SyncPoint::GetInstance()->Process(x, y) +#define INIT_SYNC_POINT_SINGLETONS() (void)braft::SyncPoint::GetInstance(); #endif // NDEBUG // Callback sync point for any read IO errors that should be ignored by @@ -145,10 +144,10 @@ void SetupSyncPointsToMockDirectIO(); #ifdef NDEBUG #define IGNORE_STATUS_IF_ERROR(_status_) #else -#define IGNORE_STATUS_IF_ERROR(_status_) \ - { \ - if (!_status_.ok()) { \ - TEST_SYNC_POINT("FaultInjectionIgnoreError"); \ - } \ - } +#define IGNORE_STATUS_IF_ERROR(_status_) \ + { \ + if (!_status_.ok()) { \ + TEST_SYNC_POINT("FaultInjectionIgnoreError"); \ + } \ + } #endif // NDEBUG diff --git a/src/braft/util.cpp b/src/braft/util.cpp index 9770b75..680f1ff 100644 --- a/src/braft/util.cpp +++ b/src/braft/util.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,11 +16,13 @@ // Wang,Yao(wangyao02@baidu.com) #include "braft/util.h" + +#include +#include +#include // butil::RawPacker #include #include -#include -#include // butil::RawPacker -#include + #include "braft/raft.h" namespace bvar { @@ -35,12 +37,12 @@ static bool valid_percentile(const char*, int32_t v) { return v > 0 && v < 100; } -const bool ALLOW_UNUSED dummy_bvar_counter_p1 = GFLAGS_NS::RegisterFlagValidator( - &FLAGS_bvar_counter_p1, valid_percentile); -const bool ALLOW_UNUSED dummy_bvar_counter_p2 = GFLAGS_NS::RegisterFlagValidator( - &FLAGS_bvar_counter_p2, valid_percentile); -const bool ALLOW_UNUSED dummy_bvar_counter_p3 = GFLAGS_NS::RegisterFlagValidator( - &FLAGS_bvar_counter_p3, valid_percentile); +const bool ALLOW_UNUSED dummy_bvar_counter_p1 = + GFLAGS_NS::RegisterFlagValidator(&FLAGS_bvar_counter_p1, valid_percentile); +const bool ALLOW_UNUSED dummy_bvar_counter_p2 = + GFLAGS_NS::RegisterFlagValidator(&FLAGS_bvar_counter_p2, valid_percentile); +const bool ALLOW_UNUSED dummy_bvar_counter_p3 = + GFLAGS_NS::RegisterFlagValidator(&FLAGS_bvar_counter_p3, valid_percentile); namespace detail { @@ -71,8 +73,8 @@ static CombinedPercentileSamples* combine(PercentileWindow* w) { template static int64_t get_counter_percetile(void* arg) { - return ((CounterRecorder*)arg)->counter_percentile( - (double)numerator / double(denominator)); + return ((CounterRecorder*)arg) + ->counter_percentile((double)numerator / double(denominator)); } static int64_t get_p1_counter(void* arg) { @@ -88,7 +90,7 @@ static int64_t get_p3_counter(void* arg) { return cr->counter_percentile(FLAGS_bvar_counter_p3 / 100.0); } -static Vector get_counters(void *arg) { +static Vector get_counters(void* arg) { std::unique_ptr cb( combine((PercentileWindow*)arg)); // NOTE: We don't show 99.99% since it's often significantly larger than @@ -103,20 +105,19 @@ static Vector get_counters(void *arg) { } CounterRecorderBase::CounterRecorderBase(time_t window_size) - : _max_counter() - , _avg_counter_window(&_avg_counter, window_size) - , _max_counter_window(&_max_counter, window_size) - , _counter_percentile_window(&_counter_percentile, window_size) - , _total_times(get_recorder_count, &_avg_counter) - , _qps(get_window_recorder_qps, &_avg_counter_window) - , _counter_p1(get_p1_counter, this) - , _counter_p2(get_p2_counter, this) - , _counter_p3(get_p3_counter, this) - , _counter_999(get_counter_percetile<999, 1000>, this) - , _counter_9999(get_counter_percetile<9999, 10000>, this) - , _counter_cdf(&_counter_percentile_window) - , _counter_percentiles(get_counters, &_counter_percentile_window) -{} + : _max_counter(), + _avg_counter_window(&_avg_counter, window_size), + _max_counter_window(&_max_counter, window_size), + _counter_percentile_window(&_counter_percentile, window_size), + _total_times(get_recorder_count, &_avg_counter), + _qps(get_window_recorder_qps, &_avg_counter_window), + _counter_p1(get_p1_counter, this), + _counter_p2(get_p2_counter, this), + _counter_p3(get_p3_counter, this), + _counter_999(get_counter_percetile<999, 1000>, this), + _counter_9999(get_counter_percetile<9999, 10000>, this), + _counter_cdf(&_counter_percentile_window), + _counter_percentiles(get_counters, &_counter_percentile_window) {} } // namespace detail @@ -156,7 +157,7 @@ int CounterRecorder::expose(const butil::StringPiece& prefix1, if (!prefix1.empty()) { tmp.reserve(prefix1.size() + prefix.size() + 1); tmp.append(prefix1.data(), prefix1.size()); - tmp.push_back('_'); // prefix1 ending with _ is good. + tmp.push_back('_'); // prefix1 ending with _ is good. tmp.append(prefix.data(), prefix.size()); prefix = tmp; } @@ -178,19 +179,23 @@ int CounterRecorder::expose(const butil::StringPiece& prefix1, return -1; } char namebuf[32]; - snprintf(namebuf, sizeof(namebuf), "counter_%d", (int)FLAGS_bvar_counter_p1); + snprintf(namebuf, sizeof(namebuf), "counter_%d", + (int)FLAGS_bvar_counter_p1); if (_counter_p1.expose_as(prefix, namebuf, DISPLAY_ON_PLAIN_TEXT) != 0) { return -1; } - snprintf(namebuf, sizeof(namebuf), "counter_%d", (int)FLAGS_bvar_counter_p2); + snprintf(namebuf, sizeof(namebuf), "counter_%d", + (int)FLAGS_bvar_counter_p2); if (_counter_p2.expose_as(prefix, namebuf, DISPLAY_ON_PLAIN_TEXT) != 0) { return -1; } - snprintf(namebuf, sizeof(namebuf), "counter_%u", (int)FLAGS_bvar_counter_p3); + snprintf(namebuf, sizeof(namebuf), "counter_%u", + (int)FLAGS_bvar_counter_p3); if (_counter_p3.expose_as(prefix, namebuf, DISPLAY_ON_PLAIN_TEXT) != 0) { return -1; } - if (_counter_999.expose_as(prefix, "counter_999", DISPLAY_ON_PLAIN_TEXT) != 0) { + if (_counter_999.expose_as(prefix, "counter_999", DISPLAY_ON_PLAIN_TEXT) != + 0) { return -1; } if (_counter_9999.expose_as(prefix, "counter_9999") != 0) { @@ -199,7 +204,8 @@ int CounterRecorder::expose(const butil::StringPiece& prefix1, if (_counter_cdf.expose_as(prefix, "counter_cdf", DISPLAY_ON_HTML) != 0) { return -1; } - if (_counter_percentiles.expose_as(prefix, "counter_percentiles", DISPLAY_ON_HTML) != 0) { + if (_counter_percentiles.expose_as(prefix, "counter_percentiles", + DISPLAY_ON_HTML) != 0) { return -1; } snprintf(namebuf, sizeof(namebuf), "%d%%,%d%%,%d%%,99.9%%", @@ -237,19 +243,17 @@ CounterRecorder& CounterRecorder::operator<<(int64_t count_num) { } std::ostream& operator<<(std::ostream& os, const CounterRecorder& rec) { - return os << "{avg=" << rec.avg_counter() - << " max" << rec.window_size() << '=' << rec.max_counter() - << " qps=" << rec.qps() + return os << "{avg=" << rec.avg_counter() << " max" << rec.window_size() + << '=' << rec.max_counter() << " qps=" << rec.qps() << " count=" << rec.total_times() << '}'; } } // namespace bvar - namespace braft { static void* run_closure(void* arg) { - ::google::protobuf::Closure *c = (google::protobuf::Closure*)arg; + ::google::protobuf::Closure* c = (google::protobuf::Closure*)arg; if (c) { c->Run(); } @@ -260,8 +264,8 @@ void run_closure_in_bthread(google::protobuf::Closure* closure, bool in_pthread) { DCHECK(closure); bthread_t tid; - bthread_attr_t attr = (in_pthread) - ? BTHREAD_ATTR_PTHREAD : BTHREAD_ATTR_NORMAL; + bthread_attr_t attr = + (in_pthread) ? BTHREAD_ATTR_PTHREAD : BTHREAD_ATTR_NORMAL; int ret = bthread_start_background(&tid, &attr, run_closure, closure); if (0 != ret) { PLOG(ERROR) << "Fail to start bthread"; @@ -273,9 +277,9 @@ void run_closure_in_bthread_nosig(google::protobuf::Closure* closure, bool in_pthread) { DCHECK(closure); bthread_t tid; - bthread_attr_t attr = (in_pthread) - ? BTHREAD_ATTR_PTHREAD : BTHREAD_ATTR_NORMAL; - attr = attr | BTHREAD_NOSIGNAL; + bthread_attr_t attr = + (in_pthread) ? BTHREAD_ATTR_PTHREAD : BTHREAD_ATTR_NORMAL; + attr = attr | BTHREAD_NOSIGNAL; int ret = bthread_start_background(&tid, &attr, run_closure, closure); if (0 != ret) { PLOG(ERROR) << "Fail to start bthread"; @@ -288,7 +292,7 @@ ssize_t file_pread(butil::IOPortal* portal, int fd, off_t offset, size_t size) { ssize_t left = size; while (left > 0) { ssize_t read_len = portal->pappend_from_file_descriptor( - fd, offset, static_cast(left)); + fd, offset, static_cast(left)); if (read_len > 0) { left -= read_len; offset += read_len; @@ -297,8 +301,8 @@ ssize_t file_pread(butil::IOPortal* portal, int fd, off_t offset, size_t size) { } else if (errno == EINTR) { continue; } else { - LOG(WARNING) << "read failed, err: " << berror() - << " fd: " << fd << " offset: " << orig_offset << " size: " << size; + LOG(WARNING) << "read failed, err: " << berror() << " fd: " << fd + << " offset: " << orig_offset << " size: " << size; return -1; } } @@ -312,15 +316,16 @@ ssize_t file_pwrite(const butil::IOBuf& data, int fd, off_t offset) { off_t orig_offset = offset; ssize_t left = size; while (left > 0) { - ssize_t written = piece_data.pcut_into_file_descriptor(fd, offset, left); + ssize_t written = + piece_data.pcut_into_file_descriptor(fd, offset, left); if (written >= 0) { offset += written; left -= written; } else if (errno == EINTR) { continue; } else { - LOG(WARNING) << "write falied, err: " << berror() - << " fd: " << fd << " offset: " << orig_offset << " size: " << size; + LOG(WARNING) << "write falied, err: " << berror() << " fd: " << fd + << " offset: " << orig_offset << " size: " << size; return -1; } } @@ -400,7 +405,8 @@ size_t FileSegData::next(uint64_t* offset, butil::IOBuf* data) { *offset = seg_offset; size_t body_len = _data.cutn(data, seg_len); - CHECK_EQ(body_len, seg_len) << "seg_len: " << seg_len << " body_len: " << body_len; + CHECK_EQ(body_len, seg_len) + << "seg_len: " << seg_len << " body_len: " << body_len; return seg_len; } diff --git a/src/braft/util.h b/src/braft/util.h index 50bec34..f2f29a8 100644 --- a/src/braft/util.h +++ b/src/braft/util.h @@ -1,11 +1,11 @@ // Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,29 +18,31 @@ #ifndef BRAFT_RAFT_UTIL_H #define BRAFT_RAFT_UTIL_H -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include #include -#include -#include +#include #include -#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include + +#include +#include + #include "braft/macros.h" #include "braft/raft.h" @@ -55,10 +57,11 @@ typedef Window PercentileWindow; // For mimic constructor inheritance. class CounterRecorderBase { -public: + public: explicit CounterRecorderBase(time_t window_size); time_t window_size() const { return _avg_counter_window.window_size(); } -protected: + + protected: IntRecorder _avg_counter; Maxer _max_counter; Percentile _counter_percentile; @@ -71,34 +74,36 @@ class CounterRecorderBase { PassiveStatus _counter_p1; PassiveStatus _counter_p2; PassiveStatus _counter_p3; - PassiveStatus _counter_999; // 99.9% - PassiveStatus _counter_9999; // 99.99% + PassiveStatus _counter_999; // 99.9% + PassiveStatus _counter_9999; // 99.99% CDF _counter_cdf; PassiveStatus > _counter_percentiles; }; -} // namespace detail +} // namespace detail // Specialized structure to record counter. // It's not a Variable, but it contains multiple bvar inside. class CounterRecorder : public detail::CounterRecorderBase { typedef detail::CounterRecorderBase Base; -public: + + public: CounterRecorder() : Base(-1) {} explicit CounterRecorder(time_t window_size) : Base(window_size) {} explicit CounterRecorder(const butil::StringPiece& prefix) : Base(-1) { expose(prefix); } - CounterRecorder(const butil::StringPiece& prefix, - time_t window_size) : Base(window_size) { + CounterRecorder(const butil::StringPiece& prefix, time_t window_size) + : Base(window_size) { expose(prefix); } CounterRecorder(const butil::StringPiece& prefix1, - const butil::StringPiece& prefix2) : Base(-1) { + const butil::StringPiece& prefix2) + : Base(-1) { expose(prefix1, prefix2); } CounterRecorder(const butil::StringPiece& prefix1, - const butil::StringPiece& prefix2, - time_t window_size) : Base(window_size) { + const butil::StringPiece& prefix2, time_t window_size) + : Base(window_size) { expose(prefix1, prefix2); } @@ -106,7 +111,7 @@ class CounterRecorder : public detail::CounterRecorderBase { // Record the counter num. CounterRecorder& operator<<(int64_t count_num); - + // Expose all internal variables using `prefix' as prefix. // Returns 0 on success, -1 otherwise. // Example: @@ -124,16 +129,18 @@ class CounterRecorder : public detail::CounterRecorderBase { } int expose(const butil::StringPiece& prefix1, const butil::StringPiece& prefix2); - + // Hide all internal variables, called in dtor as well. void hide(); // Get the average counter num in recent |window_size| seconds // If |window_size| is absent, use the window_size to ctor. - int64_t avg_counter(time_t window_size) const - { return _avg_counter_window.get_value(window_size).get_average_int(); } - int64_t avg_counter() const - { return _avg_counter_window.get_value().get_average_int(); } + int64_t avg_counter(time_t window_size) const { + return _avg_counter_window.get_value(window_size).get_average_int(); + } + int64_t avg_counter() const { + return _avg_counter_window.get_value().get_average_int(); + } // Get p1/p2/p3/99.9-ile counter num in recent window_size-to-ctor seconds. Vector counter_percentiles() const; @@ -155,12 +162,16 @@ class CounterRecorder : public detail::CounterRecorderBase { int64_t counter_percentile(double ratio) const; // Get name of a sub-bvar. - const std::string& avg_counter_name() const { return _avg_counter_window.name(); } - const std::string& counter_percentiles_name() const - { return _counter_percentiles.name(); } + const std::string& avg_counter_name() const { + return _avg_counter_window.name(); + } + const std::string& counter_percentiles_name() const { + return _counter_percentiles.name(); + } const std::string& counter_cdf_name() const { return _counter_cdf.name(); } - const std::string& max_counter_name() const - { return _max_counter_window.name(); } + const std::string& max_counter_name() const { + return _max_counter_window.name(); + } const std::string& total_times_name() const { return _total_times.name(); } const std::string& qps_name() const { return _qps.name(); } }; @@ -176,16 +187,18 @@ class Closure; inline bool is_zero(const char* buff, const size_t size) { if (size >= sizeof(uint64_t)) { return (0 == *(uint64_t*)buff) && - (0 == memcmp(buff, buff + sizeof(uint64_t), size - sizeof(uint64_t))); + (0 == + memcmp(buff, buff + sizeof(uint64_t), size - sizeof(uint64_t))); } else if (size > 0) { return (0 == *(uint8_t*)buff) && - (0 == memcmp(buff, buff + sizeof(uint8_t), size - sizeof(uint8_t))); + (0 == + memcmp(buff, buff + sizeof(uint8_t), size - sizeof(uint8_t))); } else { return 0; } } -inline uint32_t murmurhash32(const void *key, int len) { +inline uint32_t murmurhash32(const void* key, int len) { uint32_t hash = 0; butil::MurmurHash3_x86_32(key, len, 0, &hash); return hash; @@ -224,7 +237,7 @@ inline uint32_t crc32(const butil::IOBuf& buf) { // Start a bthread to run closure void run_closure_in_bthread(::google::protobuf::Closure* closure, - bool in_pthread = false); + bool in_pthread = false); struct RunClosureInBthread { void operator()(google::protobuf::Closure* done) { @@ -233,7 +246,7 @@ struct RunClosureInBthread { }; typedef std::unique_ptr - AsyncClosureGuard; + AsyncClosureGuard; // Start a bthread to run closure without signal other worker thread to steal // it. You should call bthread_flush() at last. @@ -252,15 +265,19 @@ ssize_t file_pwrite(const butil::IOBuf& data, int fd, off_t offset); // unsequence file data, reduce the overhead of copy some files have hole. class FileSegData { -public: + public: // for reader FileSegData(const butil::IOBuf& data) - : _data(data), _seg_header(butil::IOBuf::INVALID_AREA), _seg_offset(0), _seg_len(0) {} + : _data(data), + _seg_header(butil::IOBuf::INVALID_AREA), + _seg_offset(0), + _seg_len(0) {} // for writer - FileSegData() : _seg_header(butil::IOBuf::INVALID_AREA), _seg_offset(0), _seg_len(0) {} - ~FileSegData() { - close(); - } + FileSegData() + : _seg_header(butil::IOBuf::INVALID_AREA), + _seg_offset(0), + _seg_len(0) {} + ~FileSegData() { close(); } // writer append void append(const butil::IOBuf& data, uint64_t offset); @@ -275,7 +292,7 @@ class FileSegData { // read next, NEED clear data when call next in loop size_t next(uint64_t* offset, butil::IOBuf* data); -private: + private: void close(); butil::IOBuf _data; @@ -286,14 +303,12 @@ class FileSegData { // A special Closure which provides synchronization primitives class SynchronizedClosure : public Closure { -public: + public: SynchronizedClosure() : _event(1) {} SynchronizedClosure(int num_signal) : _event(num_signal) {} // Implements braft::Closure - void Run() { - _event.signal(); - } + void Run() { _event.signal(); } // Block the thread until Run() has been called void wait() { _event.wait(); } // Reset the event @@ -301,10 +316,11 @@ class SynchronizedClosure : public Closure { status().reset(); _event.reset(); } -private: + + private: bthread::CountdownEvent _event; }; } // namespace braft -#endif // BRAFT_RAFT_UTIL_H +#endif // BRAFT_RAFT_UTIL_H