diff --git a/src/test/app/LedgerReplay_test.cpp b/src/test/app/LedgerReplay_test.cpp index baf2f348e7e..bd7175857ff 100644 --- a/src/test/app/LedgerReplay_test.cpp +++ b/src/test/app/LedgerReplay_test.cpp @@ -75,8 +75,11 @@ class MagicInboundLedgers : public InboundLedgers virtual ~MagicInboundLedgers() = default; virtual std::shared_ptr - acquire(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason) - override + acquire( + uint256 const& hash, + std::uint32_t seq, + InboundLedger::Reason, + char const*) override { if (bhvr == InboundLedgersBehavior::DropAll) return {}; diff --git a/src/xrpld/app/consensus/RCLValidations.cpp b/src/xrpld/app/consensus/RCLValidations.cpp index d6a8747c20a..d4893d34d44 100644 --- a/src/xrpld/app/consensus/RCLValidations.cpp +++ b/src/xrpld/app/consensus/RCLValidations.cpp @@ -127,7 +127,7 @@ RCLValidationsAdaptor::acquire(LedgerHash const& hash) JLOG(j_.debug()) << "JOB advanceLedger getConsensusLedger2 started"; pApp->getInboundLedgers().acquireAsync( - hash, 0, InboundLedger::Reason::CONSENSUS); + hash, 0, InboundLedger::Reason::PREFERRED); }); return std::nullopt; } diff --git a/src/xrpld/app/ledger/InboundLedger.h b/src/xrpld/app/ledger/InboundLedger.h index e99fb9a219b..9de327ccd82 100644 --- a/src/xrpld/app/ledger/InboundLedger.h +++ b/src/xrpld/app/ledger/InboundLedger.h @@ -24,9 +24,10 @@ class InboundLedger final : public TimeoutCounter, // These are the reasons we might acquire a ledger enum class Reason { - HISTORY, // Acquiring past ledger - GENERIC, // Generic other reasons - CONSENSUS // We believe the consensus round requires this ledger + HISTORY, // Acquiring past ledger + GENERIC, // Generic other reasons + CONSENSUS, // We believe the consensus round requires this ledger + PREFERRED // We need this ledger for preferred ledger analysis }; InboundLedger( @@ -40,8 +41,10 @@ class InboundLedger final : public TimeoutCounter, ~InboundLedger(); // Called when another attempt is made to fetch this same ledger - void - update(std::uint32_t seq); + // + // Returns true if this triggers requests to be sent + bool + update(std::uint32_t seq, bool broadcast); /** Returns true if we got all the data. */ bool @@ -72,7 +75,7 @@ class InboundLedger final : public TimeoutCounter, bool checkLocal(); void - init(ScopedLockType& collectionLock); + init(ScopedLockType& collectionLock, bool broadcast); bool gotData( @@ -179,24 +182,8 @@ class InboundLedger final : public TimeoutCounter, std::unique_ptr mPeerSet; }; -inline std::string -to_string(InboundLedger::Reason reason) -{ - using enum InboundLedger::Reason; - switch (reason) - { - case HISTORY: - return "HISTORY"; - case GENERIC: - return "GENERIC"; - case CONSENSUS: - return "CONSENSUS"; - default: - UNREACHABLE( - "ripple::to_string(InboundLedger::Reason) : unknown value"); - return "unknown"; - } -} +std::string +to_string(InboundLedger::Reason reason); } // namespace ripple diff --git a/src/xrpld/app/ledger/InboundLedgers.h b/src/xrpld/app/ledger/InboundLedgers.h index 6eba2eec87d..d1b6bb613e7 100644 --- a/src/xrpld/app/ledger/InboundLedgers.h +++ b/src/xrpld/app/ledger/InboundLedgers.h @@ -21,7 +21,11 @@ class InboundLedgers // Callers should use this if they possibly need an authoritative // response immediately. virtual std::shared_ptr - acquire(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason) = 0; + acquire( + uint256 const& hash, + std::uint32_t seq, + InboundLedger::Reason, + char const* context) = 0; // Callers should use this if they are known to be executing on the Job // Queue. TODO review whether all callers of acquire() can use this diff --git a/src/xrpld/app/ledger/detail/InboundLedger.cpp b/src/xrpld/app/ledger/detail/InboundLedger.cpp index 7ebb21ecc3f..3e37ee2fd91 100644 --- a/src/xrpld/app/ledger/detail/InboundLedger.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedger.cpp @@ -54,6 +54,27 @@ enum { // millisecond for each ledger timeout auto constexpr ledgerAcquireTimeout = 3000ms; +std::string +to_string(InboundLedger::Reason reason) +{ + using enum InboundLedger::Reason; + switch (reason) + { + case HISTORY: + return "HISTORY"; + case GENERIC: + return "GENERIC"; + case CONSENSUS: + return "CONSENSUS"; + case PREFERRED: + return "PREFERRED"; + default: + UNREACHABLE( + "ripple::to_string(InboundLedger::Reason) : unknown value"); + return "unknown"; + } +} + InboundLedger::InboundLedger( Application& app, uint256 const& hash, @@ -83,7 +104,7 @@ InboundLedger::InboundLedger( } void -InboundLedger::init(ScopedLockType& collectionLock) +InboundLedger::init(ScopedLockType& collectionLock, bool broadcast) { ScopedLockType sl(mtx_); collectionLock.unlock(); @@ -94,8 +115,18 @@ InboundLedger::init(ScopedLockType& collectionLock) if (!complete_) { - addPeers(); - queueJob(sl); + if (broadcast) + { + addPeers(); + queueJob(sl); + } + else + { + // Delay to give time to build the ledger before sending + JLOG(journal_.debug()) << "init: Deferring peer requests"; + deferred_ = true; + setTimer(sl); + } return; } @@ -113,7 +144,7 @@ InboundLedger::init(ScopedLockType& collectionLock) app_.getLedgerMaster().storeLedger(mLedger); // Check if this could be a newer fully-validated ledger - if (mReason == Reason::CONSENSUS) + if (mReason >= Reason::CONSENSUS) app_.getLedgerMaster().checkAccept(mLedger); } @@ -126,8 +157,8 @@ InboundLedger::getPeerCount() const }); } -void -InboundLedger::update(std::uint32_t seq) +bool +InboundLedger::update(std::uint32_t seq, bool broadcast) { ScopedLockType sl(mtx_); @@ -137,6 +168,27 @@ InboundLedger::update(std::uint32_t seq) // Prevent this from being swept touch(); + + // If the signal is to broadcast, and this request has never tried to + // broadcast before, cancel any waiting timer, then fire off the job to + // broadcast. Note that this is calling mPeerSet->getPeerIds(), not + // getPeerCount(), because the latter will filter out peers that have been + // tried, but are since lost. This wants to check if peers have _ever_ been + // tried. If they have, stick with the normal timer flow. + if (broadcast && mPeerSet->getPeerIds().empty()) + { + if (cancelTimer(sl)) + { + JLOG(journal_.debug()) + << "update: cancelling timer to send peer requests"; + deferred_ = false; + skipNext_ = true; + addPeers(); + queueJob(sl); + return true; + } + } + return false; } bool diff --git a/src/xrpld/app/ledger/detail/InboundLedgers.cpp b/src/xrpld/app/ledger/detail/InboundLedgers.cpp index 136cf239232..66350deebb4 100644 --- a/src/xrpld/app/ledger/detail/InboundLedgers.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedgers.cpp @@ -51,7 +51,8 @@ class InboundLedgersImp : public InboundLedgers acquire( uint256 const& hash, std::uint32_t seq, - InboundLedger::Reason reason) override + InboundLedger::Reason reason, + char const* context) override { auto doAcquire = [&, seq, reason]() -> std::shared_ptr { XRPL_ASSERT( @@ -64,7 +65,7 @@ class InboundLedgersImp : public InboundLedgers return true; if (reason == InboundLedger::Reason::GENERIC) return true; - if (reason == InboundLedger::Reason::CONSENSUS) + if (reason >= InboundLedger::Reason::CONSENSUS) return true; return false; }(); @@ -73,7 +74,7 @@ class InboundLedgersImp : public InboundLedgers ss << "InboundLedger::acquire: " << "Request: " << to_string(hash) << ", " << seq << " NeedNetworkLedger: " << (needNetworkLedger ? "yes" : "no") - << " Reason: " << to_string(reason) + << " Reason: " << to_string(reason) << " Context: " << context << " Should acquire: " << (shouldAcquire ? "true." : "false."); /* Acquiring ledgers is somewhat expensive. It requires lots of @@ -87,20 +88,34 @@ class InboundLedgersImp : public InboundLedgers // the network, and doesn't have the necessary tx's and // ledger entries to build the ledger. bool const isFull = app_.getOPs().isFull(); + // fallingBehind means the last closed ledger is at least 2 + // behind the validated ledger. If the node is falling + // behind the network, it probably needs information from + // the network to catch up. + // + // The reason this should not simply be only at least 1 + // behind the validated ledger is that a slight lag is + // normal case because some nodes get there slightly later + // than others. A difference of 2 means that at least a full + // ledger interval has passed, so the node is beginning to + // fall behind. + bool const fallingBehind = app_.getOPs().isFallingBehind(); + // If the ledger is needed for preferred ledger analysis and we + // don't have it, chances are we're not going to build it, + // because someone else has built it, so download it. + bool const preferred = + reason == InboundLedger::Reason::PREFERRED; // If everything else is ok, don't try to acquire the ledger // if the requested seq is in the near future relative to - // the validated ledger. If the requested ledger is between - // 1 and 19 inclusive ledgers ahead of the valid ledger this - // node has not built it yet, but it's possible/likely it - // has the tx's necessary to build it and get caught up. - // Plus it might not become validated. On the other hand, if - // it's more than 20 in the future, this node should request - // it so that it can jump ahead and get caught up. + // the validated ledger. Because validations lag behind + // consensus, if we get any further behind than this, we + // risk losing sync, because we don't have the preferred + // ledger available. LedgerIndex const validSeq = app_.getLedgerMaster().getValidLedgerIndex(); - constexpr std::size_t lagLeeway = 20; - bool const nearFuture = - (seq > validSeq) && (seq < validSeq + lagLeeway); + constexpr std::size_t lagLeeway = 2; + bool const nearFuture = (validSeq > 0) && (seq > validSeq) && + (seq < validSeq + lagLeeway); // If everything else is ok, don't try to acquire the ledger // if the request is related to consensus. (Note that // consensus calls usually pass a seq of 0, so nearFuture @@ -109,8 +124,10 @@ class InboundLedgersImp : public InboundLedgers reason == InboundLedger::Reason::CONSENSUS; ss << " Evaluating whether to broadcast requests to peers" << ". full: " << (isFull ? "true" : "false") - << ". ledger sequence " << seq - << ". Valid sequence: " << validSeq + << ". falling behind: " << (fallingBehind ? "true" : "false") + << ". needed for preferred ledger analysis: " + << (preferred ? "true" : "false") << ". ledger sequence " + << seq << ". Valid sequence: " << validSeq << ". Lag leeway: " << lagLeeway << ". request for near future ledger: " << (nearFuture ? "true" : "false") @@ -119,6 +136,12 @@ class InboundLedgersImp : public InboundLedgers // If the node is not synced, send requests. if (!isFull) return true; + // If the node is falling behind, send requests. + if (fallingBehind) + return true; + // If needed for preferred analysis, send requests. + if (preferred) + return true; // If the ledger is in the near future, do NOT send requests. // This node is probably about to build it. if (nearFuture) @@ -129,7 +152,7 @@ class InboundLedgersImp : public InboundLedgers return false; return true; }(); - ss << ". Would broadcast to peers? " + ss << ". Broadcast to peers? " << (shouldBroadcast ? "true." : "false."); if (!shouldAcquire) @@ -164,7 +187,7 @@ class InboundLedgersImp : public InboundLedgers std::ref(m_clock), mPeerSetBuilder->build()); mLedgers.emplace(hash, inbound); - inbound->init(sl); + inbound->init(sl, shouldBroadcast); ++mCounter; } } @@ -176,8 +199,12 @@ class InboundLedgersImp : public InboundLedgers return {}; } - if (!isNew) - inbound->update(seq); + bool const didBroadcast = [&]() { + if (!isNew) + return inbound->update(seq, shouldBroadcast); + return shouldBroadcast; + }(); + ss << " First broadcast: " << (didBroadcast ? "true" : "false"); if (!inbound->isComplete()) { @@ -203,7 +230,7 @@ class InboundLedgersImp : public InboundLedgers { try { - acquire(hash, seq, reason); + acquire(hash, seq, reason, "acquireAsync"); } catch (std::exception const& e) { diff --git a/src/xrpld/app/ledger/detail/LedgerCleaner.cpp b/src/xrpld/app/ledger/detail/LedgerCleaner.cpp index 5de811fc395..5b10727ad28 100644 --- a/src/xrpld/app/ledger/detail/LedgerCleaner.cpp +++ b/src/xrpld/app/ledger/detail/LedgerCleaner.cpp @@ -237,7 +237,8 @@ class LedgerCleanerImp : public LedgerCleaner app_.getInboundLedgers().acquire( ledger->info().hash, ledger->info().seq, - InboundLedger::Reason::GENERIC); + InboundLedger::Reason::GENERIC, + "getLedgerHash"); } return hash ? *hash : beast::zero; // kludge } @@ -257,13 +258,19 @@ class LedgerCleanerImp : public LedgerCleaner bool doTxns) { auto nodeLedger = app_.getInboundLedgers().acquire( - ledgerHash, ledgerIndex, InboundLedger::Reason::GENERIC); + ledgerHash, + ledgerIndex, + InboundLedger::Reason::GENERIC, + "doLedger"); if (!nodeLedger) { JLOG(j_.debug()) << "Ledger " << ledgerIndex << " not available"; app_.getLedgerMaster().clearLedger(ledgerIndex); app_.getInboundLedgers().acquire( - ledgerHash, ledgerIndex, InboundLedger::Reason::GENERIC); + ledgerHash, + ledgerIndex, + InboundLedger::Reason::GENERIC, + "doLedger not available"); return false; } @@ -289,7 +296,10 @@ class LedgerCleanerImp : public LedgerCleaner JLOG(j_.debug()) << "Ledger " << ledgerIndex << " is missing nodes"; app_.getLedgerMaster().clearLedger(ledgerIndex); app_.getInboundLedgers().acquire( - ledgerHash, ledgerIndex, InboundLedger::Reason::GENERIC); + ledgerHash, + ledgerIndex, + InboundLedger::Reason::GENERIC, + "doLedger missing nodes"); return false; } @@ -345,7 +355,10 @@ class LedgerCleanerImp : public LedgerCleaner // We found the hash and sequence of a better reference // ledger. referenceLedger = app_.getInboundLedgers().acquire( - refHash, refIndex, InboundLedger::Reason::GENERIC); + refHash, + refIndex, + InboundLedger::Reason::GENERIC, + "getHash"); if (referenceLedger) ledgerHash = getLedgerHash(referenceLedger, ledgerIndex); diff --git a/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp b/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp index 4a91c597d53..35eb3e8c14b 100644 --- a/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp +++ b/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp @@ -91,7 +91,10 @@ LedgerDeltaAcquire::trigger(std::size_t limit, ScopedLockType& sl) if (fallBack_) inboundLedgers_.acquire( - hash_, ledgerSeq_, InboundLedger::Reason::GENERIC); + hash_, + ledgerSeq_, + InboundLedger::Reason::GENERIC, + "LedgerDeltaAcquire::trigger"); } void diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index eb86c099dc0..fa123845edb 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -848,7 +848,8 @@ void LedgerMaster::failedSave(std::uint32_t seq, uint256 const& hash) { clearLedger(seq); - app_.getInboundLedgers().acquire(hash, seq, InboundLedger::Reason::GENERIC); + app_.getInboundLedgers().acquire( + hash, seq, InboundLedger::Reason::GENERIC, "failedSave"); } // Check if the specified ledger can become the new last fully-validated @@ -896,7 +897,7 @@ LedgerMaster::checkAccept(uint256 const& hash, std::uint32_t seq) // FIXME: We may not want to fetch a ledger with just one // trusted validation ledger = app_.getInboundLedgers().acquire( - hash, seq, InboundLedger::Reason::GENERIC); + hash, seq, InboundLedger::Reason::GENERIC, "checkAccept"); } if (ledger) @@ -1273,7 +1274,10 @@ LedgerMaster::findNewLedgersToPublish( // Can we try to acquire the ledger we need? if (!ledger && (++acqCount < ledger_fetch_size_)) ledger = app_.getInboundLedgers().acquire( - *hash, seq, InboundLedger::Reason::GENERIC); + *hash, + seq, + InboundLedger::Reason::GENERIC, + "findNewLedgersToPublish"); } // Did we acquire the next ledger we need to publish? @@ -1464,7 +1468,8 @@ LedgerMaster::updatePaths() app_.getInboundLedgers().acquire( lastLedger->info().parentHash, lastLedger->info().seq - 1, - InboundLedger::Reason::GENERIC); + InboundLedger::Reason::GENERIC, + "updatePaths open"); } else { @@ -1472,7 +1477,8 @@ LedgerMaster::updatePaths() app_.getInboundLedgers().acquire( lastLedger->info().hash, lastLedger->info().seq, - InboundLedger::Reason::GENERIC); + InboundLedger::Reason::GENERIC, + "updatePaths closed"); } } } @@ -1673,7 +1679,7 @@ LedgerMaster::walkHashBySeq( if (!ledger) { if (auto const l = app_.getInboundLedgers().acquire( - *refHash, refIndex, reason)) + *refHash, refIndex, reason, "walkHashBySeq")) { ledgerHash = hashOfSeq(*l, index, m_journal); XRPL_ASSERT( @@ -1799,8 +1805,8 @@ LedgerMaster::fetchForHistory( { if (!app_.getInboundLedgers().isFailure(*hash)) { - ledger = - app_.getInboundLedgers().acquire(*hash, missing, reason); + ledger = app_.getInboundLedgers().acquire( + *hash, missing, reason, "fetchForHistory"); if (!ledger && missing != fetch_seq_ && missing > app_.getNodeStore().earliestLedgerSeq()) { @@ -1867,7 +1873,8 @@ LedgerMaster::fetchForHistory( h->isNonZero(), "ripple::LedgerMaster::fetchForHistory : " "prefetched ledger"); - app_.getInboundLedgers().acquire(*h, seq, reason); + app_.getInboundLedgers().acquire( + *h, seq, reason, "fetchForHistory no ledger"); } } } diff --git a/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp b/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp index 4b068882c91..c7092f9276e 100644 --- a/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp +++ b/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp @@ -142,7 +142,8 @@ LedgerReplayTask::trigger(ScopedLockType& sl) parent_ = inboundLedgers_.acquire( parameter_.startHash_, parameter_.startSeq_, - InboundLedger::Reason::GENERIC); + InboundLedger::Reason::GENERIC, + "LedgerReplayTask::trigger"); } if (parent_) { diff --git a/src/xrpld/app/ledger/detail/SkipListAcquire.cpp b/src/xrpld/app/ledger/detail/SkipListAcquire.cpp index a6dbf40d4d7..60550609c7a 100644 --- a/src/xrpld/app/ledger/detail/SkipListAcquire.cpp +++ b/src/xrpld/app/ledger/detail/SkipListAcquire.cpp @@ -89,7 +89,11 @@ SkipListAcquire::trigger(std::size_t limit, ScopedLockType& sl) } if (fallBack_) - inboundLedgers_.acquire(hash_, 0, InboundLedger::Reason::GENERIC); + inboundLedgers_.acquire( + hash_, + 0, + InboundLedger::Reason::GENERIC, + "SkipListAcquire::trigger"); } void diff --git a/src/xrpld/app/ledger/detail/TimeoutCounter.cpp b/src/xrpld/app/ledger/detail/TimeoutCounter.cpp index 19410b77c1e..e069f25836a 100644 --- a/src/xrpld/app/ledger/detail/TimeoutCounter.cpp +++ b/src/xrpld/app/ledger/detail/TimeoutCounter.cpp @@ -36,23 +36,31 @@ TimeoutCounter::setTimer(ScopedLockType& sl) JLOG(journal_.debug()) << "Setting timer for " << timerInterval_.count() << "ms"; timer_.expires_after(timerInterval_); - timer_.async_wait( - [wptr = pmDowncast()](boost::system::error_code const& ec) { - if (ec == boost::asio::error::operation_aborted) - return; - - if (auto ptr = wptr.lock()) + timer_.async_wait([wptr = + pmDowncast()](boost::system::error_code const& ec) { + if (auto ptr = wptr.lock()) + { + ScopedLockType sl(ptr->mtx_); + if (ec == boost::asio::error::operation_aborted || ptr->skipNext_) { JLOG(ptr->journal_.debug()) - << "timer: ec: " << ec << " (operation_aborted: " - << boost::asio::error::operation_aborted << " - " - << (ec == boost::asio::error::operation_aborted ? "aborted" - : "other") - << ")"; - ScopedLockType sl(ptr->mtx_); - ptr->queueJob(sl); + << "Aborting setTimer: " << ec + << ", skip: " << (ptr->skipNext_ ? "true" : "false"); + ptr->skipNext_ = false; + return; } - }); + + ptr->queueJob(sl); + } + }); +} + +std::size_t +TimeoutCounter::cancelTimer(ScopedLockType& sl) +{ + auto const ret = timer_.cancel(); + JLOG(journal_.debug()) << "Cancelled " << ret << " timer(s)"; + return ret; } void @@ -89,7 +97,10 @@ TimeoutCounter::invokeOnTimer() if (!progress_) { - ++timeouts_; + if (deferred_) + deferred_ = false; + else + ++timeouts_; JLOG(journal_.debug()) << "Timeout(" << timeouts_ << ") " << " acquiring " << hash_; onTimer(false, sl); diff --git a/src/xrpld/app/ledger/detail/TimeoutCounter.h b/src/xrpld/app/ledger/detail/TimeoutCounter.h index 956b0e9ea85..d7e4e4432db 100644 --- a/src/xrpld/app/ledger/detail/TimeoutCounter.h +++ b/src/xrpld/app/ledger/detail/TimeoutCounter.h @@ -84,6 +84,10 @@ class TimeoutCounter void setTimer(ScopedLockType&); + /** Cancel any waiting timer */ + std::size_t + cancelTimer(ScopedLockType&); + /** Queue a job to call invokeOnTimer(). */ void queueJob(ScopedLockType&); @@ -115,6 +119,11 @@ class TimeoutCounter int timeouts_; bool complete_; bool failed_; + /** Whether the initialization deferred doing any work until the first + * timeout. */ + bool deferred_ = false; + /** Skip the next timeout, regardless of ec */ + bool skipNext_ = false; /** Whether forward progress has been made. */ bool progress_; /** The minimum time to wait between calls to execute(). */ diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 56be2f1f6aa..6fca73d19b6 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -439,6 +439,8 @@ class NetworkOPsImp final : public NetworkOPs clearLedgerFetch() override; Json::Value getLedgerFetchInfo() override; + bool + isFallingBehind() const override; std::uint32_t acceptLedger( std::optional consensusDelay) override; @@ -733,6 +735,7 @@ class NetworkOPsImp final : public NetworkOPs std::atomic amendmentBlocked_{false}; std::atomic amendmentWarned_{false}; std::atomic unlBlocked_{false}; + std::atomic fallingBehind_{false}; ClosureCounter waitHandlerCounter_; boost::asio::steady_timer heartbeatTimer_; @@ -1939,7 +1942,10 @@ NetworkOPsImp::checkLastClosedLedger( if (!consensus) consensus = app_.getInboundLedgers().acquire( - closedLedger, 0, InboundLedger::Reason::CONSENSUS); + closedLedger, + 0, + InboundLedger::Reason::CONSENSUS, + "checkLastClosedLedger"); if (consensus && (!m_ledgerMaster.canBeCurrent(consensus) || @@ -2038,13 +2044,25 @@ NetworkOPsImp::beginConsensus( auto closingInfo = m_ledgerMaster.getCurrentLedger()->info(); - JLOG(m_journal.info()) << "Consensus time for #" << closingInfo.seq + JLOG(m_journal.info()) << "beginConsensus time for #" << closingInfo.seq << " with LCL " << closingInfo.parentHash; - auto prevLedger = m_ledgerMaster.getLedgerByHash(closingInfo.parentHash); + fallingBehind_ = false; + if (closingInfo.seq < m_ledgerMaster.getValidLedgerIndex() - 1) + { + fallingBehind_ = true; + JLOG(m_journal.warn()) + << "beginConsensus Current ledger " << closingInfo.seq + << " is at least 2 behind validated " + << m_ledgerMaster.getValidLedgerIndex(); + } + + auto const prevLedger = + m_ledgerMaster.getLedgerByHash(closingInfo.parentHash); if (!prevLedger) { + fallingBehind_ = true; // this shouldn't happen unless we jump ledgers if (mMode == OperatingMode::FULL) { @@ -2098,7 +2116,7 @@ NetworkOPsImp::beginConsensus( mLastConsensusPhase = currPhase; } - JLOG(m_journal.debug()) << "Initiating consensus engine"; + JLOG(m_journal.debug()) << "beginConsensus Initiating consensus engine"; return true; } @@ -2194,7 +2212,7 @@ NetworkOPsImp::endConsensus(std::unique_ptr const& clog) { // check if the ledger is good enough to go to FULL // Note: Do not go to FULL if we don't have the previous ledger - // check if the ledger is bad enough to go to CONNECTE D -- TODO + // check if the ledger is bad enough to go to CONNECTED -- TODO auto current = m_ledgerMaster.getCurrentLedger(); if (app_.timeKeeper().now() < (current->info().parentCloseTime + 2 * current->info().closeTimeResolution)) @@ -3033,6 +3051,12 @@ NetworkOPsImp::getLedgerFetchInfo() return app_.getInboundLedgers().getInfo(); } +bool +NetworkOPsImp::isFallingBehind() const +{ + return fallingBehind_; +} + void NetworkOPsImp::pubProposedTransaction( std::shared_ptr const& ledger, diff --git a/src/xrpld/app/misc/NetworkOPs.h b/src/xrpld/app/misc/NetworkOPs.h index 7d526e0b89f..ff5a227ebb7 100644 --- a/src/xrpld/app/misc/NetworkOPs.h +++ b/src/xrpld/app/misc/NetworkOPs.h @@ -221,6 +221,8 @@ class NetworkOPs : public InfoSub::Source clearLedgerFetch() = 0; virtual Json::Value getLedgerFetchInfo() = 0; + virtual bool + isFallingBehind() const = 0; /** Accepts the current transaction tree, return the new ledger's sequence diff --git a/src/xrpld/shamap/NodeFamily.cpp b/src/xrpld/shamap/NodeFamily.cpp index b9cca65d360..023e24d3722 100644 --- a/src/xrpld/shamap/NodeFamily.cpp +++ b/src/xrpld/shamap/NodeFamily.cpp @@ -83,7 +83,7 @@ NodeFamily::acquire(uint256 const& hash, std::uint32_t seq) JLOG(j_.error()) << "Missing node in " << to_string(hash); app_.getInboundLedgers().acquire( - hash, seq, InboundLedger::Reason::GENERIC); + hash, seq, InboundLedger::Reason::GENERIC, "NodeFamily::acquire"); } }