diff --git a/src/Makefile.am b/src/Makefile.am index b92061afbc..ad67958e78 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -117,6 +117,7 @@ GRIDCOIN_CORE_H = \ gridcoin/contract/registry.h \ gridcoin/contract/registry_db.h \ gridcoin/cpid.h \ + gridcoin/fwd.h \ gridcoin/gridcoin.h \ gridcoin/magnitude.h \ gridcoin/mrc.h \ diff --git a/src/gridcoin/fwd.h b/src/gridcoin/fwd.h new file mode 100644 index 0000000000..1719d0352b --- /dev/null +++ b/src/gridcoin/fwd.h @@ -0,0 +1,37 @@ +// Copyright (c) 2014-2025 The Gridcoin developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or https://opensource.org/licenses/mit-license.php. + +#ifndef GRIDCOIN_FWD_H +#define GRIDCOIN_FWD_H + +namespace GRC +{ +//! +//! \brief Enumeration of project entry status. Unlike beacons this is for both storage +//! and memory. +//! +//! UNKNOWN status is only encountered in trivially constructed empty +//! project entries and should never be seen on the blockchain. +//! +//! DELETED status corresponds to a removed entry. +//! +//! ACTIVE corresponds to an active entry. +//! +//! GREYLISTED means that the project temporarily does not meet the whitelist qualification criteria. +//! +//! OUT_OF_BOUND must go at the end and be retained for the EnumBytes wrapper. +//! +enum class ProjectEntryStatus +{ + UNKNOWN, + DELETED, + ACTIVE, + MAN_GREYLISTED, + AUTO_GREYLISTED, + OUT_OF_BOUND +}; +} // namespace GRC + +#endif // GRIDCOIN_FWD_H + diff --git a/src/gridcoin/project.cpp b/src/gridcoin/project.cpp index 00e521b555..d7b3074131 100644 --- a/src/gridcoin/project.cpp +++ b/src/gridcoin/project.cpp @@ -2,8 +2,12 @@ // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or https://opensource.org/licenses/mit-license.php. +#include "gridcoin/claim.h" #include "main.h" +#include "node/blockstorage.h" +#include "gridcoin/support/block_finder.h" #include "gridcoin/project.h" +#include "gridcoin/quorum.h" #include "node/ui_interface.h" #include @@ -296,6 +300,140 @@ WhitelistSnapshot WhitelistSnapshot::Sorted() const return WhitelistSnapshot(std::make_shared(sorted), m_filter); } +// ----------------------------------------------------------------------------- +// Class: AutoGreylist - automatic greylisting +// ----------------------------------------------------------------------------- + +AutoGreylist::AutoGreylist() + : m_greylist_ptr(std::make_shared()) + , m_superblock_hash(uint256 {}) +{ + //m_greylist_ptr = std::make_shared(); + + Refresh(); +} + +AutoGreylist::Greylist::const_iterator AutoGreylist::begin() const +{ + return m_greylist_ptr->begin(); +} + +AutoGreylist::Greylist::const_iterator AutoGreylist::end() const +{ + return m_greylist_ptr->end(); +} + +AutoGreylist::Greylist::size_type AutoGreylist::size() const +{ + return m_greylist_ptr->size(); +} + +void AutoGreylist::Refresh() +{ + LOCK(cs_main); + + // If the current superblock has not changed, then no need to do anything. + if (!m_superblock_hash.IsNull() && Quorum::CurrentSuperblock()->GetHash() == m_superblock_hash) { + return; + } + + SuperblockPtr superblock_ptr = Quorum::CurrentSuperblock(); + + if (!superblock_ptr.IsEmpty()) { + RefreshWithSuperblock(superblock_ptr); + } +} + +void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in) +{ + LOCK(lock); + + // We need the current whitelist, including all records except deleted. This will include greylisted projects, + // whether currently marked as manually greylisted from protocol or overridden to auto greylisted by the auto greylist class. + const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED); + + m_greylist_ptr->clear(); + + for (const auto& iter : whitelist) { + if (auto project = superblock_ptr_in->m_projects.Try(iter.m_name)) { + // Record new greylist candidate entry baseline with the total credit for each project present in superblock. + m_greylist_ptr->insert(std::make_pair(iter.m_name, GreylistCandidateEntry(iter.m_name, project->m_total_credit))); + } else { + // Record new greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist, + // but does not have a project entry in the superblock. This would be because the scrapers could not converge on the + // project. + m_greylist_ptr->insert(std::make_pair(iter.m_name, GreylistCandidateEntry(iter.m_name, + std::optional(std::nullopt)))); + } + } + + CBlockIndex* index_ptr; + { + // Find the block index entry for the block before the provided superblock_ptr. + index_ptr = GRC::BlockFinder::FindByHeight(superblock_ptr_in.m_height - 1); + } + + //SuperblockPtr superblock_ptr; + unsigned int superblock_count = 1; // The 0 (baseline) superblock was processed above. Here we start with 1 and go up to 40 + + while (index_ptr != nullptr && index_ptr->pprev != nullptr && superblock_count <= 40) { + + if (!index_ptr->IsSuperblock()) { + index_ptr = index_ptr->pprev; + continue; + } + + // For some reason this is not working. + //superblock_ptr.ReadFromDisk(index_ptr); + + CBlock block; + if (!ReadBlockFromDisk(block, index_ptr, Params().GetConsensus())) { + error("%s: Failed to read block from disk with requested height %u", + __func__, + index_ptr->nHeight); + continue; + } + + SuperblockPtr superblock_ptr = block.GetClaim().m_superblock; + + for (const auto& iter : whitelist) { + // This is guaranteed to succeed, because every whitelisted project was inserted as a new baseline entry above. + auto greylist_entry = m_greylist_ptr->find(iter.m_name); + + if (auto project = superblock_ptr->m_projects.Try(iter.m_name)) { + // Update greylist candidate entry with the total credit for each project present in superblock. + greylist_entry->second.UpdateGreylistCandidateEntry(project->m_total_credit, superblock_count); + } else { + // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist, + // but does not have a project entry in this superblock. This would be because the scrapers could not converge on the + // project for this superblock. + greylist_entry->second.UpdateGreylistCandidateEntry(std::optional(std::nullopt), superblock_count); + } + } + + ++superblock_count; + index_ptr = index_ptr->pprev; + } + + // Purge candidate elements that do not meet auto greylist criteria. + for (auto iter = m_greylist_ptr->begin(); iter != m_greylist_ptr->end(); ) { + if (iter->second.GetZCD() < 7 && iter->second.GetWAS() >= Fraction(1, 10)) { + // Candidate greylist entry does not meet auto greylist criteria, so remove from greylist entry map. + iter = m_greylist_ptr->erase(iter); + } else { + iter++; + } + } +} + +// This is the global cached (singleton) for the auto greylist. +std::shared_ptr g_autogreylist_ptr = std::make_shared(); + +std::shared_ptr AutoGreylist::GetAutoGreylistCache() +{ + return g_autogreylist_ptr; +} + // ----------------------------------------------------------------------------- // Class: Whitelist (Registry) // ----------------------------------------------------------------------------- @@ -304,6 +442,8 @@ WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& fil { LOCK(cs_lock); + //AutoGreylist::GetAutoGreylistCache()->Refresh(); + ProjectList projects; for (const auto& iter : m_project_entries) { diff --git a/src/gridcoin/project.h b/src/gridcoin/project.h index c63c4d8bd1..484183f5d2 100644 --- a/src/gridcoin/project.h +++ b/src/gridcoin/project.h @@ -7,6 +7,8 @@ #include "amount.h" #include "contract/contract.h" +#include "gridcoin/fwd.h" +#include "gridcoin/superblock.h" #include "gridcoin/contract/handler.h" #include "gridcoin/contract/payload.h" #include "gridcoin/contract/registry_db.h" @@ -21,30 +23,8 @@ namespace GRC { -//! -//! \brief Enumeration of project entry status. Unlike beacons this is for both storage -//! and memory. -//! -//! UNKNOWN status is only encountered in trivially constructed empty -//! project entries and should never be seen on the blockchain. -//! -//! DELETED status corresponds to a removed entry. -//! -//! ACTIVE corresponds to an active entry. -//! -//! GREYLISTED means that the project temporarily does not meet the whitelist qualification criteria. -//! -//! OUT_OF_BOUND must go at the end and be retained for the EnumBytes wrapper. -//! -enum class ProjectEntryStatus -{ - UNKNOWN, - DELETED, - ACTIVE, - MAN_GREYLISTED, - AUTO_GREYLISTED, - OUT_OF_BOUND -}; +// See gridcoin/fwd.h for ProjectEntryStatus. It is in the forward declaration file because it is +// also needed in the superblock class, and this prevents recursive includes. class ProjectEntry { @@ -517,6 +497,186 @@ class WhitelistSnapshot const ProjectEntry::ProjectFilterFlag m_filter; //!< The filter used to populate the readonly list. }; +class AutoGreylist +{ +public: + class GreylistCandidateEntry + { + public: + GreylistCandidateEntry() + : m_project_name(std::string {}) + , m_zcd_20_SB_count(0) + , m_TC_7_SB_sum(0) + , m_TC_40_SB_sum(0) + , m_TC_initial_bookmark(0) + , m_TC_bookmark(0) + , m_sb_from_baseline_processed(0) + , m_updates(0) + {} + + GreylistCandidateEntry(std::string project_name, std::optional TC_initial_bookmark) + : m_project_name(project_name) + , m_zcd_20_SB_count(0) + , m_TC_7_SB_sum(0) + , m_TC_40_SB_sum(0) + , m_TC_initial_bookmark(TC_initial_bookmark) + , m_TC_bookmark(0) + , m_sb_from_baseline_processed(0) + , m_updates(0) + {} + + uint8_t GetZCD() + { + return m_zcd_20_SB_count; + } + + Fraction GetWAS() + { + if (!m_sb_from_baseline_processed) { + return Fraction(0); + } + + // The Fraction class is implemented with int64_t as the underlying data type for the numerator and denominator; + // however, the total credit is stored in the superblock as a uint64_t. To be thorough, check if either of the unsigned + // 64 bit TC averages will overflow a signed 64 integer, and if so, then simply divide by 2 before constructing the + // fraction. This is extremely unlikely, given that the number of SB's processed from the baseline would have to be + // one, and then the sum overflow the int64_t max. + uint64_t TC_7_SB_avg = m_TC_7_SB_sum / (uint64_t) m_sb_from_baseline_processed; + uint64_t TC_40_SB_avg = m_TC_40_SB_sum / (uint64_t) m_sb_from_baseline_processed; + + if (TC_7_SB_avg > (uint64_t) std::numeric_limits::max() + || TC_40_SB_avg > (uint64_t) std::numeric_limits::max()) { + TC_7_SB_avg /= 2; + TC_40_SB_avg /= 2; + } + + if (TC_7_SB_avg == 0 && TC_40_SB_avg == 0) { + return Fraction(0); + } else { + return Fraction(TC_7_SB_avg, TC_40_SB_avg); + } + } + + void UpdateGreylistCandidateEntry(std::optional total_credit, uint8_t sb_from_baseline) + { + uint8_t sbs_from_baseline_no_update = sb_from_baseline - m_sb_from_baseline_processed - 1; + + m_sb_from_baseline_processed = sb_from_baseline; + + // ZCD part. Remember we are going backwards, so if total_credit is greater than or equal to + // the bookmark, then we have zero or even negative project total credit between superblocks, so + // this qualifies as a ZCD. + if (m_sb_from_baseline_processed <= 20) { + // Skipped updates count as ZCDs. We stop counting at 20 superblocks (back) from + // the initial SB baseline. The skipped updates may actually not be used in practice, because we are + // iterating over the entire whitelist for each SB and inserting std::nullopt TC updates for each project. + m_zcd_20_SB_count += sbs_from_baseline_no_update; + + // If total credit is greater than the bookmark, this means that (going forward in time) credit actually + // declined, so in addition to the zero credit days recorded for the skipped updates, we must add an + // additional day for the credit decline (going forward in time). If total credit is std::nullopt, then + // this means no statistics available, which also is a ZCD. + if ((total_credit && total_credit >= m_TC_bookmark) || !total_credit){ + ++m_zcd_20_SB_count; + } + } + + // WAS part. Here we deal with two numbers, the 40 SB from baseline, and the 7 SB from baseline. We use + // the initial bookmark, and compute the difference, which is the same as adding up the deltas between + // the TC's in each superblock. For updates with no total credit entry (i.e. std::optional is nullopt), + // do not change the sums. + if (total_credit && m_TC_initial_bookmark && m_TC_initial_bookmark > total_credit) { + if (m_sb_from_baseline_processed <= 7) { + m_TC_7_SB_sum = *m_TC_initial_bookmark - *total_credit; + } + + if (m_sb_from_baseline_processed <= 40) { + m_TC_40_SB_sum = *m_TC_initial_bookmark - *total_credit; + } + } + + // Update bookmark. + if (total_credit) { + m_TC_bookmark = *total_credit; + } + + ++m_updates; + } + + const std::string m_project_name; + + uint8_t m_zcd_20_SB_count; + uint64_t m_TC_7_SB_sum; + uint64_t m_TC_40_SB_sum; + + private: + std::optional m_TC_initial_bookmark; //!< This is a "reverse" bookmark - we are going backwards in SB's. + uint64_t m_TC_bookmark; + uint8_t m_sb_from_baseline_processed; + uint8_t m_updates; + }; + + typedef std::map Greylist; + + //! + //! \brief Smart pointer around a collection of projects. + //! + typedef std::shared_ptr GreylistPtr; + + typedef Greylist::size_type size_type; + typedef Greylist::iterator iterator; + typedef Greylist::const_iterator const_iterator; + + AutoGreylist(); + + //! + //! \brief Returns an iterator to the beginning. + //! + const_iterator begin() const; + + //! + //! \brief Returns an iterator to the end. + //! + const_iterator end() const; + + //! + //! \brief Get the number of projects in the auto greylist. + //! + size_type size() const; + + //! + //! \brief Determine whether the specified project exists in the auto greylist. + //! + //! \param name Project name matching the contract key. + //! + //! \return \c true if the auto greylist contains a project with matching name. + //! + bool Contains(const std::string& name) const; + + void Refresh(); + + //! + //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock. This mode is used + //! in the scraper during the construction of the superblock contract. + //! + //! Note that the AutoGreylist object refreshed this way will also be used to update the referenced superblock + //! object + //! + //! \param superblock The superblock with which to refresh the automatic greylist. This can be a candidate superblock + //! from a scraper convergence, or in the instance of this being called from Refresh(), could be the current + //! superblock on the chain. + //! + void RefreshWithSuperblock(SuperblockPtr superblock_ptr_in); + + static std::shared_ptr GetAutoGreylistCache(); + +private: + CCriticalSection lock; + + GreylistPtr m_greylist_ptr; + uint256 m_superblock_hash; +}; + //! //! \brief Registry that manages the collection of BOINC projects in the Gridcoin whitelist. //! diff --git a/src/gridcoin/superblock.h b/src/gridcoin/superblock.h index 03f7d355d4..df9de84e9b 100644 --- a/src/gridcoin/superblock.h +++ b/src/gridcoin/superblock.h @@ -5,10 +5,11 @@ #ifndef GRIDCOIN_SUPERBLOCK_H #define GRIDCOIN_SUPERBLOCK_H -#include "gridcoin/project.h" +#include "gridcoin/fwd.h" #include "gridcoin/cpid.h" #include "gridcoin/magnitude.h" #include "gridcoin/scraper/fwd.h" +#include "gridcoin/support/enumbytes.h" #include "serialize.h" #include "uint256.h" @@ -1187,11 +1188,15 @@ class Superblock uint64_t m_total_rac; }; // ProjectIndex + //! + //! \brief This is a wrapper around m_project_status. To conserve space, project status entries with a status + //! of ACTIVE are omitted. + //! struct ProjectStatus { ProjectStatus() {}; - std::vector m_project_status; + std::vector>> m_project_status; ADD_SERIALIZE_METHODS; @@ -1484,6 +1489,15 @@ class SuperblockPtr return now - m_timestamp; } + bool IsEmpty() + { + if (m_superblock == nullptr && m_height == 0 && m_timestamp == 0) { + return true; + } else { + return false; + } + } + ADD_SERIALIZE_METHODS; template diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e88022d219..a250494bae 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -6,6 +6,7 @@ #include "chainparams.h" #include "blockchain.h" #include "gridcoin/protocol.h" +#include "gridcoin/project.h" #include "gridcoin/scraper/scraper_registry.h" #include "gridcoin/sidestake.h" #include "node/blockstorage.h" @@ -2631,6 +2632,38 @@ UniValue listprojects(const UniValue& params, bool fHelp) return res; } +UniValue getautogreylist(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() > 0) { + throw runtime_error( + "getautogreylist \n" + "\n" + "Displays information about projects that meet auto greylisting criteria."); + } + + UniValue res(UniValue::VOBJ); + + std::shared_ptr greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache(); + + greylist_ptr->Refresh(); + + UniValue autogreylist(UniValue::VARR); + + for (auto iter : *greylist_ptr) { + UniValue entry(UniValue::VOBJ); + + entry.pushKV("project:", iter.first); + entry.pushKV("zcd", iter.second.GetZCD()); + entry.pushKV("WAS", iter.second.GetWAS().ToDouble()); + + autogreylist.push_back(entry); + } + + res.pushKV("auto_greylist_projects", autogreylist); + + return res; +} + UniValue listscrapers(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 0) diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 497c8bbd30..1d6b5e73ea 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -385,6 +385,7 @@ static const CRPCCommand vRPCCommands[] = { "inspectaccrualsnapshot", &inspectaccrualsnapshot, cat_developer }, { "listalerts", &listalerts, cat_developer }, { "listprojects", &listprojects, cat_developer }, + { "getautogreylist", &getautogreylist, cat_developer }, { "listprotocolentries", &listprotocolentries, cat_developer }, { "listresearcheraccounts", &listresearcheraccounts, cat_developer }, { "listscrapers", &listscrapers, cat_developer }, diff --git a/src/rpc/server.h b/src/rpc/server.h index e1ae8b6874..1b803386a4 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -195,6 +195,7 @@ extern UniValue rpc_getblockstats(const UniValue& params, bool fHelp); extern UniValue inspectaccrualsnapshot(const UniValue& params, bool fHelp); extern UniValue listalerts(const UniValue& params, bool fHelp); extern UniValue listprojects(const UniValue& params, bool fHelp); +extern UniValue getautogreylist(const UniValue& params, bool fHelp); extern UniValue listprotocolentries(const UniValue& params, bool fHelp); extern UniValue listresearcheraccounts(const UniValue& params, bool fHelp); extern UniValue listscrapers(const UniValue& params, bool fHelp);