diff --git a/src/gridcoin/project.cpp b/src/gridcoin/project.cpp index d7b3074131..c3ea702484 100644 --- a/src/gridcoin/project.cpp +++ b/src/gridcoin/project.cpp @@ -308,8 +308,6 @@ AutoGreylist::AutoGreylist() : m_greylist_ptr(std::make_shared()) , m_superblock_hash(uint256 {}) { - //m_greylist_ptr = std::make_shared(); - Refresh(); } @@ -328,6 +326,21 @@ AutoGreylist::Greylist::size_type AutoGreylist::size() const return m_greylist_ptr->size(); } +bool AutoGreylist::Contains(const std::string& name, const bool& only_auto_greylisted) const +{ + auto iter = m_greylist_ptr->find(name); + + if (iter != m_greylist_ptr->end()) { + if (only_auto_greylisted) { + return (only_auto_greylisted && iter->second.m_meets_greylisting_crit); + } else { + return true; + } + } else { + return false; + } +} + void AutoGreylist::Refresh() { LOCK(cs_main); @@ -349,15 +362,28 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in) LOCK(lock); // We need the current whitelist, including all records except deleted. This will include greylisted projects, - // whether currently marked as manually greylisted from protocol or overridden to auto greylisted by the auto greylist class. - const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED); + // whether currently marked as manually greylisted from protocol or overridden to auto greylisted by the auto greylist class, + // based on the current state of the autogreylist. NOTE that the refresh_greylist is set to false here and MUST be this + // when called in the AutoGreylist class itself, to avoid an infinite loop. + const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED, false); m_greylist_ptr->clear(); + unsigned int v3_superblock_count = 0; + + if (superblock_ptr_in->m_version > 2) { + ++v3_superblock_count; + } + + // Notice the superblock_ptr_in m_projects_all_cpid_total_credits MUST ALEADY BE POPULATED to record the TC state into + // the auto greylist. for (const auto& iter : whitelist) { - if (auto project = superblock_ptr_in->m_projects.Try(iter.m_name)) { + auto project = superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.find(iter.m_name); + + if (project != superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) { // Record new greylist candidate entry baseline with the total credit for each project present in superblock. - m_greylist_ptr->insert(std::make_pair(iter.m_name, GreylistCandidateEntry(iter.m_name, project->m_total_credit))); + m_greylist_ptr->insert(std::make_pair(iter.m_name, + GreylistCandidateEntry(iter.m_name, std::nearbyint(project->second)))); } else { // Record new greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist, // but does not have a project entry in the superblock. This would be because the scrapers could not converge on the @@ -373,7 +399,6 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in) index_ptr = GRC::BlockFinder::FindByHeight(superblock_ptr_in.m_height - 1); } - //SuperblockPtr superblock_ptr; unsigned int superblock_count = 1; // The 0 (baseline) superblock was processed above. Here we start with 1 and go up to 40 while (index_ptr != nullptr && index_ptr->pprev != nullptr && superblock_count <= 40) { @@ -396,32 +421,67 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in) SuperblockPtr superblock_ptr = block.GetClaim().m_superblock; - for (const auto& iter : whitelist) { - // This is guaranteed to succeed, because every whitelisted project was inserted as a new baseline entry above. - auto greylist_entry = m_greylist_ptr->find(iter.m_name); - - if (auto project = superblock_ptr->m_projects.Try(iter.m_name)) { - // Update greylist candidate entry with the total credit for each project present in superblock. - greylist_entry->second.UpdateGreylistCandidateEntry(project->m_total_credit, superblock_count); - } else { - // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist, - // but does not have a project entry in this superblock. This would be because the scrapers could not converge on the - // project for this superblock. - greylist_entry->second.UpdateGreylistCandidateEntry(std::optional(std::nullopt), superblock_count); + if (superblock_ptr->m_version > 2) { + for (const auto& iter : whitelist) { + // This is guaranteed to succeed, because every whitelisted project was inserted as a new baseline entry above. + auto greylist_entry = m_greylist_ptr->find(iter.m_name); + + auto project = superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.find(iter.m_name); + + if (project != superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) { + // Update greylist candidate entry with the total credit for each project present in superblock. + greylist_entry->second.UpdateGreylistCandidateEntry(project->second, superblock_count); + } else { + // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist, + // but does not have a project entry in this superblock. This would be because the scrapers could not converge on the + // project for this superblock. + greylist_entry->second.UpdateGreylistCandidateEntry(std::optional(std::nullopt), superblock_count); + } } + + ++v3_superblock_count; } ++superblock_count; + index_ptr = index_ptr->pprev; } - // Purge candidate elements that do not meet auto greylist criteria. - for (auto iter = m_greylist_ptr->begin(); iter != m_greylist_ptr->end(); ) { - if (iter->second.GetZCD() < 7 && iter->second.GetWAS() >= Fraction(1, 10)) { - // Candidate greylist entry does not meet auto greylist criteria, so remove from greylist entry map. - iter = m_greylist_ptr->erase(iter); - } else { - iter++; + // Mark elements with whether they meet greylist criteria. + for (auto iter = m_greylist_ptr->begin(); iter != m_greylist_ptr->end(); ++iter) { + // the v3_superblock_count >= 2 test is to ensure there are at least two v3 superblocks with the total credits + // filled in to sample for the auto greylist determination. A two sb sample with positive change in TC will + // cause the ZCD and WAS rules to pass. + if (iter->second.GetZCD() < 7 && iter->second.GetWAS() >= Fraction(1, 10) && v3_superblock_count >= 2) { + iter->second.m_meets_greylisting_crit = true; + } + } +} + +void AutoGreylist::RefreshWithSuperblock(Superblock& superblock) +{ + SuperblockPtr superblock_ptr; + + // For the purposes of forming a superblock as the mining of a new block adding to the head of the chain, we will + // form a superblock ptr referencing the current head of the chain. The actual superblock will be the next block if it + // is added to the chain, for for the purposes here, this is what we want, because it is simply used to feed the + // overloaded version which takes the superblock_ptr and follows the chain backwards to do the greylist calculations. + superblock_ptr.Replace(superblock); + + { + LOCK(cs_main); + + superblock_ptr.Rebind(pindexBest); + } + + RefreshWithSuperblock(superblock_ptr); + + const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED, false); + + // Update the superblock object with the project greylist status. + for (const auto& project : whitelist) { + if (project.m_status == ProjectEntryStatus::AUTO_GREYLISTED || project.m_status == ProjectEntryStatus::MAN_GREYLISTED) { + superblock.m_project_status.m_project_status.insert(std::make_pair(project.m_name, project.m_status)); } } } @@ -438,15 +498,27 @@ std::shared_ptr AutoGreylist::GetAutoGreylistCache() // Class: Whitelist (Registry) // ----------------------------------------------------------------------------- -WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& filter) const +WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& filter, const bool& refresh_greylist) const { LOCK(cs_lock); - //AutoGreylist::GetAutoGreylistCache()->Refresh(); + std::shared_ptr greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache(); + + if (refresh_greylist) { + greylist_ptr->Refresh(); + } ProjectList projects; - for (const auto& iter : m_project_entries) { + // This is the override for automatic greylisting. If the AutoGreylist class refresh determines + // that the project meets greylisting criteria, it will be in the AutoGreylist object pointed to + // by the greylist_ptr. This will override the whitelist entries from the project whitelist registry. + for (auto iter : m_project_entries) { + if ((iter.second->m_status == ProjectEntryStatus::ACTIVE || iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED) + && greylist_ptr->Contains(iter.first)) { + iter.second->m_status = ProjectEntryStatus::AUTO_GREYLISTED; + } + switch (filter) { case ProjectEntry::ProjectFilterFlag::ACTIVE: if (iter.second->m_status == ProjectEntryStatus::ACTIVE) { diff --git a/src/gridcoin/project.h b/src/gridcoin/project.h index 17fb8abc83..f0a6c007a2 100644 --- a/src/gridcoin/project.h +++ b/src/gridcoin/project.h @@ -508,6 +508,7 @@ class AutoGreylist , m_zcd_20_SB_count(0) , m_TC_7_SB_sum(0) , m_TC_40_SB_sum(0) + , m_meets_greylisting_crit(0) , m_TC_initial_bookmark(0) , m_TC_bookmark(0) , m_sb_from_baseline_processed(0) @@ -519,6 +520,7 @@ class AutoGreylist , m_zcd_20_SB_count(0) , m_TC_7_SB_sum(0) , m_TC_40_SB_sum(0) + , m_meets_greylisting_crit(0) , m_TC_initial_bookmark(TC_initial_bookmark) , m_TC_bookmark(0) , m_sb_from_baseline_processed(0) @@ -608,6 +610,7 @@ class AutoGreylist uint8_t m_zcd_20_SB_count; uint64_t m_TC_7_SB_sum; uint64_t m_TC_40_SB_sum; + bool m_meets_greylisting_crit; private: std::optional m_TC_initial_bookmark; //!< This is a "reverse" bookmark - we are going backwards in SB's. @@ -649,25 +652,40 @@ class AutoGreylist //! //! \param name Project name matching the contract key. //! + //! \param only_auto_greylisted A boolean that specifies whether the search is against all projects or only those + //! that meet auto greylisting criteria. + //! //! \return \c true if the auto greylist contains a project with matching name. //! - bool Contains(const std::string& name) const; + bool Contains(const std::string& name, const bool& only_auto_greylisted = true) const; void Refresh(); //! - //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock. This mode is used - //! in the scraper during the construction of the superblock contract. + //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock pointer. //! //! Note that the AutoGreylist object refreshed this way will also be used to update the referenced superblock //! object //! - //! \param superblock The superblock with which to refresh the automatic greylist. This can be a candidate superblock + //! \param superblock_ptr The superblock pointer with which to refresh the automatic greylist. This can be a candidate superblock //! from a scraper convergence, or in the instance of this being called from Refresh(), could be the current //! superblock on the chain. //! void RefreshWithSuperblock(SuperblockPtr superblock_ptr_in); + //! + //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock that is going to be associated + //! with the current head of the chain. This mode is used in the scraper during the construction of the superblock contract. + //! + //! Note that the AutoGreylist object refreshed this way will also be used to update the referenced superblock + //! object + //! + //! \param superblock The superblock with which to refresh the automatic greylist. This will generally be a candidate superblock + //! from a scraper convergence, and is used in the call chain from the miner loop. The superblock object project status + //! will be updated. + //! + void RefreshWithSuperblock(Superblock& superblock); + static std::shared_ptr GetAutoGreylistCache(); private: @@ -717,9 +735,11 @@ class Whitelist : public IContractHandler //! //! \brief Get a read-only view of the projects in the whitelist. The default filter is ACTIVE, which - //! provides the original ACTIVE project only view. + //! provides the original ACTIVE project only view. The refresh_greylist filter is used to refresh + //! the AutoGreylist as part of taking the snapshot. //! - WhitelistSnapshot Snapshot(const ProjectEntry::ProjectFilterFlag& filter = ProjectEntry::ProjectFilterFlag::ACTIVE) const; + WhitelistSnapshot Snapshot(const ProjectEntry::ProjectFilterFlag& filter = ProjectEntry::ProjectFilterFlag::ACTIVE, + const bool& refresh_greylist = false) const; //! //! \brief Destroy the contract handler state to prepare for historical diff --git a/src/gridcoin/superblock.cpp b/src/gridcoin/superblock.cpp index 4e4e63364d..018315c7dc 100644 --- a/src/gridcoin/superblock.cpp +++ b/src/gridcoin/superblock.cpp @@ -4,6 +4,7 @@ #include "chainparams.h" #include "compat/endian.h" +#include "gridcoin/project.h" #include "hash.h" #include "main.h" #include @@ -607,20 +608,29 @@ Superblock Superblock::FromConvergence( if (!stats.Convergence.bByParts) { superblock.m_manifest_content_hint = stats.Convergence.nUnderlyingManifestContentHash.GetUint64(0) >> 32; + } else { + ProjectIndex& projects = superblock.m_projects; - return superblock; + // Add hints created from the hashes of converged manifest parts to each + // superblock project section to assist receiving nodes with validation: + // + for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) { + const std::string& project_name = part_pair.first; + const CSplitBlob::CPart* part_data_ptr = part_pair.second; + + projects.SetHint(project_name, part_data_ptr); + } } - ProjectIndex& projects = superblock.m_projects; + if (version > 2) { + // Populate the total credits into the superblock object from the convergence. This must be done BEFORE + // the auto greylist RefreshWithSuperblock is called. + superblock.m_projects_all_cpids_total_credits.Reset(stats.mScraperConvergedStats.m_total_credit_map); - // Add hints created from the hashes of converged manifest parts to each - // superblock project section to assist receiving nodes with validation: - // - for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) { - const std::string& project_name = part_pair.first; - const CSplitBlob::CPart* part_data_ptr = part_pair.second; + // Refresh the auto greylist and refresh this superblock with the greylist status. + std::shared_ptr greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache(); - projects.SetHint(project_name, part_data_ptr); + greylist_ptr->RefreshWithSuperblock(superblock); } return superblock; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4a07bffe6d..03c8b357aa 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2634,13 +2634,21 @@ UniValue listprojects(const UniValue& params, bool fHelp) UniValue getautogreylist(const UniValue& params, bool fHelp) { - if (fHelp || params.size() > 0) { + if (fHelp || params.size() > 1) { throw runtime_error( - "getautogreylist \n" + "getautogreylist \n" + "\n" + " -> true to show all projects, including those that do not meet greylisting criteria. Defaults to false. \n" "\n" "Displays information about projects that meet auto greylisting criteria."); } + bool show_all_projects = false; + + if (params.size()) { + show_all_projects = params[0].get_bool(); + } + UniValue res(UniValue::VOBJ); std::shared_ptr greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache(); @@ -2650,11 +2658,16 @@ UniValue getautogreylist(const UniValue& params, bool fHelp) UniValue autogreylist(UniValue::VARR); for (auto iter : *greylist_ptr) { + if (!show_all_projects && !iter.second.m_meets_greylisting_crit) { + continue; + } + UniValue entry(UniValue::VOBJ); entry.pushKV("project:", iter.first); entry.pushKV("zcd", iter.second.GetZCD()); entry.pushKV("WAS", iter.second.GetWAS().ToDouble()); + entry.pushKV("meets_greylist_criteria", iter.second.m_meets_greylisting_crit); autogreylist.push_back(entry); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 689aa6134c..372b9b437a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -215,6 +215,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "inspectaccrualsnapshot" , 0 }, { "listmanifests" , 0 }, { "listprojects" , 0 }, + { "getautogreylist" , 0 }, { "sendalert" , 2 }, { "sendalert" , 3 }, { "sendalert" , 4 }, diff --git a/src/test/gridcoin/superblock_tests.cpp b/src/test/gridcoin/superblock_tests.cpp index f37c08eadc..000abb7dd4 100644 --- a/src/test/gridcoin/superblock_tests.cpp +++ b/src/test/gridcoin/superblock_tests.cpp @@ -656,9 +656,9 @@ BOOST_AUTO_TEST_CASE(it_initializes_from_a_fallback_by_project_scraper_convergen { const ScraperStatsMeta meta; GRC::Superblock superblock = GRC::Superblock::FromConvergence( - GetTestConvergence(meta, true)); // Set fallback by project flag + GetTestConvergence(meta, true), 2); // Set fallback by project flag - BOOST_CHECK(superblock.m_version == GRC::Superblock::CURRENT_VERSION); + BOOST_CHECK(superblock.m_version == 2); BOOST_CHECK(superblock.m_convergence_hint == 0x11111111); // Manifest content hint not set for fallback convergence: BOOST_CHECK(superblock.m_manifest_content_hint == 0x00000000);