Modify scraper and superblock code (part 3)

This actually makes the changes to wire the autogreylist in.

src/gridcoin/project.cpp

@@ -308,8 +308,6 @@ AutoGreylist::AutoGreylist()
     : m_greylist_ptr(std::make_shared<Greylist>())
     , m_superblock_hash(uint256 {})
 {
-    //m_greylist_ptr = std::make_shared<Greylist>();
-
     Refresh();
 }
 
@@ -328,6 +326,21 @@ AutoGreylist::Greylist::size_type AutoGreylist::size() const
     return m_greylist_ptr->size();
 }
 
+bool AutoGreylist::Contains(const std::string& name, const bool& only_auto_greylisted) const
+{
+    auto iter = m_greylist_ptr->find(name);
+
+    if (iter != m_greylist_ptr->end()) {
+        if (only_auto_greylisted) {
+            return (only_auto_greylisted && iter->second.m_meets_greylisting_crit);
+        } else {
+            return true;
+        }
+    } else {
+        return false;
+    }
+}
+
 void AutoGreylist::Refresh()
 {
     LOCK(cs_main);
@@ -349,15 +362,28 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
     LOCK(lock);
 
     // We need the current whitelist, including all records except deleted. This will include greylisted projects,
-    // whether currently marked as manually greylisted from protocol or overridden to auto greylisted by the auto greylist class.
-    const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED);
+    // whether currently marked as manually greylisted from protocol or overridden to auto greylisted by the auto greylist class,
+    // based on the current state of the autogreylist. NOTE that the refresh_greylist is set to false here and MUST be this
+    // when called in the AutoGreylist class itself, to avoid an infinite loop.
+    const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED, false);
 
     m_greylist_ptr->clear();
 
+    unsigned int v3_superblock_count = 0;
+
+    if (superblock_ptr_in->m_version > 2) {
+        ++v3_superblock_count;
+    }
+
+    // Notice the superblock_ptr_in m_projects_all_cpid_total_credits MUST ALEADY BE POPULATED to record the TC state into
+    // the auto greylist.
     for (const auto& iter : whitelist) {
-        if (auto project = superblock_ptr_in->m_projects.Try(iter.m_name)) {
+        auto project = superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.find(iter.m_name);
+
+        if (project != superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) {
             // Record new greylist candidate entry baseline with the total credit for each project present in superblock.
-            m_greylist_ptr->insert(std::make_pair(iter.m_name, GreylistCandidateEntry(iter.m_name, project->m_total_credit)));
+            m_greylist_ptr->insert(std::make_pair(iter.m_name,
+                                                  GreylistCandidateEntry(iter.m_name, std::nearbyint(project->second))));
         } else {
             // Record new greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist,
             // but does not have a project entry in the superblock. This would be because the scrapers could not converge on the
@@ -373,7 +399,6 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
         index_ptr = GRC::BlockFinder::FindByHeight(superblock_ptr_in.m_height - 1);
     }
 
-    //SuperblockPtr superblock_ptr;
     unsigned int superblock_count = 1; // The 0 (baseline) superblock was processed above. Here we start with 1 and go up to 40
 
     while (index_ptr != nullptr && index_ptr->pprev != nullptr && superblock_count <= 40) {
@@ -396,32 +421,67 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
 
         SuperblockPtr superblock_ptr = block.GetClaim().m_superblock;
 
-        for (const auto& iter : whitelist) {
-            // This is guaranteed to succeed, because every whitelisted project was inserted as a new baseline entry above.
-            auto greylist_entry = m_greylist_ptr->find(iter.m_name);
-
-            if (auto project = superblock_ptr->m_projects.Try(iter.m_name)) {
-                // Update greylist candidate entry with the total credit for each project present in superblock.
-                greylist_entry->second.UpdateGreylistCandidateEntry(project->m_total_credit, superblock_count);
-            } else {
-                // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist,
-                // but does not have a project entry in this superblock. This would be because the scrapers could not converge on the
-                // project for this superblock.
-                greylist_entry->second.UpdateGreylistCandidateEntry(std::optional<uint64_t>(std::nullopt), superblock_count);
+        if (superblock_ptr->m_version > 2) {
+            for (const auto& iter : whitelist) {
+                // This is guaranteed to succeed, because every whitelisted project was inserted as a new baseline entry above.
+                auto greylist_entry = m_greylist_ptr->find(iter.m_name);
+
+                auto project = superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.find(iter.m_name);
+
+                if (project != superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) {
+                    // Update greylist candidate entry with the total credit for each project present in superblock.
+                    greylist_entry->second.UpdateGreylistCandidateEntry(project->second, superblock_count);
+                } else {
+                    // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist,
+                    // but does not have a project entry in this superblock. This would be because the scrapers could not converge on the
+                    // project for this superblock.
+                    greylist_entry->second.UpdateGreylistCandidateEntry(std::optional<uint64_t>(std::nullopt), superblock_count);
+                }
             }
+
+            ++v3_superblock_count;
         }
 
         ++superblock_count;
+
         index_ptr = index_ptr->pprev;
     }
 
-    // Purge candidate elements that do not meet auto greylist criteria.
-    for (auto iter = m_greylist_ptr->begin(); iter != m_greylist_ptr->end(); ) {
-        if (iter->second.GetZCD() < 7 && iter->second.GetWAS() >= Fraction(1, 10)) {
-            // Candidate greylist entry does not meet auto greylist criteria, so remove from greylist entry map.
-            iter = m_greylist_ptr->erase(iter);
-        } else {
-            iter++;
+    // Mark elements with whether they meet greylist criteria.
+    for (auto iter = m_greylist_ptr->begin(); iter != m_greylist_ptr->end(); ++iter) {
+        // the v3_superblock_count >= 2 test is to ensure there are at least two v3 superblocks with the total credits
+        // filled in to sample for the auto greylist determination. A two sb sample with positive change in TC will
+        // cause the ZCD and WAS rules to pass.
+        if (iter->second.GetZCD() < 7 && iter->second.GetWAS() >= Fraction(1, 10) && v3_superblock_count >= 2) {
+            iter->second.m_meets_greylisting_crit = true;
+         }
+    }
+}
+
+void AutoGreylist::RefreshWithSuperblock(Superblock& superblock)
+{
+    SuperblockPtr superblock_ptr;
+
+    // For the purposes of forming a superblock as the mining of a new block adding to the head of the chain, we will
+    // form a superblock ptr referencing the current head of the chain. The actual superblock will be the next block if it
+    // is added to the chain, for for the purposes here, this is what we want, because it is simply used to feed the
+    // overloaded version which takes the superblock_ptr and follows the chain backwards to do the greylist calculations.
+    superblock_ptr.Replace(superblock);
+
+    {
+        LOCK(cs_main);
+
+        superblock_ptr.Rebind(pindexBest);
+    }
+
+    RefreshWithSuperblock(superblock_ptr);
+
+    const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED, false);
+
+    // Update the superblock object with the project greylist status.
+    for (const auto& project : whitelist) {
+        if (project.m_status == ProjectEntryStatus::AUTO_GREYLISTED || project.m_status == ProjectEntryStatus::MAN_GREYLISTED) {
+            superblock.m_project_status.m_project_status.insert(std::make_pair(project.m_name, project.m_status));
         }
     }
 }
@@ -438,15 +498,27 @@ std::shared_ptr<AutoGreylist> AutoGreylist::GetAutoGreylistCache()
 // Class: Whitelist (Registry)
 // -----------------------------------------------------------------------------
 
-WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& filter) const
+WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& filter, const bool& refresh_greylist) const
 {
     LOCK(cs_lock);
 
-    //AutoGreylist::GetAutoGreylistCache()->Refresh();
+    std::shared_ptr<GRC::AutoGreylist> greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache();
+
+    if (refresh_greylist) {
+        greylist_ptr->Refresh();
+    }
 
     ProjectList projects;
 
-    for (const auto& iter : m_project_entries) {
+    // This is the override for automatic greylisting. If the AutoGreylist class refresh determines
+    // that the project meets greylisting criteria, it will be in the AutoGreylist object pointed to
+    // by the greylist_ptr. This will override the whitelist entries from the project whitelist registry.
+    for (auto iter : m_project_entries) {
+        if ((iter.second->m_status == ProjectEntryStatus::ACTIVE || iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED)
+            && greylist_ptr->Contains(iter.first)) {
+            iter.second->m_status = ProjectEntryStatus::AUTO_GREYLISTED;
+        }
+
         switch (filter) {
         case ProjectEntry::ProjectFilterFlag::ACTIVE:
             if (iter.second->m_status == ProjectEntryStatus::ACTIVE) {

src/gridcoin/project.h

@@ -508,6 +508,7 @@ class AutoGreylist
             , m_zcd_20_SB_count(0)
             , m_TC_7_SB_sum(0)
             , m_TC_40_SB_sum(0)
+            , m_meets_greylisting_crit(0)
             , m_TC_initial_bookmark(0)
             , m_TC_bookmark(0)
             , m_sb_from_baseline_processed(0)
@@ -519,6 +520,7 @@ class AutoGreylist
             , m_zcd_20_SB_count(0)
             , m_TC_7_SB_sum(0)
             , m_TC_40_SB_sum(0)
+            , m_meets_greylisting_crit(0)
             , m_TC_initial_bookmark(TC_initial_bookmark)
             , m_TC_bookmark(0)
             , m_sb_from_baseline_processed(0)
@@ -608,6 +610,7 @@ class AutoGreylist
         uint8_t m_zcd_20_SB_count;
         uint64_t m_TC_7_SB_sum;
         uint64_t m_TC_40_SB_sum;
+        bool m_meets_greylisting_crit;
 
     private:
         std::optional<uint64_t> m_TC_initial_bookmark; //!< This is a "reverse" bookmark - we are going backwards in SB's.
@@ -649,25 +652,40 @@ class AutoGreylist
     //!
     //! \param name Project name matching the contract key.
     //!
+    //! \param only_auto_greylisted A boolean that specifies whether the search is against all projects or only those
+    //! that meet auto greylisting criteria.
+    //!
     //! \return \c true if the auto greylist contains a project with matching name.
     //!
-    bool Contains(const std::string& name) const;
+    bool Contains(const std::string& name, const bool& only_auto_greylisted = true) const;
 
     void Refresh();
 
     //!
-    //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock. This mode is used
-    //! in the scraper during the construction of the superblock contract.
+    //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock pointer.
     //!
     //! Note that the AutoGreylist object refreshed this way will also be used to update the referenced superblock
     //! object
     //!
-    //! \param superblock The superblock with which to refresh the automatic greylist. This can be a candidate superblock
+    //! \param superblock_ptr The superblock pointer with which to refresh the automatic greylist. This can be a candidate superblock
     //! from a scraper convergence, or in the instance of this being called from Refresh(), could be the current
     //! superblock on the chain.
     //!
     void RefreshWithSuperblock(SuperblockPtr superblock_ptr_in);
 
+    //!
+    //! \brief This refreshes a local instantiation of the AutoGreylist from an input Superblock that is going to be associated
+    //! with the current head of the chain. This mode is used in the scraper during the construction of the superblock contract.
+    //!
+    //! Note that the AutoGreylist object refreshed this way will also be used to update the referenced superblock
+    //! object
+    //!
+    //! \param superblock The superblock with which to refresh the automatic greylist. This will generally be a candidate superblock
+    //! from a scraper convergence, and is used in the call chain from the miner loop. The superblock object project status
+    //! will be updated.
+    //!
+    void RefreshWithSuperblock(Superblock& superblock);
+
     static std::shared_ptr<AutoGreylist> GetAutoGreylistCache();
 
 private:
@@ -717,9 +735,11 @@ class Whitelist : public IContractHandler
 
     //!
     //! \brief Get a read-only view of the projects in the whitelist. The default filter is ACTIVE, which
-    //! provides the original ACTIVE project only view.
+    //! provides the original ACTIVE project only view. The refresh_greylist filter is used to refresh
+    //! the AutoGreylist as part of taking the snapshot.
     //!
-    WhitelistSnapshot Snapshot(const ProjectEntry::ProjectFilterFlag& filter = ProjectEntry::ProjectFilterFlag::ACTIVE) const;
+    WhitelistSnapshot Snapshot(const ProjectEntry::ProjectFilterFlag& filter = ProjectEntry::ProjectFilterFlag::ACTIVE,
+                               const bool& refresh_greylist = false) const;
 
     //!
     //! \brief Destroy the contract handler state to prepare for historical

src/gridcoin/superblock.cpp

@@ -4,6 +4,7 @@
 
 #include "chainparams.h"
 #include "compat/endian.h"
+#include "gridcoin/project.h"
 #include "hash.h"
 #include "main.h"
 #include <gridcoin/md5.h>
@@ -607,20 +608,29 @@ Superblock Superblock::FromConvergence(
     if (!stats.Convergence.bByParts) {
         superblock.m_manifest_content_hint
             = stats.Convergence.nUnderlyingManifestContentHash.GetUint64(0) >> 32;
+    } else {
+        ProjectIndex& projects = superblock.m_projects;
 
-        return superblock;
+               // Add hints created from the hashes of converged manifest parts to each
+               // superblock project section to assist receiving nodes with validation:
+               //
+        for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) {
+            const std::string& project_name = part_pair.first;
+            const CSplitBlob::CPart* part_data_ptr = part_pair.second;
+
+            projects.SetHint(project_name, part_data_ptr);
+        }
     }
 
-    ProjectIndex& projects = superblock.m_projects;
+    if (version > 2) {
+        // Populate the total credits into the superblock object from the convergence. This must be done BEFORE
+        // the auto greylist RefreshWithSuperblock is called.
+        superblock.m_projects_all_cpids_total_credits.Reset(stats.mScraperConvergedStats.m_total_credit_map);
 
-    // Add hints created from the hashes of converged manifest parts to each
-    // superblock project section to assist receiving nodes with validation:
-    //
-    for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) {
-        const std::string& project_name = part_pair.first;
-        const CSplitBlob::CPart* part_data_ptr = part_pair.second;
+        // Refresh the auto greylist and refresh this superblock with the greylist status.
+        std::shared_ptr<GRC::AutoGreylist> greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache();
 
-        projects.SetHint(project_name, part_data_ptr);
+        greylist_ptr->RefreshWithSuperblock(superblock);
     }
 
     return superblock;

src/rpc/blockchain.cpp

@@ -2634,13 +2634,21 @@ UniValue listprojects(const UniValue& params, bool fHelp)
 
 UniValue getautogreylist(const UniValue& params, bool fHelp)
 {
-    if (fHelp || params.size() > 0) {
+    if (fHelp || params.size() > 1) {
         throw runtime_error(
-            "getautogreylist \n"
+            "getautogreylist <bool> \n"
+            "\n"
+            "<bool> -> true to show all projects, including those that do not meet greylisting criteria. Defaults to false. \n"
             "\n"
             "Displays information about projects that meet auto greylisting criteria.");
     }
 
+    bool show_all_projects = false;
+
+    if (params.size()) {
+        show_all_projects = params[0].get_bool();
+    }
+
     UniValue res(UniValue::VOBJ);
 
     std::shared_ptr<GRC::AutoGreylist> greylist_ptr = GRC::AutoGreylist::GetAutoGreylistCache();
@@ -2650,11 +2658,16 @@ UniValue getautogreylist(const UniValue& params, bool fHelp)
     UniValue autogreylist(UniValue::VARR);
 
     for (auto iter : *greylist_ptr) {
+        if (!show_all_projects && !iter.second.m_meets_greylisting_crit) {
+            continue;
+        }
+
         UniValue entry(UniValue::VOBJ);
 
         entry.pushKV("project:", iter.first);
         entry.pushKV("zcd", iter.second.GetZCD());
         entry.pushKV("WAS", iter.second.GetWAS().ToDouble());
+        entry.pushKV("meets_greylist_criteria", iter.second.m_meets_greylisting_crit);
 
         autogreylist.push_back(entry);
     }

src/rpc/client.cpp

@@ -215,6 +215,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "inspectaccrualsnapshot" , 0 },
     { "listmanifests"          , 0 },
     { "listprojects"           , 0 },
+    { "getautogreylist"        , 0 },
     { "sendalert"              , 2 },
     { "sendalert"              , 3 },
     { "sendalert"              , 4 },

src/test/gridcoin/superblock_tests.cpp

@@ -656,9 +656,9 @@ BOOST_AUTO_TEST_CASE(it_initializes_from_a_fallback_by_project_scraper_convergen
 {
     const ScraperStatsMeta meta;
     GRC::Superblock superblock = GRC::Superblock::FromConvergence(
-        GetTestConvergence(meta, true)); // Set fallback by project flag
+        GetTestConvergence(meta, true), 2); // Set fallback by project flag
 
-    BOOST_CHECK(superblock.m_version == GRC::Superblock::CURRENT_VERSION);
+    BOOST_CHECK(superblock.m_version == 2);
     BOOST_CHECK(superblock.m_convergence_hint == 0x11111111);
     // Manifest content hint not set for fallback convergence:
     BOOST_CHECK(superblock.m_manifest_content_hint == 0x00000000);