Add NP rewards with dashboard logs (#978)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

rs/dre-canisters/trustworthy-node-metrics/add_np.py

@@ -1,60 +1,51 @@
-import csv
+import json
 import subprocess
-
+import time
 # Define the file paths and constants
 csv_file_path = "node_info_api.csv" 
 did_file_path = "rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics/trustworthy-node-metrics.did"
 network = "ic"
 canister_name = "trustworthy-node-metrics"
 
-# Function to generate the dfx command for node_metadata
-def generate_dfx_command(node_mappings):
-    nodes = [
-        f'record {{ node_id = principal "{row["node_id"]}"; node_provider_id = principal "{row["node_provider_id"]}"; node_provider_name = "{row["node_provider_name"]}"; }}'
-        for row in node_mappings
-    ]
-    # Prepare the argument for the canister call
-    node_metadata_args = f"vec {{ {'; '.join(nodes)} ; }}"
-
+# Function to generate the dfx command for a single node
+def generate_dfx_command(node_data):
     command = [
-        "dfx", "canister", "call", canister_name, "node_metadata",
-        f'({node_metadata_args})',
+        "dfx", "canister", "call", canister_name, "np_rewardable_backfill",
+        f'(record {{ node_provider_id = principal "{node_data["node_provider_id"]}"; region = "{node_data["region"]}"; node_type = "{node_data["node_type"]}"; count = {node_data["count"]}; }})',
         "--candid", did_file_path,
         "--network", network
     ]
-
     return command
 
-# Read the CSV file and filter out missing providers
-def read_csv(csv_file_path):
-    node_mappings = []
-    with open(csv_file_path, mode='r') as file:
-        csv_reader = csv.DictReader(file)
-        for row in csv_reader:
-            if row['node_provider_id'] != "missing":  # Ignore missing providers
-                node_mappings.append({
-                    "node_id": row["node_id"],
-                    "node_provider_id": row["node_provider_id"],
-                    "node_provider_name": row["node_provider_name"]
-                })
+# Read the .json file and extract node information
+def read_json(json_file_path):
+    with open(json_file_path, mode='r') as file:
+        # Each line is a separate JSON object, so we use a list to store them
+        node_mappings = [json.loads(line) for line in file]
     return node_mappings
 
+# Execute the dfx command for each node
+def execute_dfx_command(command):
+    try:
+        result = subprocess.run(command, capture_output=True, text=True)
+        print("Command executed successfully:", result.stdout)
+        if result.stderr:
+            print("Error:", result.stderr)
+    except Exception as e:
+        print("Failed to execute the dfx command:", str(e))
+
 # Main execution
 if __name__ == "__main__":
-    # Read the CSV and get node mappings
-    node_mappings = read_csv(csv_file_path)
+    # Read the JSON file and get node mappings
+    node_mappings = read_json(json_file_path)
 
     if node_mappings:
-        # Generate the dfx command
-        dfx_command = generate_dfx_command(node_mappings)
-
-        # Execute the dfx command
-        try:
-            result = subprocess.run(dfx_command, capture_output=True, text=True)
-            print(result.stdout)
-            if result.stderr:
-                print("Error:", result.stderr)
-        except Exception as e:
-            print("Failed to execute the dfx command:", str(e))
+        # Loop through each node and execute a separate command
+        for node in node_mappings:
+            # Generate the dfx command for the current node
+            dfx_command = generate_dfx_command(node)
+
+            # Execute the dfx command for the current node
+            execute_dfx_command(dfx_command)
     else:
-        print("No valid node mappings found.")
+        print("No valid node mappings found.")

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/components/NodePerformanceChart.tsx

@@ -3,7 +3,7 @@ import { ChartData, formatDateToUTC, generateChartData, LoadingIndicator, NodeMe
 import { PeriodFilter } from './FilterBar';
 import { Box, Grid } from '@mui/material';
 import PerformanceChart from './PerformanceChart';
-import { NodeRewards } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
+import { NodeRewardsMultiplier } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
 import { ExportTable } from './ExportTable';
 import { GridColDef, GridRowsProp } from '@mui/x-data-grid';
 import { Principal } from '@dfinity/principal';
@@ -16,7 +16,7 @@ export interface NodePerformanceChartProps {
   }
 
 export const NodePerformanceChart: React.FC<NodePerformanceChartProps> = ({ node, periodFilter }) => {
-    const [performanceData, setPerformanceData] = useState<NodeRewards | null>(null);
+    const [performanceData, setPerformanceData] = useState<NodeRewardsMultiplier | null>(null);
     const [isLoading, setIsLoading] = useState(true);
 
     useEffect(() => {
@@ -36,7 +36,7 @@ export const NodePerformanceChart: React.FC<NodePerformanceChartProps> = ({ node
     }
 
     const performanceDailyData: ChartData[] = generateChartData(periodFilter, performanceData.daily_node_metrics);
-    const failureRateAvg = Math.round(performanceData.rewards_computation.failure_rate * 100);
+    const failureRateAvg = Math.round(performanceData.rewards_multiplier.failure_rate * 100);
 
     const rows: GridRowsProp = performanceData.daily_node_metrics.map((data, index) => {
         return { 
@@ -62,7 +62,7 @@ export const NodePerformanceChart: React.FC<NodePerformanceChartProps> = ({ node
     return (
         <>
             <Grid item xs={12} md={6}>
-                <NodeMetricsStats stats={performanceData.rewards_computation} />
+                <NodeMetricsStats stats={performanceData.rewards_multiplier} />
             </Grid>
             <Grid item xs={12} md={6}>
                 <Box sx={boxStyleWidget('right')}>

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/components/NodeProviderChart.tsx

@@ -53,7 +53,7 @@ export const NodeProviderChart: React.FC<NodeProviderChartProps> = ({ provider,
     const providerNodeMetrics = providerRewards.nodes_rewards;
 
     const highFailureRateChart = providerNodeMetrics
-        .sort((a, b) => b.rewards_computation.failure_rate - a.rewards_computation.failure_rate)
+        .sort((a, b) => b.rewards_multiplier.failure_rate - a.rewards_multiplier.failure_rate)
         .slice(0, 3)
         .flatMap(nodeMetrics => {
             const chartData = generateChartData(periodFilter, nodeMetrics.daily_node_metrics);

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/components/NodeProviderPage.tsx

@@ -63,7 +63,7 @@ export const NodeProviderPage: React.FC<NodeProviderPageProps> = ({ nodeMetadata
                 </Grid>
                 <Grid item xs={12}>
                     <Typography variant="h6" component="div" >
-                        Provider Rewards
+                        Last Rewarding Period
                     </Typography>
                     <Divider/>
                 </Grid>

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/components/NodeProviderRewards.tsx

@@ -1,10 +1,12 @@
 import React, { useEffect, useState } from 'react';
 import { getLatestRewardRange, LoadingIndicator, setNodeProviderRewardsData } from '../utils/utils';
-import { Box, Grid } from '@mui/material';
+import { Box, Grid, Typography } from '@mui/material';
 import { Principal } from '@dfinity/principal';
 import { NodeProviderRewards } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
 import { WidgetNumber } from './Widgets';
 import { boxStyleWidget } from '../Styles';
+import { ExportTable } from './ExportTable';
+import { GridColDef, GridRowsProp } from '@mui/x-data-grid';
 
 export interface NodeProviderRewardsChartProps {
     provider: string;
@@ -36,6 +38,21 @@ export const NodeProviderRewardsChart: React.FC<NodeProviderRewardsChartProps> =
         return <p>No latestNodeRewards</p>;
     }
     const distribution_date = new Date(Number(latestProviderRewards.ts_distribution) * 1000);
+    const rows: GridRowsProp = latestProviderRewards.computation_log.map((data, index) => {
+        return { 
+            id: index,
+            col0: index,
+            col1: data.reason, 
+            col2: data.operation,
+            col3: data.result
+            };
+    });
+    const colDef: GridColDef[] = [
+        { field: 'col0', headerName: 'Step', width: 100},
+        { field: 'col1', headerName: 'Description', width: 1500},
+        { field: 'col2', headerName: 'Operation', width: 500 },
+        { field: 'col3', headerName: 'Result', width: 200 },
+        ];
 
     return (
         <>
@@ -49,10 +66,16 @@ export const NodeProviderRewardsChart: React.FC<NodeProviderRewardsChartProps> =
             </Grid>
             <Grid item xs={12} md={6}>
             <Box sx={boxStyleWidget('right')}>
-                <WidgetNumber value={Math.round(Number(latestProviderRewards.rewards_xdr) / Number(latestProviderRewards.xdr_conversion_rate)).toString()} title="Expected Rewards ICP (Excluding never assigned nodes)"  sxValue={{ color: '#FFCC00' }} />
+                <WidgetNumber value={Math.round(Number(latestProviderRewards.rewards_xdr) / Number(latestProviderRewards.xdr_conversion_rate)).toString()} title="Expected Rewards ICP"  sxValue={{ color: '#FFCC00' }} />
                 <WidgetNumber value={Math.round(Number(rewards_xdr_old[0]) / 100000000).toString()} title="Last Rewards ICP Received"  sxValue={{ color: '#FFCC00' }} />
             </Box>
             </Grid>
+            <Grid item xs={12} md={12}>
+                <Typography variant="body1" gutterBottom>
+                    Computation Log
+                </Typography>
+                <ExportTable colDef={colDef} rows={rows}/>
+            </Grid>
 
         </> 
     );

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/components/NodeRewardsChart.tsx

@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react';
 import { ChartData, generateChartData, getLatestRewardRange, LoadingIndicator, NodeMetricsStats, NodePerformanceStats, setNodeRewardsData } from '../utils/utils';
 import { Grid, Typography } from '@mui/material';
 import PerformanceChart from './PerformanceChart';
-import { NodeRewards } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
+import { NodeRewardsMultiplier } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
 import RewardsInfo, { LinearReductionChart } from './RewardsInfo';
 import { Principal } from '@dfinity/principal';
 import { ExportTable } from './ExportTable';
@@ -14,7 +14,7 @@ export interface NodeRewardsChartProps {
 
 export const NodeRewardsChart: React.FC<NodeRewardsChartProps> = ({ node }) => {
     const latestRewardRange = getLatestRewardRange();
-    const [latestNodeRewards, setLatestNodeRewards] = useState<NodeRewards | null>(null);
+    const [latestNodeRewards, setLatestNodeRewards] = useState<NodeRewardsMultiplier | null>(null);
     const [isLoading, setIsLoading] = useState(true);
 
     useEffect(() => {
@@ -33,10 +33,10 @@ export const NodeRewardsChart: React.FC<NodeRewardsChartProps> = ({ node }) => {
     }
 
     const rewardsDailyData: ChartData[] = generateChartData(latestRewardRange, latestNodeRewards.daily_node_metrics);
-    const failureRateAvg = Math.round((latestNodeRewards.rewards_computation.failure_rate) * 100)
-    const rewardsMultiplier = Math.round((latestNodeRewards.rewards_computation.rewards_multiplier) * 100);
+    const failureRateAvg = Math.round((latestNodeRewards.rewards_multiplier.failure_rate) * 100)
+    const rewardsMultiplier = Math.round((latestNodeRewards.rewards_multiplier.rewards_multiplier) * 100);
     const rewardsReduction = 100 - rewardsMultiplier;
-    const rows: GridRowsProp = latestNodeRewards.rewards_computation.computation_log.map((data, index) => {
+    const rows: GridRowsProp = latestNodeRewards.rewards_multiplier.computation_log.map((data, index) => {
         return { 
             id: index,
             col0: index,
@@ -55,7 +55,7 @@ export const NodeRewardsChart: React.FC<NodeRewardsChartProps> = ({ node }) => {
     return (
         <>
             <Grid item xs={12} md={6}>
-                <NodeMetricsStats stats={latestNodeRewards.rewards_computation} />
+                <NodeMetricsStats stats={latestNodeRewards.rewards_multiplier} />
             </Grid>
             <Grid item xs={12} md={6}>
                 <NodePerformanceStats 

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/components/RewardTable.tsx

@@ -7,10 +7,10 @@ import TableHead from '@mui/material/TableHead';
 import TableRow from '@mui/material/TableRow';
 import Paper from '@mui/material/Paper';
 import { SxProps, Theme } from '@mui/material';
-import { NodeRewards } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
+import { NodeRewardsMultiplier } from '../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did';
 
 interface RewardTableProps {
-    nodeRewards: NodeRewards[],
+    nodeRewards: NodeRewardsMultiplier[],
     sx?: SxProps<Theme>;
   }
 
@@ -35,7 +35,7 @@ const RewardTable: React.FC<RewardTableProps> = ({ nodeRewards }) => {
                 {nodeMetrics.node_id.toText()}
               </TableCell>
               <TableCell component="th" scope="row">
-                {nodeMetrics.rewards_computation.failure_rate * 100}%
+                {nodeMetrics.rewards_multiplier.failure_rate * 100}%
               </TableCell>
             </TableRow>
           ))}

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-frontend/src/utils/utils.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { Principal } from "@dfinity/principal";
 import { trustworthy_node_metrics } from "../../../declarations/trustworthy-node-metrics";
-import { DailyNodeMetrics, NodeRewardsArgs, NodeRewards, NodeProviderRewardsArgs, NodeProviderRewards } from "../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did";
+import { DailyNodeMetrics, NodeRewardsArgs, NodeRewardsMultiplier, NodeProviderRewardsArgs, NodeProviderRewards } from "../../../declarations/trustworthy-node-metrics/trustworthy-node-metrics.did";
 import { PeriodFilter } from "../components/FilterBar";
 import { Box, CircularProgress } from "@mui/material";
 import { WidgetNumber } from '../components/Widgets';
@@ -130,7 +130,7 @@ export const getLatestRewardRange = () => {
 export const setNodeRewardsData = async (
   periodFilter: PeriodFilter, 
   node_id: Principal,
-  setNodeRewards: React.Dispatch<React.SetStateAction<NodeRewards | null>>,
+  setNodeRewards: React.Dispatch<React.SetStateAction<NodeRewardsMultiplier | null>>,
   setIsLoading: React.Dispatch<React.SetStateAction<boolean>>) => {
   try {
     setIsLoading(true);
@@ -186,7 +186,7 @@ export const LoadingIndicator: React.FC = () => (
   </Box>
 );
 
-export const NodeMetricsStats: React.FC<{ stats: NodeRewards['rewards_computation'] | null }> = ({ stats }) => (
+export const NodeMetricsStats: React.FC<{ stats: NodeRewardsMultiplier['rewards_multiplier'] | null }> = ({ stats }) => (
   <Box sx={boxStyleWidget('left')}>
       <WidgetNumber value={stats ? stats.days_assigned.toString() : "0"} title="Days Assigned" />
       <WidgetNumber value={stats ? stats.days_unassigned.toString() : "0"} title="Days Unassigned" />

rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-types/src/types.rs

@@ -37,6 +37,30 @@ impl Storable for MonthlyNodeProviderRewardsStored {
     };
 }
 
+#[derive(Debug, Deserialize, Serialize, CandidType, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct NodeProviderRewardableKey {
+    pub node_provider_id: Principal,
+    pub region: String,
+    pub node_type: String,
+}
+
+const MAX_VALUE_SIZE_REWARDABLE_NODES: u32 = 300;
+
+impl Storable for NodeProviderRewardableKey {
+    fn to_bytes(&self) -> std::borrow::Cow<[u8]> {
+        Cow::Owned(Encode!(self).unwrap())
+    }
+
+    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
+        Decode!(bytes.as_ref(), Self).unwrap()
+    }
+
+    const BOUND: Bound = Bound::Bounded {
+        max_size: MAX_VALUE_SIZE_REWARDABLE_NODES,
+        is_fixed_size: false,
+    };
+}
+
 #[derive(Debug, Deserialize, Serialize, CandidType, Clone)]
 pub struct NodeMetricsStored {
     pub subnet_assigned: Principal,
@@ -68,7 +92,7 @@ pub struct NodeRewardRatesStored {
     pub rewards_rates: NodeRewardRates,
 }
 
-const MAX_VALUE_SIZE_BYTES_REWARD_RATES: u32 = 133;
+const MAX_VALUE_SIZE_BYTES_REWARD_RATES: u32 = 200;
 
 impl Storable for NodeRewardRatesStored {
     fn to_bytes(&self) -> std::borrow::Cow<[u8]> {
@@ -116,7 +140,7 @@ pub struct NodeMetadataStoredV2 {
     pub node_type: String,
 }
 
-const MAX_VALUE_SIZE_BYTES_NODE_METADATA: u32 = 204;
+const MAX_VALUE_SIZE_BYTES_NODE_METADATA: u32 = 400;
 
 impl Storable for NodeMetadataStoredV2 {
     fn to_bytes(&self) -> std::borrow::Cow<[u8]> {
@@ -212,7 +236,7 @@ impl DailyNodeMetrics {
 }
 
 #[derive(Debug, Deserialize, CandidType)]
-pub struct RewardMultiplierResult {
+pub struct RewardsMultiplier {
     pub days_assigned: u64,
     pub days_unassigned: u64,
     pub rewards_multiplier: f64,
@@ -225,11 +249,17 @@ pub struct RewardMultiplierResult {
 }
 
 #[derive(Debug, Deserialize, CandidType)]
-pub struct NodeRewards {
+pub struct NodeRewardsMultiplier {
     pub node_id: Principal,
     pub daily_node_metrics: Vec<DailyNodeMetrics>,
     pub node_rate: NodeRewardRate,
-    pub rewards_computation: RewardMultiplierResult,
+    pub rewards_multiplier: RewardsMultiplier,
+}
+
+pub struct NodeProviderRewardsComputation {
+    pub rewards_xdr: u64,
+    pub rewards_xdr_no_reduction: u64,
+    pub computation_log: Vec<OperationExecutorLog>,
 }
 
 #[derive(Debug, Deserialize, CandidType)]
@@ -240,7 +270,8 @@ pub struct NodeProviderRewards {
     pub rewards_xdr_old: Option<u64>,
     pub ts_distribution: u64,
     pub xdr_conversion_rate: Option<u64>,
-    pub nodes_rewards: Vec<NodeRewards>,
+    pub nodes_rewards: Vec<NodeRewardsMultiplier>,
+    pub computation_log: Vec<OperationExecutorLog>,
 }
 
 #[derive(Debug, Deserialize, CandidType)]