Add passport expiry status #98

.github/workflows/generate-datasets-weekly.yml

@@ -36,9 +36,6 @@ jobs:
       - name: Build
         run: npm run build
 
-        # Generate Citizen Count CSV 🧮
-      - run: npm run generate-count
-
         # Generate Citizen Data CSV 🧮
       - run: npm run generate-csv
 
@@ -47,6 +44,9 @@ jobs:
 
         # Generate Historical Citizen Data CSVs 🧮
       - run: npm run generate-power
+
+        # Generate Citizen Count CSV 🧮
+      - run: npm run generate-count
 
       - name: Commit Changes
         run: |

data-sources/citizens/README.md

@@ -3,10 +3,6 @@
 https://etherscan.io/token/0x3337dac9f251d4e403d6030e18e3cfb6a2cb1333#inventory
 
 ## Datasets
-
-- `citizen-count-peer-week.csv`
-
-    - Contains the total number of Nation3 Citizens week by week, and the total number of [_active_](https://github.com/nation3/nationcred-datasets/tree/main/nationcred#definition-of-active) Citizens week by week.
 
 - `citizens.csv`
 
@@ -22,6 +18,10 @@ https://etherscan.io/token/0x3337dac9f251d4e403d6030e18e3cfb6a2cb1333#inventory
 
     - Contains a Citizen's voting power week by week.
 
+- `citizen-count-peer-week.csv`
+
+    - Contains the total number of Nation3 Citizens week by week, and the total number of [_active_](https://github.com/nation3/nationcred-datasets/tree/main/nationcred#definition-of-active) Citizens week by week.
+
 ## Install
 
 ```
@@ -36,12 +36,6 @@ npm install
 npm run build
 ```
 
-### Generate Citizen Count CSV
-
-```
-npm run generate-count
-```
-
 ### Generate Citizen Data CSV
 
 ```
@@ -59,3 +53,9 @@ npm run generate-json
 ```
 npm run generate-power
 ```
+
+### Generate Citizen Count CSV
+
+```
+npm run generate-count
+```

data-sources/citizens/generate-citizen-count-csv.ts

@@ -2,6 +2,7 @@ const Web3 = require('web3')
 const Passport = require('../abis/Passport.json')
 const csvWriter = require('csv-writer')
 const fs = require('fs')
+const Papa = require('papaparse')
 
 const web3 = new Web3('https://rpc.ankr.com/eth')
 console.info('web3.version:', web3.version)
@@ -25,7 +26,8 @@ async function loadPassportMintsByWeek() {
     header: [
       { id: 'week_end', title: 'week_end' },
       { id: 'total_citizens', title: 'total_citizens' },
-      { id: 'new_citizens', title: 'new_citizens' }
+      { id: 'new_citizens', title: 'new_citizens' },
+      { id: 'total_expired_passports', title: 'total_expired_passports' }
     ]
   })
   let csvRows = []
@@ -45,26 +47,28 @@ async function loadPassportMintsByWeek() {
     console.info('week:', `[${weekBeginDate.toISOString()} → ${weekEndDate.toISOString()}]`)
 
     let newCitizensCount: number = 0
-    if (id < nextId) {
-      while (await getTimestamp(id) < (weekEndDate.getTime() / 1000)) {
-        id++
-        console.info('id:', id)
-
-        newCitizensCount++
-        console.info('newCitizensCount:', newCitizensCount)
-
-        if (id == nextId) {
-          console.info('Reached last passport ID:', id)
-          break
-        }
+    while (await getTimestamp(id) < (weekEndDate.getTime() / 1000)) {
+      console.info('id:', id)
+
+      newCitizensCount++
+      console.info('newCitizensCount:', newCitizensCount)
+
+      id++
+      if (id == nextId) {
+        console.info('Reached last passport ID:', (nextId - 1))
+        break
       }
     }
+
+    const totalExpiredPassports: number = getTotalExpiredPassports(weekEndDate, id)
+    console.info('totalExpiredPassports:', totalExpiredPassports)
 
     // Export to CSV
     const csvRow = {
       week_end: weekEndDate.toISOString().substring(0, 10),
-      total_citizens: id + 1,
-      new_citizens: newCitizensCount
+      total_citizens: id,
+      new_citizens: newCitizensCount,
+      total_expired_passports: totalExpiredPassports
     }
     csvRows.push(csvRow)
 
@@ -85,4 +89,42 @@ async function getTimestamp(id: number): Promise<number> {
   return await PassportContract.methods.timestampOf(id).call()
 }
 
+function getTotalExpiredPassports(weekEndDate: Date, maxPassportID: number): number {
+  console.info('getTotalExpiredPassports')
+
+  const weekEndDateString: string = weekEndDate.toISOString().substring(0, 10)
+
+  let totalExpiredPassports = 0
+  for (let passportID = 0; passportID < maxPassportID; passportID++) {
+    console.info(`weekEndDate: ${weekEndDateString}, passportID: ${passportID}`)
+
+    // Fetch voting power data from the citizen's data CSV
+    const citizenFilePath: string = `output/citizen-${passportID}.csv`
+    console.info('Fetching citizen data:', citizenFilePath)
+    const file: File = fs.readFileSync(citizenFilePath)
+    const csvData = file.toString()
+    // console.info('csvData:\n', csvData)
+    Papa.parse(csvData, {
+      header: true,
+      skipEmptyLines: true,
+      dynamicTyping: true,
+      complete: (result: any) => {
+        // console.info('result:', result)
+        result.data.forEach((row: any, i: number) => {
+          if (row.week_end == weekEndDateString) {
+            console.info(`row.week_end ${row.week_end}, row.voting_power: ${row.voting_power}`)
+            if (row.voting_power < 1.5) {
+              // https://etherscan.io/address/0x279c0b6bfcbba977eaf4ad1b2ffe3c208aa068ac#readContract#F9
+              console.info('Passport ID expired:', passportID)
+              totalExpiredPassports++
+            }
+          }
+        })
+      }
+    })
+  }
+
+  return totalExpiredPassports
+}
+
 export {}

data-sources/citizens/output/citizen-count-per-week.csv

@@ -1,62 +1,62 @@
-week_end,total_citizens,new_citizens
-2022-06-05,156,155
-2022-06-12,158,2
-2022-06-19,161,3
-2022-06-26,164,3
-2022-07-03,168,4
-2022-07-10,168,0
-2022-07-17,168,0
-2022-07-24,171,3
-2022-07-31,173,2
-2022-08-07,174,1
-2022-08-14,175,1
-2022-08-21,176,1
-2022-08-28,176,0
-2022-09-04,177,1
-2022-09-11,178,1
-2022-09-18,179,1
-2022-09-25,179,0
-2022-10-02,179,0
-2022-10-09,182,3
-2022-10-16,183,1
-2022-10-23,185,2
-2022-10-30,185,0
-2022-11-06,189,4
-2022-11-13,222,33
-2022-11-20,232,10
-2022-11-27,234,2
-2022-12-04,236,2
-2022-12-11,241,5
-2022-12-18,242,1
-2022-12-25,242,0
-2023-01-01,242,0
-2023-01-08,245,3
-2023-01-15,249,4
-2023-01-22,251,2
-2023-01-29,255,4
-2023-02-05,258,3
-2023-02-12,259,1
-2023-02-19,259,0
-2023-02-26,260,1
-2023-03-05,261,1
-2023-03-12,261,0
-2023-03-19,262,1
-2023-03-26,263,1
-2023-04-02,264,1
-2023-04-09,264,0
-2023-04-16,265,1
-2023-04-23,265,0
-2023-04-30,266,1
-2023-05-07,266,0
-2023-05-14,266,0
-2023-05-21,266,0
-2023-05-28,266,0
-2023-06-04,266,0
-2023-06-11,266,0
-2023-06-18,266,0
-2023-06-25,266,0
-2023-07-02,267,1
-2023-07-09,270,3
-2023-07-16,273,3
-2023-07-23,274,1
-2023-07-30,275,1
+week_end,total_citizens,new_citizens,total_expired_passports
+2022-06-05,155,155,0
+2022-06-12,157,2,0
+2022-06-19,160,3,0
+2022-06-26,163,3,0
+2022-07-03,167,4,0
+2022-07-10,167,0,0
+2022-07-17,167,0,0
+2022-07-24,170,3,0
+2022-07-31,172,2,0
+2022-08-07,173,1,0
+2022-08-14,174,1,0
+2022-08-21,175,1,0
+2022-08-28,175,0,0
+2022-09-04,176,1,0
+2022-09-11,177,1,0
+2022-09-18,178,1,0
+2022-09-25,178,0,0
+2022-10-02,178,0,0
+2022-10-09,181,3,0
+2022-10-16,182,1,0
+2022-10-23,184,2,0
+2022-10-30,184,0,0
+2022-11-06,188,4,0
+2022-11-13,221,33,1
+2022-11-20,231,10,1
+2022-11-27,233,2,1
+2022-12-04,235,2,1
+2022-12-11,240,5,1
+2022-12-18,241,1,1
+2022-12-25,241,0,1
+2023-01-01,241,0,1
+2023-01-08,244,3,1
+2023-01-15,248,4,1
+2023-01-22,250,2,1
+2023-01-29,254,4,2
+2023-02-05,257,3,2
+2023-02-12,258,1,2
+2023-02-19,258,0,2
+2023-02-26,259,1,2
+2023-03-05,260,1,2
+2023-03-12,260,0,2
+2023-03-19,261,1,2
+2023-03-26,262,1,2
+2023-04-02,263,1,2
+2023-04-09,263,0,3
+2023-04-16,264,1,4
+2023-04-23,264,0,7
+2023-04-30,265,1,6
+2023-05-07,265,0,7
+2023-05-14,265,0,7
+2023-05-21,265,0,8
+2023-05-28,265,0,8
+2023-06-04,265,0,8
+2023-06-11,265,0,11
+2023-06-18,265,0,25
+2023-06-25,265,0,28
+2023-07-02,266,1,29
+2023-07-09,269,3,28
+2023-07-16,272,3,30
+2023-07-23,273,1,34
+2023-07-30,274,1,44

data-sources/citizens/package.json

@@ -28,6 +28,7 @@
     "ethereum-block-by-date": "^1.4.6",
     "ethers": "^5.7.2",
     "jsonfile": "^6.1.0",
+    "papaparse": "^5.4.1",
     "typescript": "^4.8.4",
     "web3": "^1.8.1"
   }