From 31f93cfe166ad7b601484fe8f3c12be45c8fd23d Mon Sep 17 00:00:00 2001
From: Aahna Ashina <95955389+aahna-ashina@users.noreply.github.com>
Date: Fri, 28 Jul 2023 10:17:06 +0800
Subject: [PATCH 1/3] fix: wrong total total citizens count (#98)

#98
---
 .../citizens/generate-citizen-count-csv.ts    |  24 ++--
 .../output/citizen-count-per-week.csv         | 120 +++++++++---------
 2 files changed, 71 insertions(+), 73 deletions(-)

diff --git a/data-sources/citizens/generate-citizen-count-csv.ts b/data-sources/citizens/generate-citizen-count-csv.ts
index d3d6cd42d..0289f62a8 100644
--- a/data-sources/citizens/generate-citizen-count-csv.ts
+++ b/data-sources/citizens/generate-citizen-count-csv.ts
@@ -45,25 +45,23 @@ async function loadPassportMintsByWeek() {
     console.info('week:', `[${weekBeginDate.toISOString()} → ${weekEndDate.toISOString()}]`)
 
     let newCitizensCount: number = 0
-    if (id < nextId) {
-      while (await getTimestamp(id) < (weekEndDate.getTime() / 1000)) {
-        id++
-        console.info('id:', id)
-
-        newCitizensCount++
-        console.info('newCitizensCount:', newCitizensCount)
-
-        if (id == nextId) {
-          console.info('Reached last passport ID:', id)
-          break
-        }
+    while (await getTimestamp(id) < (weekEndDate.getTime() / 1000)) {
+      console.info('id:', id)
+      
+      newCitizensCount++
+      console.info('newCitizensCount:', newCitizensCount)
+      
+      id++
+      if (id == nextId) {
+        console.info('Reached last passport ID:', (nextId - 1))
+        break
       }
     }
     
     // Export to CSV
     const csvRow = {
       week_end: weekEndDate.toISOString().substring(0, 10),
-      total_citizens: id + 1,
+      total_citizens: id,
       new_citizens: newCitizensCount
     }
     csvRows.push(csvRow)
diff --git a/data-sources/citizens/output/citizen-count-per-week.csv b/data-sources/citizens/output/citizen-count-per-week.csv
index b9c2ec24d..dc6018ccf 100644
--- a/data-sources/citizens/output/citizen-count-per-week.csv
+++ b/data-sources/citizens/output/citizen-count-per-week.csv
@@ -1,61 +1,61 @@
 week_end,total_citizens,new_citizens
-2022-06-05,156,155
-2022-06-12,158,2
-2022-06-19,161,3
-2022-06-26,164,3
-2022-07-03,168,4
-2022-07-10,168,0
-2022-07-17,168,0
-2022-07-24,171,3
-2022-07-31,173,2
-2022-08-07,174,1
-2022-08-14,175,1
-2022-08-21,176,1
-2022-08-28,176,0
-2022-09-04,177,1
-2022-09-11,178,1
-2022-09-18,179,1
-2022-09-25,179,0
-2022-10-02,179,0
-2022-10-09,182,3
-2022-10-16,183,1
-2022-10-23,185,2
-2022-10-30,185,0
-2022-11-06,189,4
-2022-11-13,222,33
-2022-11-20,232,10
-2022-11-27,234,2
-2022-12-04,236,2
-2022-12-11,241,5
-2022-12-18,242,1
-2022-12-25,242,0
-2023-01-01,242,0
-2023-01-08,245,3
-2023-01-15,249,4
-2023-01-22,251,2
-2023-01-29,255,4
-2023-02-05,258,3
-2023-02-12,259,1
-2023-02-19,259,0
-2023-02-26,260,1
-2023-03-05,261,1
-2023-03-12,261,0
-2023-03-19,262,1
-2023-03-26,263,1
-2023-04-02,264,1
-2023-04-09,264,0
-2023-04-16,265,1
-2023-04-23,265,0
-2023-04-30,266,1
-2023-05-07,266,0
-2023-05-14,266,0
-2023-05-21,266,0
-2023-05-28,266,0
-2023-06-04,266,0
-2023-06-11,266,0
-2023-06-18,266,0
-2023-06-25,266,0
-2023-07-02,267,1
-2023-07-09,270,3
-2023-07-16,273,3
-2023-07-23,274,1
+2022-06-05,155,155
+2022-06-12,157,2
+2022-06-19,160,3
+2022-06-26,163,3
+2022-07-03,167,4
+2022-07-10,167,0
+2022-07-17,167,0
+2022-07-24,170,3
+2022-07-31,172,2
+2022-08-07,173,1
+2022-08-14,174,1
+2022-08-21,175,1
+2022-08-28,175,0
+2022-09-04,176,1
+2022-09-11,177,1
+2022-09-18,178,1
+2022-09-25,178,0
+2022-10-02,178,0
+2022-10-09,181,3
+2022-10-16,182,1
+2022-10-23,184,2
+2022-10-30,184,0
+2022-11-06,188,4
+2022-11-13,221,33
+2022-11-20,231,10
+2022-11-27,233,2
+2022-12-04,235,2
+2022-12-11,240,5
+2022-12-18,241,1
+2022-12-25,241,0
+2023-01-01,241,0
+2023-01-08,244,3
+2023-01-15,248,4
+2023-01-22,250,2
+2023-01-29,254,4
+2023-02-05,257,3
+2023-02-12,258,1
+2023-02-19,258,0
+2023-02-26,259,1
+2023-03-05,260,1
+2023-03-12,260,0
+2023-03-19,261,1
+2023-03-26,262,1
+2023-04-02,263,1
+2023-04-09,263,0
+2023-04-16,264,1
+2023-04-23,264,0
+2023-04-30,265,1
+2023-05-07,265,0
+2023-05-14,265,0
+2023-05-21,265,0
+2023-05-28,265,0
+2023-06-04,265,0
+2023-06-11,265,0
+2023-06-18,265,0
+2023-06-25,265,0
+2023-07-02,266,1
+2023-07-09,269,3
+2023-07-16,272,3
+2023-07-23,273,1

From ee879a649e99fbc0d11fa309309d1fc837315123 Mon Sep 17 00:00:00 2001
From: Aahna Ashina <95955389+aahna-ashina@users.noreply.github.com>
Date: Sat, 29 Jul 2023 21:26:33 +0800
Subject: [PATCH 2/3] feat: add passport expiry status (#98)

- run passport expiry data generation _after_ citizens data generation
- check if veNATION balance is below 1.5
---
 .../workflows/generate-datasets-weekly.yml    |   6 +-
 data-sources/citizens/README.md               |  20 +--
 .../citizens/generate-citizen-count-csv.ts    |  48 ++++++-
 .../output/citizen-count-per-week.csv         | 122 +++++++++---------
 data-sources/citizens/package-lock.json       |  11 ++
 data-sources/citizens/package.json            |   1 +
 6 files changed, 132 insertions(+), 76 deletions(-)

diff --git a/.github/workflows/generate-datasets-weekly.yml b/.github/workflows/generate-datasets-weekly.yml
index aa5016290..2f6d197df 100644
--- a/.github/workflows/generate-datasets-weekly.yml
+++ b/.github/workflows/generate-datasets-weekly.yml
@@ -36,9 +36,6 @@ jobs:
       - name: Build
         run: npm run build
 
-        # Generate Citizen Count CSV 🧮
-      - run: npm run generate-count
-
         # Generate Citizen Data CSV 🧮
       - run: npm run generate-csv
 
@@ -47,6 +44,9 @@ jobs:
 
         # Generate Historical Citizen Data CSVs 🧮
       - run: npm run generate-power
+
+        # Generate Citizen Count CSV 🧮
+      - run: npm run generate-count
       
       - name: Commit Changes
         run: |
diff --git a/data-sources/citizens/README.md b/data-sources/citizens/README.md
index 9b89debb8..df4802a50 100644
--- a/data-sources/citizens/README.md
+++ b/data-sources/citizens/README.md
@@ -3,10 +3,6 @@
 https://etherscan.io/token/0x3337dac9f251d4e403d6030e18e3cfb6a2cb1333#inventory
 
 ## Datasets
-
-- `citizen-count-peer-week.csv`
-
-    - Contains the total number of Nation3 Citizens week by week, and the total number of [_active_](https://github.com/nation3/nationcred-datasets/tree/main/nationcred#definition-of-active) Citizens week by week.
     
 - `citizens.csv`
 
@@ -22,6 +18,10 @@ https://etherscan.io/token/0x3337dac9f251d4e403d6030e18e3cfb6a2cb1333#inventory
 
     - Contains a Citizen's voting power week by week.
 
+- `citizen-count-peer-week.csv`
+
+    - Contains the total number of Nation3 Citizens week by week, and the total number of [_active_](https://github.com/nation3/nationcred-datasets/tree/main/nationcred#definition-of-active) Citizens week by week.
+
 ## Install
 
 ```
@@ -36,12 +36,6 @@ npm install
 npm run build
 ```
 
-### Generate Citizen Count CSV
-
-```
-npm run generate-count
-```
-
 ### Generate Citizen Data CSV
 
 ```
@@ -59,3 +53,9 @@ npm run generate-json
 ```
 npm run generate-power
 ```
+
+### Generate Citizen Count CSV
+
+```
+npm run generate-count
+```
diff --git a/data-sources/citizens/generate-citizen-count-csv.ts b/data-sources/citizens/generate-citizen-count-csv.ts
index 0289f62a8..a79aa65fc 100644
--- a/data-sources/citizens/generate-citizen-count-csv.ts
+++ b/data-sources/citizens/generate-citizen-count-csv.ts
@@ -2,6 +2,7 @@ const Web3 = require('web3')
 const Passport = require('../abis/Passport.json')
 const csvWriter = require('csv-writer')
 const fs = require('fs')
+const Papa = require('papaparse')
 
 const web3 = new Web3('https://rpc.ankr.com/eth')
 console.info('web3.version:', web3.version)
@@ -25,7 +26,8 @@ async function loadPassportMintsByWeek() {
     header: [
       { id: 'week_end', title: 'week_end' },
       { id: 'total_citizens', title: 'total_citizens' },
-      { id: 'new_citizens', title: 'new_citizens' }
+      { id: 'new_citizens', title: 'new_citizens' },
+      { id: 'total_expired_passports', title: 'total_expired_passports' }
     ]
   })
   let csvRows = []
@@ -57,12 +59,16 @@ async function loadPassportMintsByWeek() {
         break
       }
     }
+
+    const totalExpiredPassports: number = getTotalExpiredPassports(weekEndDate, id)
+    console.info('totalExpiredPassports:', totalExpiredPassports)
     
     // Export to CSV
     const csvRow = {
       week_end: weekEndDate.toISOString().substring(0, 10),
       total_citizens: id,
-      new_citizens: newCitizensCount
+      new_citizens: newCitizensCount,
+      total_expired_passports: totalExpiredPassports
     }
     csvRows.push(csvRow)
 
@@ -83,4 +89,42 @@ async function getTimestamp(id: number): Promise<number> {
   return await PassportContract.methods.timestampOf(id).call()
 }
 
+function getTotalExpiredPassports(weekEndDate: Date, maxPassportID: number): number {
+  console.info('getTotalExpiredPassports')
+
+  const weekEndDateString: string = weekEndDate.toISOString().substring(0, 10)
+
+  let totalExpiredPassports = 0
+  for (let passportID = 0; passportID < maxPassportID; passportID++) {
+    console.info(`weekEndDate: ${weekEndDateString}, passportID: ${passportID}`)
+
+    // Fetch voting power data from the citizen's data CSV
+    const citizenFilePath: string = `output/citizen-${passportID}.csv`
+    console.info('Fetching citizen data:', citizenFilePath)
+    const file: File = fs.readFileSync(citizenFilePath)
+    const csvData = file.toString()
+    // console.info('csvData:\n', csvData)
+    Papa.parse(csvData, {
+      header: true,
+      skipEmptyLines: true,
+      dynamicTyping: true,
+      complete: (result: any) => {
+        // console.info('result:', result)
+        result.data.forEach((row: any, i: number) => {
+          if (row.week_end == weekEndDateString) {
+            console.info(`row.week_end ${row.week_end}, row.voting_power: ${row.voting_power}`)
+            if (row.voting_power < 1.5) {
+              // https://etherscan.io/address/0x279c0b6bfcbba977eaf4ad1b2ffe3c208aa068ac#readContract#F9
+              console.info('Passport ID expired:', passportID)
+              totalExpiredPassports++
+            }
+          }
+        })
+      }
+    })
+  }
+
+  return totalExpiredPassports
+}
+
 export {}
diff --git a/data-sources/citizens/output/citizen-count-per-week.csv b/data-sources/citizens/output/citizen-count-per-week.csv
index dc6018ccf..51ff5895c 100644
--- a/data-sources/citizens/output/citizen-count-per-week.csv
+++ b/data-sources/citizens/output/citizen-count-per-week.csv
@@ -1,61 +1,61 @@
-week_end,total_citizens,new_citizens
-2022-06-05,155,155
-2022-06-12,157,2
-2022-06-19,160,3
-2022-06-26,163,3
-2022-07-03,167,4
-2022-07-10,167,0
-2022-07-17,167,0
-2022-07-24,170,3
-2022-07-31,172,2
-2022-08-07,173,1
-2022-08-14,174,1
-2022-08-21,175,1
-2022-08-28,175,0
-2022-09-04,176,1
-2022-09-11,177,1
-2022-09-18,178,1
-2022-09-25,178,0
-2022-10-02,178,0
-2022-10-09,181,3
-2022-10-16,182,1
-2022-10-23,184,2
-2022-10-30,184,0
-2022-11-06,188,4
-2022-11-13,221,33
-2022-11-20,231,10
-2022-11-27,233,2
-2022-12-04,235,2
-2022-12-11,240,5
-2022-12-18,241,1
-2022-12-25,241,0
-2023-01-01,241,0
-2023-01-08,244,3
-2023-01-15,248,4
-2023-01-22,250,2
-2023-01-29,254,4
-2023-02-05,257,3
-2023-02-12,258,1
-2023-02-19,258,0
-2023-02-26,259,1
-2023-03-05,260,1
-2023-03-12,260,0
-2023-03-19,261,1
-2023-03-26,262,1
-2023-04-02,263,1
-2023-04-09,263,0
-2023-04-16,264,1
-2023-04-23,264,0
-2023-04-30,265,1
-2023-05-07,265,0
-2023-05-14,265,0
-2023-05-21,265,0
-2023-05-28,265,0
-2023-06-04,265,0
-2023-06-11,265,0
-2023-06-18,265,0
-2023-06-25,265,0
-2023-07-02,266,1
-2023-07-09,269,3
-2023-07-16,272,3
-2023-07-23,273,1
+week_end,total_citizens,new_citizens,total_expired_passports
+2022-06-05,155,155,0
+2022-06-12,157,2,0
+2022-06-19,160,3,0
+2022-06-26,163,3,0
+2022-07-03,167,4,0
+2022-07-10,167,0,0
+2022-07-17,167,0,0
+2022-07-24,170,3,0
+2022-07-31,172,2,0
+2022-08-07,173,1,0
+2022-08-14,174,1,0
+2022-08-21,175,1,0
+2022-08-28,175,0,0
+2022-09-04,176,1,0
+2022-09-11,177,1,0
+2022-09-18,178,1,0
+2022-09-25,178,0,0
+2022-10-02,178,0,0
+2022-10-09,181,3,0
+2022-10-16,182,1,0
+2022-10-23,184,2,0
+2022-10-30,184,0,0
+2022-11-06,188,4,0
+2022-11-13,221,33,1
+2022-11-20,231,10,1
+2022-11-27,233,2,1
+2022-12-04,235,2,1
+2022-12-11,240,5,1
+2022-12-18,241,1,1
+2022-12-25,241,0,1
+2023-01-01,241,0,1
+2023-01-08,244,3,1
+2023-01-15,248,4,1
+2023-01-22,250,2,1
+2023-01-29,254,4,2
+2023-02-05,257,3,2
+2023-02-12,258,1,2
+2023-02-19,258,0,2
+2023-02-26,259,1,2
+2023-03-05,260,1,2
+2023-03-12,260,0,2
+2023-03-19,261,1,2
+2023-03-26,262,1,2
+2023-04-02,263,1,2
+2023-04-09,263,0,3
+2023-04-16,264,1,4
+2023-04-23,264,0,7
+2023-04-30,265,1,6
+2023-05-07,265,0,7
+2023-05-14,265,0,7
+2023-05-21,265,0,8
+2023-05-28,265,0,8
+2023-06-04,265,0,8
+2023-06-11,265,0,11
+2023-06-18,265,0,25
+2023-06-25,265,0,28
+2023-07-02,266,1,29
+2023-07-09,269,3,28
+2023-07-16,272,3,30
+2023-07-23,273,1,34
diff --git a/data-sources/citizens/package-lock.json b/data-sources/citizens/package-lock.json
index b53e46085..0f4123029 100644
--- a/data-sources/citizens/package-lock.json
+++ b/data-sources/citizens/package-lock.json
@@ -15,6 +15,7 @@
         "ethereum-block-by-date": "^1.4.6",
         "ethers": "^5.7.2",
         "jsonfile": "^6.1.0",
+        "papaparse": "^5.4.1",
         "typescript": "^4.8.4",
         "web3": "^1.8.1"
       }
@@ -2798,6 +2799,11 @@
         "node": ">=12.20"
       }
     },
+    "node_modules/papaparse": {
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz",
+      "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw=="
+    },
     "node_modules/parse-asn1": {
       "version": "5.1.6",
       "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
@@ -6217,6 +6223,11 @@
       "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
       "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="
     },
+    "papaparse": {
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz",
+      "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw=="
+    },
     "parse-asn1": {
       "version": "5.1.6",
       "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
diff --git a/data-sources/citizens/package.json b/data-sources/citizens/package.json
index d825a0492..67e782f79 100644
--- a/data-sources/citizens/package.json
+++ b/data-sources/citizens/package.json
@@ -28,6 +28,7 @@
     "ethereum-block-by-date": "^1.4.6",
     "ethers": "^5.7.2",
     "jsonfile": "^6.1.0",
+    "papaparse": "^5.4.1",
     "typescript": "^4.8.4",
     "web3": "^1.8.1"
   }

From 1ef5ab33c40890e6b9b5674e75a19c3e37a64eca Mon Sep 17 00:00:00 2001
From: Aahna Ashina <95955389+aahna-ashina@users.noreply.github.com>
Date: Tue, 1 Aug 2023 14:59:58 +0800
Subject: [PATCH 3/3] chore: re-generate count csvs (#98)

---
 data-sources/citizens/output/citizen-count-per-week.csv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/data-sources/citizens/output/citizen-count-per-week.csv b/data-sources/citizens/output/citizen-count-per-week.csv
index 51ff5895c..08e609f30 100644
--- a/data-sources/citizens/output/citizen-count-per-week.csv
+++ b/data-sources/citizens/output/citizen-count-per-week.csv
@@ -59,3 +59,4 @@ week_end,total_citizens,new_citizens,total_expired_passports
 2023-07-09,269,3,28
 2023-07-16,272,3,30
 2023-07-23,273,1,34
+2023-07-30,274,1,44