feat(data): collect data locally (#28)

.github/workflows/update-pages.yml

@@ -61,6 +61,20 @@ jobs:
           # copy dependencies
           cp -f ./node_modules/plotly.js/dist/plotly.min.js ./gh-pages/plotly.js
 
+      - name: Update
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_API_TOKEN }}
+          CROWDIN_TOKEN: ${{ secrets.CROWDIN_TOKEN }}
+          DISCORD_INVITE: ${{ secrets.DISCORD_INVITE }}
+          FACEBOOK_GROUP_ID: ${{ secrets.FACEBOOK_GROUP_ID }}
+          FACEBOOK_PAGE_ID: ${{ secrets.FACEBOOK_PAGE_ID }}
+          FACEBOOK_TOKEN: ${{ secrets.FACEBOOK_ACCESS_TOKEN }}
+          GITHUB_REPOSITORY_OWNER: ${{ secrets.GH_ORG_NAME }}
+          GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN || secrets.GITHUB_TOKEN }}
+          READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }}
+        run: |
+          python -u ./src/update.py
+
       - name: Convert notebook
         env:
           GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN || secrets.GITHUB_TOKEN }}

notebook/dashboard.ipynb

@@ -78,19 +78,20 @@
     "# Imports\n",
     "\n",
     "# standard imports\n",
+    "import json\n",
     "import os\n",
-    "import numpy as np\n",
+    "import time\n",
     "\n",
     "# lib imports\n",
     "from dotenv import load_dotenv\n",
     "from github import Github, UnknownObjectException\n",
     "from IPython.display import HTML, display\n",
     "from itables import init_notebook_mode, show\n",
+    "import numpy as np\n",
     "import pandas as pd\n",
     "import plotly.express as px\n",
     "import plotly.graph_objects as go\n",
-    "import plotly.io as pio\n",
-    "import requests"
+    "import plotly.io as pio"
    ]
   },
   {
@@ -119,15 +120,14 @@
     "\n",
     "# constants\n",
     "text_template = '%{text}'\n",
-    "uno_base_url = 'https://app.lizardbyte.dev/uno'\n",
+    "data_dir = os.path.join(os.path.dirname(os.getcwd()), 'gh-pages')\n",
     "\n",
     "# all readthedocs projects\n",
     "# readthedocs data\n",
-    "readthedocs_data_url = f'{uno_base_url}/readthedocs/projects.json'\n",
-    "readthedocs_response = requests.get(readthedocs_data_url)\n",
-    "if not readthedocs_response.ok:\n",
-    "    raise LookupError(\"Failed to fetch data from uno\")\n",
-    "readthedocs_data = readthedocs_response.json()"
+    "readthedocs_path = os.path.join(data_dir, 'readthedocs', 'projects.json')\n",
+    "\n",
+    "with open(readthedocs_path, 'r') as f:\n",
+    "    readthedocs_data = json.load(f)"
    ]
   },
   {
@@ -149,15 +149,13 @@
     "    open_issues = [issue for issue in open_issues if issue.pull_request is None]\n",
     "\n",
     "    # coverage data\n",
-    "    coverage_url = f'{uno_base_url}/codecov/{repo.name}.json'\n",
-    "    coverage_response = requests.get(coverage_url)\n",
     "    coverage = 0\n",
-    "    if coverage_response.status_code == 200:\n",
-    "        try:\n",
-    "            coverage_data = coverage_response.json()\n",
-    "            coverage = coverage_data['totals']['coverage']\n",
-    "        except Exception:\n",
-    "            pass\n",
+    "    try:\n",
+    "        with open(os.path.join(data_dir, 'codecov', f'{repo.name}.json')) as f:\n",
+    "            coverage_data = json.load(f)\n",
+    "        coverage = coverage_data['totals']['coverage']\n",
+    "    except Exception:\n",
+    "        pass\n",
     "\n",
     "    # readthedocs data\n",
     "    readthedocs_project = None\n",
@@ -222,6 +220,7 @@
    "outputs": [],
    "source": [
     "# Initial Results\n",
+    "print(f'Last Updated: {time.strftime(\"%Y-%m-%d %H:%M:%S\", time.gmtime())} UTC')\n",
     "print(f'Total Repositories: {len(repo_data)}')\n",
     "print(f'Archived Repositories: {df[\"archived\"].sum()}')\n",
     "print(f'Forked Repositories: {df[\"fork\"].sum()}')\n",

requirements.txt

@@ -1,8 +1,14 @@
+cloudscraper==1.2.71
+crowdin-api-client==1.17.0
 ipython==8.27.0
 itables==2.1.4
 notebook==7.2.2
 pandas==2.2.2
+pillow==10.4.0
 plotly==5.24.0
 pygithub==2.4.0
 python-dotenv==1.0.1
 requests==2.32.3
+svgwrite==1.4.3
+tqdm==4.66.5
+unhandled_exit==1.0.0