Merge branch 'release/3.0.0-milestone2'

Colm3na · Apr 22, 2020 · 93a6383 · 93a6383
2 parents 60fe28f + a84fbd5
commit 93a6383
Show file tree

Hide file tree

Showing 15 changed files with 966 additions and 194 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 node_modules
+.env
diff --git a/README.md b/README.md
@@ -6,13 +6,21 @@ New improved backend for https://polkastats.io!
 
 ### Table of Contents
 
-   * [Installation Instructions](#installation-instructions)
-   * [Usage Instructions](#usage-instructions)
-   * [List of current containers](#list-of-current-containers)
-   * [Updating containers](#updating-containers)
-   * [Crawler](#crawler)
-   * [Phragmen](#phragmen)
-
+- [PolkaStats Backend v3](#polkastats-backend-v3)
+    - [Table of Contents](#table-of-contents)
+  - [Installation Instructions](#installation-instructions)
+  - [Usage Instructions](#usage-instructions)
+  - [List of current containers](#list-of-current-containers)
+  - [Updating containers](#updating-containers)
+  - [Crawler](#crawler)
+  - [Phragmen](#phragmen)
+  - [Hasura demo](#hasura-demo)
+    - [Query example. Static](#query-example-static)
+    - [Subscription example. Dynamic](#subscription-example-dynamic)
+  - [Configuration](#configuration)
+    - [Substrate](#substrate)
+    - [Database](#database)
+    - [Crawlers](#crawlers)
 
 <!--te-->
 
@@ -27,10 +35,13 @@ npm install
 ## Usage Instructions
 
 To launch all docker containers at once:
+
 ```
 npm run docker
 ```
+
 To run them separately:
+
 ```
 npm run docker:<container-name>
 ```
@@ -41,7 +52,7 @@ npm run docker:<container-name>
 - postgres
 - graphql-engine
 - crawler
-- phragmen  (temporarily disabled)
+- phragmen (temporarily disabled)
 
 ## Updating containers
 
@@ -104,6 +115,7 @@ In order to check it and see its power you could start a new subscription or jus
 ### Query example. Static
 
 - Block query example:
+
 ```
 query {
   block  {
@@ -122,6 +134,7 @@ query {
 ```
 
 - Rewards query example:
+
 ```
 query {
   rewards {
@@ -134,6 +147,7 @@ query {
 ```
 
 - Validator by number of nominators example:
+
 ```
 query {
   validator_num_nominators {
@@ -145,6 +159,7 @@ query {
 ```
 
 - Account query example:
+
 ```
 query {
   account {
@@ -158,6 +173,7 @@ query {
 ### Subscription example. Dynamic
 
 - Block subscription example:
+
 ```
 subscription {
   block {
@@ -170,6 +186,7 @@ subscription {
 ```
 
 - Validator active subscription example:
+
 ```
 subscription MySubscription {
 	validator_active {
@@ -183,6 +200,7 @@ subscription MySubscription {
 ```
 
 - Account subscription example:
+
 ```
 subscription MySubscription {
   account {
@@ -191,3 +209,46 @@ subscription MySubscription {
   }
 }
 ```
+
+## Configuration
+
+You can customize your configuration through the following environment variables:
+
+### Substrate
+
+| Env name        | Description           | Default Value            |
+| --------------- | --------------------- | ------------------------ |
+| WS_PROVIDER_URL | Substrate node to use | ws://substrate-node:9944 |
+
+### Database
+
+| Env name          | Description              | Default Value |
+| ----------------- | ------------------------ | ------------- |
+| POSTGRES_USER     | PostgreSQL username      | polkastats    |
+| POSTGRES_PASSWORD | PostgreSQL user password | polkastats    |
+| POSTGRES_HOST     | PostgreSQL host          | postgres      |
+| POSTGRES_PORT     | PostgreSQL port          | 5432          |
+| POSTGRES_DATABASE | PostgreSQL database name | polkastats    |
+
+### Crawlers
+
+| Env name                                | Description                                 | Default Value                                  |
+| --------------------------------------- | ------------------------------------------- | ---------------------------------------------- |
+| CRAWLER_BLOCK_LISTENER_POLLING_TIME_MS  | Polling time for block listener (ms)        | 60000                                          |
+| CRAWLER_ACTIVE_ACCOUNTS_POLLING_TIME_MS | Polling time for accounts crawler (ms)      | 600000                                         |
+| CRAWLER_PHRAGMEN_POLLING_TIME_MS        | Polling time for phragmen executions (ms)   | 300000                                         |
+| CRAWLER_PHRAGMEN_OUTPUT_DIR             | Directory to store the phgramen JSON result | /tmp/phragmen                                  |
+| CRAWLER_PHRAGMEN_BINARY_PATH            | Path to the phragmen executable             | /usr/app/polkastats-backend-v3/offline-phragme |
+
+You can also disable specific crawlers with the following environment variables:
+
+- `CRAWLER_SYSTEM_DISABLE`
+- `CRAWLER_BLOCK_LISTENER_DISABLE`
+- `CRAWLER_BLOCK_HARVESTER_DISABLE`
+- `CRAWLER_STAKING_DISABLE`
+- `CRAWLER_ACTIVE_ACCOUNTS_DISABLE`
+- `CRAWLER_CHAIN_DISABLE`
+- `CRAWLER_REWARDS_DISABLE`
+- `CRAWLER_PHRAGMEN_DISABLE`
+
+For instance, if you want to disable phgramen crawler just set `CRAWLER_PHRAGMEN_DISABLE=true`.
diff --git a/backend.config.js b/backend.config.js
@@ -1,56 +1,77 @@
-const DEFAULT_WS_PROVIDER_URL = 'ws://substrate-node:9944';
+require('dotenv').config();
 
 module.exports = {
-  wsProviderUrl: process.env.WS_PROVIDER_URL || DEFAULT_WS_PROVIDER_URL,
+  wsProviderUrl: process.env.WS_PROVIDER_URL || 'ws://substrate-node:9944',
 
   postgresConnParams: {
-    user: 'polkastats',
-    host: 'postgres',
-    database: 'polkastats',
-    password: 'polkastats',
-    port: 5432,
+    user: process.env.POSTGRES_USER || 'polkastats',
+    host: process.env.POSTGRES_HOST || 'postgres',
+    database: process.env.POSTGRES_DATABASE || 'polkastats',
+    password: process.env.POSTGRES_PASSWORD || 'polkastats',
+    port: process.env.POSTGRES_PORT || 5432,
   },
 
   crawlers: [
+
+    {
+      enabled: !process.env.CRAWLER_SYSTEM_DISABLE,
+      module: require('./lib/crawlers/system'),
+    },
+
     {
-      enabled: true,
-      module: require('./lib/crawlers/blockListener.js'),
+      enabled: !process.env.CRAWLER_BLOCK_LISTENER_DISABLE,
+      module: require('./lib/crawlers/blockListener'),
     },
 
     {
-      enabled: true,
-      module: require('./lib/crawlers/blockHarvester.js'),
+      enabled: !process.env.CRAWLER_BLOCK_HARVESTER_DISABLE,
+      module: require('./lib/crawlers/blockHarvester'),
       config: {
-        pollingTime: 1 * 60 * 1000,
+        pollingTime:
+          parseInt(process.env.CRAWLER_BLOCK_LISTENER_POLLING_TIME_MS) ||
+          1 * 60 * 1000,
       },
     },
 
     {
-      enabled: true,
-      module: require('./lib/crawlers/staking.js'),
+      enabled: !process.env.CRAWLER_STAKING_DISABLE,
+      module: require('./lib/crawlers/staking'),
     },
 
     {
-      enabled: true,
-      module: require('./lib/crawlers/activeAccounts.js'),
+      enabled: !process.env.CRAWLER_ACTIVE_ACCOUNTS_DISABLE,
+      module: require('./lib/crawlers/activeAccounts'),
       config: {
-        pollingTime: 10 * 60 * 1000,
+        pollingTime:
+          parseInt(process.env.CRAWLER_ACTIVE_ACCOUNTS_POLLING_TIME_MS) ||
+          60 * 60 * 1000,
       },
     },
 
     {
-      enabled: true,
-      module: require('./lib/crawlers/rewards.js'),
+      enabled: !process.env.CRAWLER_CHAIN_DISABLE,
+      module: require('./lib/crawlers/chain'),
+    },
+
+    {
+      enabled: !process.env.CRAWLER_REWARDS_DISABLE,
+      module: require('./lib/crawlers/rewards'),
     },
 
     {
-      enabled: false,
-      module: require('./lib/crawlers/phragmen.js'),
+      enabled: !process.env.CRAWLER_PHRAGMEN_DISABLE,
+      module: require('./lib/crawlers/phragmen'),
       config: {
-        wsProviderUrl: process.env.WS_PROVIDER_URL || DEFAULT_WS_PROVIDER_URL,
-        pollingTime: 5 * 60 * 1000,
-        phragmenOutputDir: '/tmp/phragmen',
-        offlinePhragmenPath: '/usr/app/polkastats-backend-v3/offline-phragmen',
+        wsProviderUrl:
+          process.env.WS_PROVIDER_URL || 'ws://substrate-node:9944',
+        pollingTime:
+          parseInt(process.env.CRAWLER_PHRAGMEN_POLLING_TIME_MS) ||
+          5 * 60 * 1000,
+        phragmenOutputDir:
+          process.env.CRAWLER_PHRAGMEN_OUTPUT_DIR || '/tmp/phragmen',
+        offlinePhragmenPath:
+          process.env.CRAWLER_PHRAGMEN_BINARY_PATH ||
+          '/usr/app/polkastats-backend-v3/offline-phragmen',
       },
     },
   ],