-

.gitignore

@@ -11,3 +11,5 @@ node_modules/
 /lib/
 /.env
 /npmvs/server/src/_archive
+/nogit/
+secrets.js

package.json

@@ -1,10 +1,10 @@
 {
   "name": "npmvs-server",
   "version": "1.0.0",
-  "description": "The machinery that collects, analyzes and stores data for NPMvs",
+  "description": "The machinery that collects, analyzes, stores and serves data for NPMvs",
   "main": "index.js",
   "author": "Vianney Stroebel",
-  "license": "ISC",
+  "private": true,
   "scripts": {
     "start": "node src/server.js",
     "test": "jest"

src/client-code-for-inspiration/sources/source_github-repo.js

@@ -2,7 +2,7 @@ import http from "../../util/http";
 import store from '../store';
 
 const endpointUrl = 'https://api.github.com/repos/';
-const apiToken = '389dfd76a9f1d65d99cf38f8f7d12ed0e97f3066';
+const apiToken = ;
 
 export default async (packName) => {
   const repoUrl = await store.detect(`rawdata.${packName}.repository_url`);

src/grabber/grab_github_contributors.js

@@ -3,6 +3,7 @@ const format= require('date-fns/format');
 const pMap = require('p-map');
 const {q, q1, insert1} = require('../db');
 const sql = require('./sql_tpl');
+const secrets = require('../secrets');
 const http = require('../http');
 const {flatten} = require('lodash');
 const {getTimestamp, throttleSleeper} = require('../util/vibl-util');
@@ -12,7 +13,7 @@ const concurrency = 5;
 // const rateLimit = 10; // Requests per second.
 // const minRequestDuration = 1000 / rateLimit; // In ms.
 // const throttleSleep = throttleSleeper(minRequestDuration);
-
+const apiToken = secrets.apiTokens.github[0];
 
 const source = {
   id: 4,

src/grabber/grab_libio_main.js

@@ -1,10 +1,10 @@
 const {q} = require('../db');
 const sql = require('./sql_tpl');
-const config = require('../config');
+const secrets = require('../secrets');
 const getData = require('./getPackageDataFromSource');
 const {sleep, throttleSleeper} = require('../util/vibl-util');
 
-const batchSize = 60;//TODO: set back to 60 !
+const batchSize = 60;
 // Don't use viblApiToken because I don't want to be identified, let alone blacklisted,
 // with my real GitHub account.
 // const viblApiToken = "653f8ad38c7c9c60ac58a88f8e9a0876";
@@ -20,7 +20,7 @@ const urlBuilder = (apiToken) => (packName) =>
 const source = {
   id: 1,
   name: 'libio_main',
-  getUrl: urlBuilder(config.apiTokens.libio[0]), // Use the first token by default (any would do).
+  getUrl: urlBuilder(secrets.apiTokens.libio[0]), // Use the first token by default (any would do).
 };
 const getAccountConfig = (apiToken, i) => {
   const accountOffsetDelay =  i * 6000;
@@ -43,7 +43,7 @@ const downloadWithAccount = async ({accountOffsetDelay, apiToken, throttleSleep}
 };
 const getMissingPackages = async () => {
   console.log('Grabbing data from Libraries.io...');
-  const accounts = config.apiTokens.libio.map(getAccountConfig);
+  const accounts = secrets.apiTokens.libio.map(getAccountConfig);
   try {
     await Promise.all(accounts.map(downloadWithAccount));
   }

src/util/vibl-fp.js

@@ -3,7 +3,6 @@ function _i(required) {
 }
 const _ = _i(require('lodash'));
 const R = _i(require('ramda'));
-const pMapOrig = _i(require('p-map'));
 const deepEql = _i(require("deep-eql"));
 
 /* eslint-disable no-unused-vars*/ 
@@ -334,8 +333,6 @@ const budge = (fn) => curry3((...args) => {
   args.unshift(args.pop());
   return fn(...args);
 });
-// Map over promises concurrently (By Sindre Sorhus)
-const pMap = flip(pMapOrig);
 
 const keep = curry2((list, arr) => {
   const res = [];
@@ -570,7 +567,7 @@ const viblPure = {
   mergeAllTables, mergeAllTablesNotBlank, mergeTables, mergeTablesNotBlank,
   notBlank, notEmpty, notMatch, nthRoot,
   mapToArray, overlaps, orNull,
-  pathFromDotpath,  dotPath: pathFromDotpath, percent, pipeD, pipeLog, pMap, prefixLine, preIntersperse, putFirst,
+  pathFromDotpath,  dotPath: pathFromDotpath, percent, pipeD, pipeLog, prefixLine, preIntersperse, putFirst,
   random, rangeMap, rangeStep, reduceFirst, reduceFirstP, reduceIndexed, reduceP,
   reduceSteps, reduceTemplate, reIndex, removed, removeShortest, rest, reverseDifference, round,
   splitLinesTrim, splitPipe, splitProperties, store, switchValue,