index.js

/*
 * index.js: Top-level include for npm-pkg-top.
 *
 * (C) 2013 Charlie Robbins.
 *
 */

var fs = require('fs'),
    path = require('path'),
    async = require('async'),
    GitHubApi = require('github'),
    githubUrl = require('github-url-from-git'),
    ProgressBar = require('progress'),
    request = require('request');

//
// The urls for the type of queries we are doing.
//
var urls = {
  binary: 'https://skimdb.npmjs.com/registry/_design/app/_view/needBuild',
  all:    'https://skimdb.npmjs.com/registry/_all_docs'
};

//
// ### function pkgTop (options, callback)
// Returns the top packages by npm stars, github stars, and depended upon.
// e.g.
//
//     [
//       { npm: 0, git: 10, dep: 5, name: 'some-pkg' },
//       (...)
//     ]
//
module.exports = function (options, callback) {
  //
  // ### function log (level, msg)
  // Logs the `msg` at the `level` provided.
  //
  function log(level, msg) {
    if (!options.logger) { return }
    options.logger[level](msg);
  }

  //
  // ### function parseNpm (body)
  // Parses the reponse from the first npm view query.
  //
  function parseNpm (body) {
    return body.rows.map(function (row) {
      var pkg   = row.doc,
          url   = pkg.repository && pkg.repository.url,
          users = pkg.users;

      return {
        name:      pkg.name,
        githubUrl: (url && githubUrl(url))              || null,
        npm:       (users && Object.keys(users).length) || 0
      };
    });
  }

  //
  // ### function saveQuery(json, next)
  // Saves the JSON query to `options['save-query']
  //
  function saveQuery(json, next) {
    log('info', 'Saving query to: ' + options['save-query']);
    fs.writeFile(options['save-query'], JSON.stringify(json, null, 2), 'utf8', function (err) {
      if (err) { log('error', 'Error saving query: ' + err.message) }
      next(null, parseNpm(json));
    });
  }

  //
  // ### function loadQuery(next)
  // Loads the JSON query to `options['load-query']
  //
  function loadQuery(next) {
    log('info', 'Loading query from: ' + options['load-query']);
    fs.readFile(options['load-query'], 'utf8', function (err, data) {
      if (err) {
        log('error', 'Failed to load ' + options['load-query'] + ': ' + err.message);
        return next(err);
      }

      try { data = JSON.parse(data) }
      catch (ex) { return next(ex) }
      next(null, parseNpm(data));
    });
  }


  //
  // ### function getGithubStars(github, stars, next)
  // Gets the npm and github stars for the
  // specified `stars` info.
  //
  function getGithubStars(github, stars, next) {
    if (!stars.githubUrl) {
      stars.git = -1;
      return next(null, stars);
    }

    var parts = stars.githubUrl
      .split('/')
      .slice(-2);

    github.repos.get({
      user: parts[0],
      repo: parts[1]
    }, function (err, repo) {
      stars.git = repo && repo.watchers
        ? repo.watchers
        : 0;

      log('info', 'git: [' + [stars.git, stars.name, parts.join('/')].join(', ') + ']');
      next(null, stars);
    });
  }

  //
  // ### function getDependents (stars, next)
  // Counts all of the dependendents for the specified
  // stars object
  //
  function getDependents(stars, next) {
    request({
      url: 'https://skimdb.npmjs.com/registry/_design/app/_view/dependedUpon',
      json: true,
      qs: {
        startkey: JSON.stringify([stars.name]),
        endkey:   JSON.stringify([stars.name, 'zzzzz'])
      }
    }, function (err, res, body) {
      stars.dep = !err && body && body.rows.length
        ? body.rows[0].value
        : 0;

      log('info', 'dep [' + [stars.dep, stars.name].join(', ') + ']');
      next();
    });
  }

  //
  // ### function sortStars (key)
  // Returns a function that sorts the stars
  // object by the given key.
  //
  function sortStars(key) {
    return function (lval, rval) {
      //
      // Get the value of the specified key
      //
      var lstar = lval[key],
          rstar = rval[key];

      if (lstar === rstar) { return 0 }
      return lstar < rstar ? 1 : -1;
    };
  }

  async.waterfall([
    //
    // 1. Get all binary packages
    //
    function listBinary(next) {
      if (options['load-query']) {
        return loadQuery(next);
      }

      var req = request({
        url: urls[options.type],
        qs: { include_docs: true },
        json: true
      }, function (err, res, body) {
        if (err || !body || !body.rows || !body.rows.length) {
          return next(err || new Error('Bad data returned from npm registry.'));
        }

        return options['save-query']
          ? saveQuery(body, next)
          : next(null, parseNpm(body));
      });

      req.on('response', function (res) {
        var bar = new ProgressBar('  Querying npm [:bar] :percent :etas', {
          width: 20,
          clear: true,
          complete: '=',
          incomplete: ' ',
          total: res.headers['content-length']
            ? parseInt(res.headers['content-length'], 10)
            : 100 * 1024 * 1024
        });

        res.on('end', function () {
          bar.complete = true;
          bar.terminate();
          console.log();
        });

        res.on('data', function (chunk) {
          bar.tick(chunk.length);
        });
      });

      req.end();
    },
    //
    // 2. Get the github and npm stars for
    //    each of the binary packages
    //
    function getAllStars(allStars, next) {
      if (~options.skip.indexOf('git')) {
        return next(null, allStars.map(function (stars) {
          stars.git = -1;
          return stars;
        }));
      }

      //
      // Create a github client.
      //
      var github = new GitHubApi({
        // required
        version: "3.0.0",
        // optional
        timeout: 5000
      });

      //
      // Authenticate if credentials are provided
      //
      if (options.username && options.password) {
        github.authenticate({
          type: 'basic',
          username: options.username,
          password: options.password
        });
      }

      async.mapLimit(allStars, 10, getGithubStars.bind(null, github), next);
    },
    //
    // 3. Get the npm dependents for
    //    each of the binary packages
    //
    function getAllDependents(allStars, next) {
      if (~options.skip.indexOf('dep')) {
        return next(null, allStars.map(function (stars) {
          stars.dep = -1;
          return stars;
        }));
      }

      async.forEachLimit(allStars, 10, getDependents, function (err) {
        return err ? next(err) : next(null, allStars);
      });
    },
    //
    // 4. Compute the top binary packages
    //
    function getTop(stars, next) {
      stars = stars
        .filter(Boolean)
        .sort(sortStars(options.sortBy));

      next(null, stars);
    }
  ], callback);
};