Implement graceful shutdown.

Gracefully stop all clients and services when application is about to
stop.

Adds async-exit-hook module that will trigger registered hook on
relevant system signals or process.exit events.

app.js

@@ -12,6 +12,7 @@ import connectDb, { waitDb } from './controllers/connection';
 import * as session from './controllers/_session';
 import CoreServer from './controllers/serviceConnector';
 import { handleSocketConnection, registerSocketRequestHendler } from './app/request';
+import exitHook from 'async-exit-hook';
 
 import { photosReady } from './controllers/photo';
 import { ready as mailReady } from './controllers/mail';
@@ -307,6 +308,11 @@ export async function configure(startStamp) {
         scheduleMemInfo(startStamp - Date.now());
     });
 
+    exitHook(cb => {
+        logger.info('HTTP server is shutting down');
+        httpServer.close(cb);
+    });
+
     // Once db is connected, start some periodic jobs.
     // Do it in app.js, not in controllers, to prevent running these jobs on other instances (sitemap, uploader, downloader etc.)
     waitDb.then(() => {

controllers/connection.js

@@ -3,6 +3,7 @@ import util from 'util';
 import log4js from 'log4js';
 import { ApplicationError } from '../app/errors';
 import constantsError from '../app/errors/constants';
+import exitHook from 'async-exit-hook';
 
 const modelPromises = [];
 let connectionPromises;
@@ -18,6 +19,7 @@ export const waitDb = new Promise((resolve, reject) => {
     getDBResolve = resolve;
     getDBReject = reject;
 });
+
 export const registerModel = modelPromise => {
     if (db) {
         modelPromise(db);
@@ -36,6 +38,7 @@ function init({ mongo, redis, logger = log4js.getLogger('app') }) {
     if (mongo) {
         const mongoose = require('mongoose');
         const { uri, poolSize = 1 } = mongo;
+        let connErrorLogLevel = 'error';
 
         // Set native Promise as mongoose promise provider
         mongoose.Promise = Promise;
@@ -62,6 +65,13 @@ function init({ mongo, redis, logger = log4js.getLogger('app') }) {
                 },
             });
 
+            exitHook(cb => {
+                // Connection related events are no longer regarded as errors.
+                connErrorLogLevel = 'info';
+                logger.info('MongoDB client is shutting down');
+                db.close(cb);
+            });
+
             async function openHandler() {
                 const adminDb = db.db.admin(); // Use the admin database for some operation
 
@@ -80,10 +90,10 @@ function init({ mongo, redis, logger = log4js.getLogger('app') }) {
                     logger.error(`MongoDB connection error to ${uri}`, err);
                 });
                 db.on('disconnected', () => {
-                    logger.error('MongoDB disconnected!');
+                    logger.log(connErrorLogLevel, 'MongoDB disconnected!');
                 });
                 db.on('close', () => {
-                    logger.error('MongoDB connection closed and onClose executed on all of this connections models!');
+                    logger.log(connErrorLogLevel, 'MongoDB connection closed and onClose executed on all of this connections models!');
                 });
                 db.on('reconnected', () => {
                     logger.info('MongoDB reconnected at ' + uri);
@@ -165,6 +175,11 @@ function init({ mongo, redis, logger = log4js.getLogger('app') }) {
                         `Time to stop trying ${(maxReconnectTime - params.total_retry_time) / 1000}s`
                     );
                 });
+
+            exitHook(cb => {
+                logger.info('Redis client is shutting down');
+                dbRedis.quit(cb);
+            });
         }));
     }
 

controllers/serviceConnector.js

@@ -1,5 +1,6 @@
 import net from 'net';
 import { handleServiceRequest } from '../app/request';
+import exitHook from 'async-exit-hook';
 
 class ClientSocket {
     constructor(server, socket, logger) {
@@ -111,6 +112,11 @@ export default class Server {
 
             this.logger.info(`${this.name} client connected. Total clients: ${this.clientSockets.length}`);
         });
+
+        exitHook(cb => {
+            this.logger.info(`${this.name} server is shutting down`);
+            this.server.close(cb);
+        });
     }
 
     listen() {

docker-compose.yml

@@ -32,7 +32,7 @@ services:
     volumes:
       - .:/code
       - store:/store
-    command: npm run app
+    command: run app
 
   notifier:
     << : *app-image
@@ -42,7 +42,7 @@ services:
       - NOTIFIER=true
     volumes:
       - .:/code
-    command: npm run notifier
+    command: run notifier
 
   uploader:
     << : *app-image
@@ -53,7 +53,7 @@ services:
     volumes:
       - .:/code
       - store:/store
-    command: npm run uploader
+    command: run uploader
 
   downloader:
     << : *app-image
@@ -64,7 +64,7 @@ services:
     volumes:
       - .:/code
       - store:/store:ro
-    command: npm run downloader
+    command: run downloader
 
   sitemap:
     << : *app-image
@@ -73,7 +73,7 @@ services:
     volumes:
       - .:/code
       - sitemap:/sitemap
-    command: npm run sitemap
+    command: run sitemap
 
   mailcatcher:
     image: sj26/mailcatcher:latest

downloader.js

@@ -12,6 +12,7 @@ import { parse as parseCookie } from 'cookie';
 import contentDisposition from 'content-disposition';
 import CorePlug from './controllers/serviceConnectorPlug';
 import { ApplicationError, AuthorizationError, BadParamsError, NotFoundError } from './app/errors';
+import exitHook from 'async-exit-hook';
 
 import connectDb, { dbRedis } from './controllers/connection';
 import { Download } from './models/Download';
@@ -294,7 +295,7 @@ export async function configure(startStamp) {
     core.connect();
 
     // Start server and do manual manual url router, express is not needed
-    http
+    const server = http
         .createServer(function handleRequest(req, res) {
             if (protectedServePattern.test(req.url)) {
                 return protectedHandler(req, res);
@@ -315,4 +316,9 @@ export async function configure(startStamp) {
 
             scheduleMemInfo(startStamp - Date.now());
         });
+
+    exitHook(cb => {
+        logger.info('Downloader server is shutting down');
+        server.close(cb);
+    });
 }

package.json

@@ -41,6 +41,7 @@
         "@mapbox/geojsonhint": "3.0.0",
         "@turf/intersect": "6.1.3",
         "@turf/turf": "5.1.6",
+        "async-exit-hook": "2.0.1",
         "aws-sdk": "2.610.0",
         "basic-auth-connect": "1.0.0",
         "bcrypt": "5.0.0",

uploader.js

@@ -9,6 +9,7 @@ import log4js from 'log4js';
 import config from './config';
 import formidable from 'formidable';
 import Utils from './commons/Utils';
+import exitHook from 'async-exit-hook';
 
 export function configure(startStamp) {
     const {
@@ -286,10 +287,15 @@ export function configure(startStamp) {
         }
     };
 
-    http.createServer(handleRequest).listen(listenport, '0.0.0.0', () => {
+    const server = http.createServer(handleRequest).listen(listenport, '0.0.0.0', () => {
         logger.info(
             `Uploader server started up in ${(Date.now() - startStamp) / 1000}s`,
             `and listening [*:${listenport}]\n`
         );
     });
+
+    exitHook(cb => {
+        logger.info('Uploader server is shutting down');
+        server.close(cb);
+    });
 }