Added support for Throttling and Rate Limiting (#27)

* fixed an issue with the AWK script discovering api-gateway-graphite * updated api-gateway-cachemanager to 1.0.1 * added throttling / rate limiting support * updated the restart logic for api-gateway-zmq-adaptor * Replaced non-functioning download URLs for ZMQ (#29) * Fixed building issues with the ZMQ packages downloaded from GitHub (#30) * Replaced non-functioning download URLs for ZMQ * Fixed building issues with the ZMQ packages downloaded from GitHub
adobe-apiplatform · Jun 3, 2016 · 1cf4fca · 1cf4fca
1 parent b265072
commit 1cf4fca
Show file tree

Hide file tree

Showing 7 changed files with 209 additions and 2 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -13,6 +13,36 @@ RUN apk update \
     perl-test-longstring perl-list-moreutils perl-http-message \
     geoip-dev
 
+ENV ZMQ_VERSION 4.0.5
+ENV CZMQ_VERSION 2.2.0
+
+# Installing throttling dependencies
+RUN echo " ... adding throttling support with ZMQ and CZMQ" \
+         && curl -L https://github.com/zeromq/zeromq4-x/archive/v${ZMQ_VERSION}.tar.gz -o /tmp/zeromq.tar.gz \
+         && cd /tmp/ \
+         && tar -xf /tmp/zeromq.tar.gz \
+         && cd /tmp/zeromq*/ \
+         && apk add automake autoconf \
+         && ./autogen.sh \
+         && ./configure --prefix=/usr \
+                        --sysconfdir=/etc \
+                        --mandir=/usr/share/man \
+                        --infodir=/usr/share/info \
+         && make && make install \
+         && curl -L https://github.com/zeromq/czmq/archive/v${CZMQ_VERSION}.tar.gz -o /tmp/czmq.tar.gz \
+         && cd /tmp/ \
+         && tar -xf /tmp/czmq.tar.gz \
+         && cd /tmp/czmq*/ \
+         && ./autogen.sh \
+         && ./configure --prefix=/usr \
+                        --sysconfdir=/etc \
+                        --mandir=/usr/share/man \
+                        --infodir=/usr/share/info \
+         && make && make install \
+         && apk del automake autoconf \
+         && rm -rf /tmp/zeromq* && rm -rf /tmp/czmq* \
+         && rm -rf /var/cache/apk/*
+
 # openresty build
 ENV OPENRESTY_VERSION=1.9.7.3 \
     NAXSI_VERSION=0.53-2 \
@@ -279,6 +309,48 @@ RUN echo " ... installing api-gateway-async-logger ..." \
     && rm -rf /var/cache/apk/* \
     && rm -rf /tmp/api-gateway
 
+ENV ZMQ_ADAPTOR_VERSION 0.1.1
+RUN echo " ... installing api-gateway-zmq-adaptor" \
+         && curl -L https://github.com/adobe-apiplatform/api-gateway-zmq-adaptor/archive/${ZMQ_ADAPTOR_VERSION}.tar.gz -o /tmp/api-gateway-zmq-adaptor-${ZMQ_ADAPTOR_VERSION} \
+         && apk update \
+         && apk add check-dev g++ gcc \
+         && cd /tmp/ \
+         && tar -xf /tmp/api-gateway-zmq-adaptor-${ZMQ_ADAPTOR_VERSION} \
+         && cd /tmp/api-gateway-zmq-adaptor-* \
+         && make test \
+         && PREFIX=/usr/local/sbin make install \
+         && rm -rf /tmp/api-gateway-zmq-adaptor-* \
+         && apk del check-dev g++ gcc \
+         && rm -rf /var/cache/apk/*
+
+ENV ZMQ_LOGGER_VERSION 1.0.0
+RUN echo " ... installing api-gateway-zmq-logger ..." \
+        && mkdir -p /tmp/api-gateway \
+        && curl -L https://github.com/adobe-apiplatform/api-gateway-zmq-logger/archive/${ZMQ_LOGGER_VERSION}.tar.gz -o /tmp/api-gateway/api-gateway-zmq-logger-${ZMQ_LOGGER_VERSION}.tar.gz \
+        && tar -xf /tmp/api-gateway/api-gateway-zmq-logger-${ZMQ_LOGGER_VERSION}.tar.gz -C /tmp/api-gateway/ \
+        && cd /tmp/api-gateway/api-gateway-zmq-logger-${ZMQ_LOGGER_VERSION} \
+        && cp -r /usr/local/test-nginx-${TEST_NGINX_VERSION}/* ./test/resources/test-nginx/ \
+        && make test \
+        && make install \
+             LUA_LIB_DIR=/usr/local/api-gateway/lualib \
+             INSTALL=/usr/local/api-gateway/bin/resty-install \
+        && rm -rf /tmp/api-gateway
+
+ENV REQUEST_TRACKING_VERSION 1.0.1
+RUN echo " ... installing api-gateway-request-tracking ..." \
+        && mkdir -p /tmp/api-gateway \
+        && curl -L https://github.com/adobe-apiplatform/api-gateway-request-tracking/archive/${REQUEST_TRACKING_VERSION}.tar.gz -o /tmp/api-gateway/api-gateway-request-tracking-${REQUEST_TRACKING_VERSION}.tar.gz \
+        && tar -xf /tmp/api-gateway/api-gateway-request-tracking-${REQUEST_TRACKING_VERSION}.tar.gz -C /tmp/api-gateway/ \
+        && cd /tmp/api-gateway/api-gateway-request-tracking-${REQUEST_TRACKING_VERSION} \
+        && cp -r /usr/local/test-nginx-${TEST_NGINX_VERSION}/* ./test/resources/test-nginx/ \
+        && apk update && apk add redis \
+        && REDIS_SERVER=/usr/bin/redis-server make test \
+        && make install \
+             LUA_LIB_DIR=/usr/local/api-gateway/lualib \
+             INSTALL=/usr/local/api-gateway/bin/resty-install \
+        && apk del redis \
+        && rm -rf /tmp/api-gateway
+
 RUN \
     curl -L -k -s -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \
     && apk update \

diff --git a/Makefile b/Makefile
@@ -10,7 +10,7 @@ docker-ssh:
 
 .PHONY: docker-run
 docker-run:
-	docker run --rm --name="apigateway" -p 80:80 adobeapiplatform/apigateway:latest ${DOCKER_ARGS}
+	docker run --rm --name="apigateway" -p 80:80 -p 5000:5000 adobeapiplatform/apigateway:latest ${DOCKER_ARGS}
 
 .PHONY: docker-debug
 docker-debug:
@@ -19,7 +19,7 @@ docker-debug:
 	rm -rf ${HOME}/tmp/apiplatform/apigateway/api-gateway-config
 	cp -r `pwd`/api-gateway-config ${HOME}/tmp/apiplatform/apigateway/
 	docker run --name="apigateway" \
-			-p 80:80 \
+			-p 80:80 -p 5000:5000 \
 			-e "LOG_LEVEL=info" -e "DEBUG=true" \
 			-v ${HOME}/tmp/apiplatform/apigateway/api-gateway-config/:/etc/api-gateway \
 			adobeapiplatform/apigateway:latest ${DOCKER_ARGS}

diff --git a/README.md b/README.md
@@ -107,6 +107,9 @@ What's inside
 | [Test Nginx](https://github.com/openresty/test-nginx) | [0.24](https://github.com/openresty/test-nginx/releases/tag/v0.24) | Useful for executing integration tests from the container. <br/> It's installed in `/usr/local/test-nginx-0.24/`. <br/> It's also used during Docker build to execute `make test` on lua modules.  |
 | [PCRE](https://sourceforge.net/projects/pcre/) | [8.37](https://sourceforge.net/projects/pcre/files/pcre/8.37/) | Enables PCRE JIT support | 
 | [NAXSI](https://github.com/nbs-system/naxsi) | [0.53-2](https://github.com/nbs-system/naxsi/releases/tag/0.53-2) |  NAXSI is an open-source, high performance, low rules maintenance WAF for NGINX |
+| [ZeroMQ](http://download.zeromq.org/) | [4.0.5](http://zeromq.org/area:download) |  ZeroMQ |
+| [CZMQ](http://download.zeromq.org/) | [2.2.0](http://czmq.zeromq.org/page:get-the-software) |  CZMQ - High-level C Binding for ZeroMQ |
+
 
 ##### Modules for API Management and Logging
 
@@ -118,6 +121,8 @@ What's inside
 | [api-gateway-aws](https://github.com/adobe-apiplatform/api-gateway-aws) | [1.7.0](https://github.com/adobe-apiplatform/api-gateway-aws/releases/tag/1.7.0) | AWS SDK for Nginx in Lua |
 | [api-gateway-request-validation](https://github.com/adobe-apiplatform/api-gateway-request-validation) | [1.1.1](https://github.com/adobe-apiplatform/api-gateway-request-validation/releases/tag/1.1.1) | API Request Validation framework  |
 | [api-gateway-async-logger](https://github.com/adobe-apiplatform/api-gateway-async-logger) | [1.0.1](https://github.com/adobe-apiplatform/api-gateway-async-logger/releases/tag/1.0.1) | Performant async logger |
+| [api-gateway-zmq-logger](https://github.com/adobe-apiplatform/api-gateway-zmq-logger) | [1.0.0](https://github.com/adobe-apiplatform/api-gateway-zmq-logger/releases/tag/1.0.0) | Lua logger for ZMQ with FFI and CZMQ |
+| [api-gateway-request-tracking](https://github.com/adobe-apiplatform/api-gateway-request-tracking) | [1.0.1](https://github.com/adobe-apiplatform/api-gateway-request-tracking/releases/tag/1.0.1) | Usage and Tracking Handler for the API Gateway |
 
 ##### Other Lua Modules
 

diff --git a/api-gateway-config/conf.d/api_gateway_init.conf b/api-gateway-config/conf.d/api_gateway_init.conf
@@ -55,6 +55,12 @@ lua_shared_dict cachedOauthTokens 50m; # caches OAuth tokens
 lua_shared_dict cachedUserProfiles 50m; # caches user profiles
 lua_shared_dict healthcheck_redis 1m; # used by lua health_check for redis cache
 
+# dictionaries used for storing the throttling / rate limiting rules
+lua_shared_dict tracking_rules_dict 5m;
+lua_shared_dict blocking_rules_dict 5m;
+lua_shared_dict delaying_rules_dict 5m;
+lua_shared_dict rewriting_rules_dict 5m;
+
 # metrics
 lua_shared_dict stats_counters 50m;
 lua_shared_dict stats_timers 50m;

diff --git a/api-gateway-config/conf.d/tracking_api.conf b/api-gateway-config/conf.d/tracking_api.conf
@@ -0,0 +1,73 @@
+# VHost exposing an API for Gateway Tracking Service ( GTS ) to control the Gateway
+# This VHost should listen on a PORT that is never exposed externally
+# This VHost should use a server_name matching the hostname
+
+server {
+  listen 5000 default_server;
+
+  #
+  # Sample message to start tracking requests that match with the given format
+  #        {
+  #          "id": 222,
+  #          "domain" : "cc-eco;cceco-consumer;*",
+  #          "format": "$publisher_org_name;$consumer_org_name;$api_key",
+  #          "expire_at_utc": 1408065588,
+  #          "action" : "track",
+  #          "data" : 0
+  #        }
+  #
+  location /tracking {
+    access_log /var/log/api-gateway/access.log;
+    limit_except POST {
+      deny  all;
+    }
+    content_by_lua 'ngx.apiGateway.tracking.POST_HANDLER()';
+  }
+
+  #
+  # location returning all TRACKing rules
+  #
+  location /tracking/track {
+    access_log /var/log/api-gateway/access.log;
+    limit_except GET {
+      deny  all;
+    }
+    default_type 'application/json';
+    content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("TRACK")';
+  }
+
+  #
+  # location retuning all BLOCKing rules
+  #
+  location /tracking/block {
+    access_log /var/log/api-gateway/access.log;
+    limit_except GET {
+      deny  all;
+    }
+    default_type 'application/json';
+    content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("BLOCK")';
+  }
+
+  #
+  # location returning all DELAYing rules
+  #
+  location /tracking/delay {
+    access_log /var/log/api-gateway/access.log;
+    limit_except GET {
+      deny  all;
+    }
+    default_type 'application/json';
+    content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("DELAY")';
+  }
+
+  #
+  # location returning all REWRITE rules
+  #
+  location /tracking/rewrite {
+    limit_except GET {
+      deny  all;
+    }
+    default_type 'application/json';
+    content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("REWRITE")';
+  }
+}
diff --git a/api-gateway-config/scripts/lua/api_gateway_init.lua b/api-gateway-config/scripts/lua/api_gateway_init.lua
@@ -44,15 +44,41 @@ local function loadrequire(module)
     return require(module)
 end
 
+--- Initializes the `zmqLogger` used by `trackingRulesLogger.lua` from api-gateway-request-tracking module
+-- @param parentObject
+--
+local function initZMQLogger(parentObject)
+    ngx.log(ngx.DEBUG, "Initializing ZMQLogger on property [zmqLogger]")
+    -- when the ZmqModule is not present the script does not break
+    local ZmqLogger = loadrequire("api-gateway.zmq.ZeroMQLogger")
+
+    if (ZmqLogger == nil) then
+        return
+    end
+
+    local zmq_publish_address = "ipc:///tmp/nginx_queue_listen"
+    ngx.log(ngx.INFO, "Starting new ZmqLogger on pid [", tostring(ngx.worker.pid()), "] on address [", zmq_publish_address, "]")
+    local zmqLogger = ZmqLogger:new()
+    zmqLogger:connect(ZmqLogger.SOCKET_TYPE.ZMQ_PUB, zmq_publish_address)
+
+    parentObject.zmqLogger = zmqLogger
+end
+
 local function initValidationFactory(parentObject)
     parentObject.validation = require "api-gateway.validation.factory"
 end
 
+local function initTrackingFactory(parentObject)
+    parentObject.tracking = require "api-gateway.tracking.factory"
+end
+
 local function initMetricsFactory(parentObject)
     parentObject.metrics = require "metrics.factory"
 end
 
 initValidationFactory(_M)
+initZMQLogger(_M)
+initTrackingFactory(_M)
 initMetricsFactory(_M)
 -- TODO: test health-check with the new version of Openresty
 -- initRedisHealthCheck()

diff --git a/init.sh b/init.sh
@@ -30,6 +30,31 @@ sleep_duration=${MARATHON_POLL_INTERVAL:-5}
 remote_config=${REMOTE_CONFIG}
 remote_config_sync_interval=${REMOTE_CONFIG_SYNC_INTERVAL:-10s}
 
+function start_zmq_adaptor()
+{
+    echo "Starting ZeroMQ adaptor ..."
+    zmq_port=$(echo $ZMQ_PUBLISHER_PORT)
+    # use -d flag to start API Gateway ZMQ adaptor in debug mode to print all messages sent by the GW
+    zmq_adaptor_cmd="api-gateway-zmq-adaptor"
+    if [[ -n "${zmq_port}" ]]; then
+        echo "... ZMQ will publish messages on:" ${zmq_port}
+        zmq_adaptor_cmd="${zmq_adaptor_cmd} -p ${zmq_port}"
+    fi
+    if [ "${debug_mode}" == "true" ]; then
+        echo "   ...  in DEBUG mode "
+        zmq_adaptor_cmd="${zmq_adaptor_cmd} -d"
+    fi
+
+    $zmq_adaptor_cmd >> /dev/stderr &
+    sleep 3s
+    # allow interprocess communication by allowing api-gateway processes to write to the socket
+    chown nginx-api-gateway:nginx-api-gateway /tmp/nginx_queue_listen
+    chown nginx-api-gateway:nginx-api-gateway /tmp/nginx_queue_push
+}
+# keep the zmq adaptor running using a simple loop
+while true; do zmq_pid=$(ps aux | grep api-gateway-zmq-adaptor | grep -v grep) || ( echo "Restarting api-gateway-zmq-adaptor" && start_zmq_adaptor ); sleep 60; done &
+
+
 echo "Starting api-gateway ..."
 if [ "${debug_mode}" == "true" ]; then
     echo "   ...  in DEBUG mode "