From 1cf4fca9035e87460e7aab8672d6428a3566888e Mon Sep 17 00:00:00 2001 From: ddragosd Date: Fri, 3 Jun 2016 14:55:46 -0600 Subject: [PATCH] Added support for Throttling and Rate Limiting (#27) * fixed an issue with the AWK script discovering api-gateway-graphite * updated api-gateway-cachemanager to 1.0.1 * added throttling / rate limiting support * updated the restart logic for api-gateway-zmq-adaptor * Replaced non-functioning download URLs for ZMQ (#29) * Fixed building issues with the ZMQ packages downloaded from GitHub (#30) * Replaced non-functioning download URLs for ZMQ * Fixed building issues with the ZMQ packages downloaded from GitHub --- Dockerfile | 72 ++++++++++++++++++ Makefile | 4 +- README.md | 5 ++ .../conf.d/api_gateway_init.conf | 6 ++ api-gateway-config/conf.d/tracking_api.conf | 73 +++++++++++++++++++ .../scripts/lua/api_gateway_init.lua | 26 +++++++ init.sh | 25 +++++++ 7 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 api-gateway-config/conf.d/tracking_api.conf diff --git a/Dockerfile b/Dockerfile index 6ca64ab..851acde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,36 @@ RUN apk update \ perl-test-longstring perl-list-moreutils perl-http-message \ geoip-dev +ENV ZMQ_VERSION 4.0.5 +ENV CZMQ_VERSION 2.2.0 + +# Installing throttling dependencies +RUN echo " ... adding throttling support with ZMQ and CZMQ" \ + && curl -L https://github.com/zeromq/zeromq4-x/archive/v${ZMQ_VERSION}.tar.gz -o /tmp/zeromq.tar.gz \ + && cd /tmp/ \ + && tar -xf /tmp/zeromq.tar.gz \ + && cd /tmp/zeromq*/ \ + && apk add automake autoconf \ + && ./autogen.sh \ + && ./configure --prefix=/usr \ + --sysconfdir=/etc \ + --mandir=/usr/share/man \ + --infodir=/usr/share/info \ + && make && make install \ + && curl -L https://github.com/zeromq/czmq/archive/v${CZMQ_VERSION}.tar.gz -o /tmp/czmq.tar.gz \ + && cd /tmp/ \ + && tar -xf /tmp/czmq.tar.gz \ + && cd /tmp/czmq*/ \ + && ./autogen.sh \ + && ./configure --prefix=/usr \ + --sysconfdir=/etc \ + --mandir=/usr/share/man \ + --infodir=/usr/share/info \ + && make && make install \ + && apk del automake autoconf \ + && rm -rf /tmp/zeromq* && rm -rf /tmp/czmq* \ + && rm -rf /var/cache/apk/* + # openresty build ENV OPENRESTY_VERSION=1.9.7.3 \ NAXSI_VERSION=0.53-2 \ @@ -279,6 +309,48 @@ RUN echo " ... installing api-gateway-async-logger ..." \ && rm -rf /var/cache/apk/* \ && rm -rf /tmp/api-gateway +ENV ZMQ_ADAPTOR_VERSION 0.1.1 +RUN echo " ... installing api-gateway-zmq-adaptor" \ + && curl -L https://github.com/adobe-apiplatform/api-gateway-zmq-adaptor/archive/${ZMQ_ADAPTOR_VERSION}.tar.gz -o /tmp/api-gateway-zmq-adaptor-${ZMQ_ADAPTOR_VERSION} \ + && apk update \ + && apk add check-dev g++ gcc \ + && cd /tmp/ \ + && tar -xf /tmp/api-gateway-zmq-adaptor-${ZMQ_ADAPTOR_VERSION} \ + && cd /tmp/api-gateway-zmq-adaptor-* \ + && make test \ + && PREFIX=/usr/local/sbin make install \ + && rm -rf /tmp/api-gateway-zmq-adaptor-* \ + && apk del check-dev g++ gcc \ + && rm -rf /var/cache/apk/* + +ENV ZMQ_LOGGER_VERSION 1.0.0 +RUN echo " ... installing api-gateway-zmq-logger ..." \ + && mkdir -p /tmp/api-gateway \ + && curl -L https://github.com/adobe-apiplatform/api-gateway-zmq-logger/archive/${ZMQ_LOGGER_VERSION}.tar.gz -o /tmp/api-gateway/api-gateway-zmq-logger-${ZMQ_LOGGER_VERSION}.tar.gz \ + && tar -xf /tmp/api-gateway/api-gateway-zmq-logger-${ZMQ_LOGGER_VERSION}.tar.gz -C /tmp/api-gateway/ \ + && cd /tmp/api-gateway/api-gateway-zmq-logger-${ZMQ_LOGGER_VERSION} \ + && cp -r /usr/local/test-nginx-${TEST_NGINX_VERSION}/* ./test/resources/test-nginx/ \ + && make test \ + && make install \ + LUA_LIB_DIR=/usr/local/api-gateway/lualib \ + INSTALL=/usr/local/api-gateway/bin/resty-install \ + && rm -rf /tmp/api-gateway + +ENV REQUEST_TRACKING_VERSION 1.0.1 +RUN echo " ... installing api-gateway-request-tracking ..." \ + && mkdir -p /tmp/api-gateway \ + && curl -L https://github.com/adobe-apiplatform/api-gateway-request-tracking/archive/${REQUEST_TRACKING_VERSION}.tar.gz -o /tmp/api-gateway/api-gateway-request-tracking-${REQUEST_TRACKING_VERSION}.tar.gz \ + && tar -xf /tmp/api-gateway/api-gateway-request-tracking-${REQUEST_TRACKING_VERSION}.tar.gz -C /tmp/api-gateway/ \ + && cd /tmp/api-gateway/api-gateway-request-tracking-${REQUEST_TRACKING_VERSION} \ + && cp -r /usr/local/test-nginx-${TEST_NGINX_VERSION}/* ./test/resources/test-nginx/ \ + && apk update && apk add redis \ + && REDIS_SERVER=/usr/bin/redis-server make test \ + && make install \ + LUA_LIB_DIR=/usr/local/api-gateway/lualib \ + INSTALL=/usr/local/api-gateway/bin/resty-install \ + && apk del redis \ + && rm -rf /tmp/api-gateway + RUN \ curl -L -k -s -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \ && apk update \ diff --git a/Makefile b/Makefile index 4a7bcaf..7c16900 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ docker-ssh: .PHONY: docker-run docker-run: - docker run --rm --name="apigateway" -p 80:80 adobeapiplatform/apigateway:latest ${DOCKER_ARGS} + docker run --rm --name="apigateway" -p 80:80 -p 5000:5000 adobeapiplatform/apigateway:latest ${DOCKER_ARGS} .PHONY: docker-debug docker-debug: @@ -19,7 +19,7 @@ docker-debug: rm -rf ${HOME}/tmp/apiplatform/apigateway/api-gateway-config cp -r `pwd`/api-gateway-config ${HOME}/tmp/apiplatform/apigateway/ docker run --name="apigateway" \ - -p 80:80 \ + -p 80:80 -p 5000:5000 \ -e "LOG_LEVEL=info" -e "DEBUG=true" \ -v ${HOME}/tmp/apiplatform/apigateway/api-gateway-config/:/etc/api-gateway \ adobeapiplatform/apigateway:latest ${DOCKER_ARGS} diff --git a/README.md b/README.md index ce81eea..10e6640 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,9 @@ What's inside | [Test Nginx](https://github.com/openresty/test-nginx) | [0.24](https://github.com/openresty/test-nginx/releases/tag/v0.24) | Useful for executing integration tests from the container.
It's installed in `/usr/local/test-nginx-0.24/`.
It's also used during Docker build to execute `make test` on lua modules. | | [PCRE](https://sourceforge.net/projects/pcre/) | [8.37](https://sourceforge.net/projects/pcre/files/pcre/8.37/) | Enables PCRE JIT support | | [NAXSI](https://github.com/nbs-system/naxsi) | [0.53-2](https://github.com/nbs-system/naxsi/releases/tag/0.53-2) | NAXSI is an open-source, high performance, low rules maintenance WAF for NGINX | +| [ZeroMQ](http://download.zeromq.org/) | [4.0.5](http://zeromq.org/area:download) | ZeroMQ | +| [CZMQ](http://download.zeromq.org/) | [2.2.0](http://czmq.zeromq.org/page:get-the-software) | CZMQ - High-level C Binding for ZeroMQ | + ##### Modules for API Management and Logging @@ -118,6 +121,8 @@ What's inside | [api-gateway-aws](https://github.com/adobe-apiplatform/api-gateway-aws) | [1.7.0](https://github.com/adobe-apiplatform/api-gateway-aws/releases/tag/1.7.0) | AWS SDK for Nginx in Lua | | [api-gateway-request-validation](https://github.com/adobe-apiplatform/api-gateway-request-validation) | [1.1.1](https://github.com/adobe-apiplatform/api-gateway-request-validation/releases/tag/1.1.1) | API Request Validation framework | | [api-gateway-async-logger](https://github.com/adobe-apiplatform/api-gateway-async-logger) | [1.0.1](https://github.com/adobe-apiplatform/api-gateway-async-logger/releases/tag/1.0.1) | Performant async logger | +| [api-gateway-zmq-logger](https://github.com/adobe-apiplatform/api-gateway-zmq-logger) | [1.0.0](https://github.com/adobe-apiplatform/api-gateway-zmq-logger/releases/tag/1.0.0) | Lua logger for ZMQ with FFI and CZMQ | +| [api-gateway-request-tracking](https://github.com/adobe-apiplatform/api-gateway-request-tracking) | [1.0.1](https://github.com/adobe-apiplatform/api-gateway-request-tracking/releases/tag/1.0.1) | Usage and Tracking Handler for the API Gateway | ##### Other Lua Modules diff --git a/api-gateway-config/conf.d/api_gateway_init.conf b/api-gateway-config/conf.d/api_gateway_init.conf index 1c206e5..a190517 100644 --- a/api-gateway-config/conf.d/api_gateway_init.conf +++ b/api-gateway-config/conf.d/api_gateway_init.conf @@ -55,6 +55,12 @@ lua_shared_dict cachedOauthTokens 50m; # caches OAuth tokens lua_shared_dict cachedUserProfiles 50m; # caches user profiles lua_shared_dict healthcheck_redis 1m; # used by lua health_check for redis cache +# dictionaries used for storing the throttling / rate limiting rules +lua_shared_dict tracking_rules_dict 5m; +lua_shared_dict blocking_rules_dict 5m; +lua_shared_dict delaying_rules_dict 5m; +lua_shared_dict rewriting_rules_dict 5m; + # metrics lua_shared_dict stats_counters 50m; lua_shared_dict stats_timers 50m; diff --git a/api-gateway-config/conf.d/tracking_api.conf b/api-gateway-config/conf.d/tracking_api.conf new file mode 100644 index 0000000..7fe450d --- /dev/null +++ b/api-gateway-config/conf.d/tracking_api.conf @@ -0,0 +1,73 @@ +# VHost exposing an API for Gateway Tracking Service ( GTS ) to control the Gateway +# This VHost should listen on a PORT that is never exposed externally +# This VHost should use a server_name matching the hostname + +server { + listen 5000 default_server; + + # + # Sample message to start tracking requests that match with the given format + # { + # "id": 222, + # "domain" : "cc-eco;cceco-consumer;*", + # "format": "$publisher_org_name;$consumer_org_name;$api_key", + # "expire_at_utc": 1408065588, + # "action" : "track", + # "data" : 0 + # } + # + location /tracking { + access_log /var/log/api-gateway/access.log; + limit_except POST { + deny all; + } + content_by_lua 'ngx.apiGateway.tracking.POST_HANDLER()'; + } + + # + # location returning all TRACKing rules + # + location /tracking/track { + access_log /var/log/api-gateway/access.log; + limit_except GET { + deny all; + } + default_type 'application/json'; + content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("TRACK")'; + } + + # + # location retuning all BLOCKing rules + # + location /tracking/block { + access_log /var/log/api-gateway/access.log; + limit_except GET { + deny all; + } + default_type 'application/json'; + content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("BLOCK")'; + } + + # + # location returning all DELAYing rules + # + location /tracking/delay { + access_log /var/log/api-gateway/access.log; + limit_except GET { + deny all; + } + default_type 'application/json'; + content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("DELAY")'; + } + + # + # location returning all REWRITE rules + # + location /tracking/rewrite { + limit_except GET { + deny all; + } + default_type 'application/json'; + content_by_lua 'ngx.apiGateway.tracking.GET_HANDLER("REWRITE")'; + } +} \ No newline at end of file diff --git a/api-gateway-config/scripts/lua/api_gateway_init.lua b/api-gateway-config/scripts/lua/api_gateway_init.lua index db5a42c..5893851 100644 --- a/api-gateway-config/scripts/lua/api_gateway_init.lua +++ b/api-gateway-config/scripts/lua/api_gateway_init.lua @@ -44,15 +44,41 @@ local function loadrequire(module) return require(module) end +--- Initializes the `zmqLogger` used by `trackingRulesLogger.lua` from api-gateway-request-tracking module +-- @param parentObject +-- +local function initZMQLogger(parentObject) + ngx.log(ngx.DEBUG, "Initializing ZMQLogger on property [zmqLogger]") + -- when the ZmqModule is not present the script does not break + local ZmqLogger = loadrequire("api-gateway.zmq.ZeroMQLogger") + + if (ZmqLogger == nil) then + return + end + + local zmq_publish_address = "ipc:///tmp/nginx_queue_listen" + ngx.log(ngx.INFO, "Starting new ZmqLogger on pid [", tostring(ngx.worker.pid()), "] on address [", zmq_publish_address, "]") + local zmqLogger = ZmqLogger:new() + zmqLogger:connect(ZmqLogger.SOCKET_TYPE.ZMQ_PUB, zmq_publish_address) + + parentObject.zmqLogger = zmqLogger +end + local function initValidationFactory(parentObject) parentObject.validation = require "api-gateway.validation.factory" end +local function initTrackingFactory(parentObject) + parentObject.tracking = require "api-gateway.tracking.factory" +end + local function initMetricsFactory(parentObject) parentObject.metrics = require "metrics.factory" end initValidationFactory(_M) +initZMQLogger(_M) +initTrackingFactory(_M) initMetricsFactory(_M) -- TODO: test health-check with the new version of Openresty -- initRedisHealthCheck() diff --git a/init.sh b/init.sh index d8c6bcc..2c7967f 100755 --- a/init.sh +++ b/init.sh @@ -30,6 +30,31 @@ sleep_duration=${MARATHON_POLL_INTERVAL:-5} remote_config=${REMOTE_CONFIG} remote_config_sync_interval=${REMOTE_CONFIG_SYNC_INTERVAL:-10s} +function start_zmq_adaptor() +{ + echo "Starting ZeroMQ adaptor ..." + zmq_port=$(echo $ZMQ_PUBLISHER_PORT) + # use -d flag to start API Gateway ZMQ adaptor in debug mode to print all messages sent by the GW + zmq_adaptor_cmd="api-gateway-zmq-adaptor" + if [[ -n "${zmq_port}" ]]; then + echo "... ZMQ will publish messages on:" ${zmq_port} + zmq_adaptor_cmd="${zmq_adaptor_cmd} -p ${zmq_port}" + fi + if [ "${debug_mode}" == "true" ]; then + echo " ... in DEBUG mode " + zmq_adaptor_cmd="${zmq_adaptor_cmd} -d" + fi + + $zmq_adaptor_cmd >> /dev/stderr & + sleep 3s + # allow interprocess communication by allowing api-gateway processes to write to the socket + chown nginx-api-gateway:nginx-api-gateway /tmp/nginx_queue_listen + chown nginx-api-gateway:nginx-api-gateway /tmp/nginx_queue_push +} +# keep the zmq adaptor running using a simple loop +while true; do zmq_pid=$(ps aux | grep api-gateway-zmq-adaptor | grep -v grep) || ( echo "Restarting api-gateway-zmq-adaptor" && start_zmq_adaptor ); sleep 60; done & + + echo "Starting api-gateway ..." if [ "${debug_mode}" == "true" ]; then echo " ... in DEBUG mode "