From 91abbe0410cf607716355d9004400396045bfcce Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Thu, 10 Oct 2024 11:44:55 +0300 Subject: [PATCH] MDBF-804 - BB NGINX configuration in GH CI About This Patch: - Using templates (present since NGINX 1.19) to populate conf.d. - Templates allow for the use of environment variables defined in the .env files, enabling us to distinguish between PROD and DEV environments, particularly for the server name and certificate paths. - A proxy_params file is required according to the PROD configuration. - Mounting /etc/letsencrypt/live for SSL certificates. The base path is the same in both environments. - NGINX_ARTIFACTS_SSL_PATH variable is necessary because, in DEV, the same certificate is used for both CI and BB. - Attaching net_back to the NGINX container to facilitate communication with master-web via DNS. - Removing net_front from master-web; communication will be handled through NGINX. - NGINX access/error logs are written to the Docker-Compose relative path logs/nginx, which is needed for Zabbix collection. TODO Before Migration to PROD: - Address all FIXME comments. - Cross-reference proxy pass - helper_files directory name on hz-bbm2. - location /cloud-init ? TODO Before Deployment in DEV: - Disable the HAProxy service. --- docker-compose/.env | 2 + docker-compose/.env.dev | 2 + docker-compose/docker-compose.yaml | 44 +++++++- docker-compose/generate-config.py | 16 ++- docker-compose/nginx/conf.d/ci.conf | 26 ----- docker-compose/nginx/proxy_params | 4 + .../nginx/templates/bb.conf.template | 101 ++++++++++++++++++ .../nginx/templates/ci.conf.template | 77 +++++++++++++ 8 files changed, 236 insertions(+), 36 deletions(-) delete mode 100644 docker-compose/nginx/conf.d/ci.conf create mode 100644 docker-compose/nginx/proxy_params create mode 100644 docker-compose/nginx/templates/bb.conf.template create mode 100644 docker-compose/nginx/templates/ci.conf.template diff --git a/docker-compose/.env b/docker-compose/.env index d062c6f7..1f9edea5 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -6,6 +6,8 @@ MASTER_PACKAGES_DIR="/mnt/autofs/master_packages" GALERA_PACKAGES_DIR="/mnt/autofs/galera_packages" ARTIFACTS_URL="https://ci.mariadb.org" NGINX_ARTIFACTS_VHOST="ci.mariadb.org" +NGINX_ARTIFACTS_SSL_PATH="ci.mariadb.org" +NGINX_BUILDBOT_VHOST="buildbot.mariadb.org" ENVIRON="PROD" BRANCH="main" MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel8-docker" diff --git a/docker-compose/.env.dev b/docker-compose/.env.dev index 759a7384..d186d05f 100644 --- a/docker-compose/.env.dev +++ b/docker-compose/.env.dev @@ -7,6 +7,8 @@ MASTER_PACKAGES_DIR="/mnt/autofs/master_dev_packages" GALERA_PACKAGES_DIR="/mnt/autofs/galera_dev_packages" ARTIFACTS_URL="https://ci.dev.mariadb.org" NGINX_ARTIFACTS_VHOST="ci.dev.mariadb.org" +NGINX_ARTIFACTS_SSL_PATH="buildbot.dev.mariadb.org" +NGINX_BUILDBOT_VHOST="buildbot.dev.mariadb.org" ENVIRON="DEV" BRANCH="dev" MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel9-docker" diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index a458a4bb..c59f9a67 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -46,14 +46,23 @@ services: hostname: nginx volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./nginx/conf.d/:/etc/nginx/conf.d/:ro + - ./nginx/proxy_params:/etc/nginx/proxy_params:ro + - ./nginx/templates/:/etc/nginx/templates/:ro - /srv/buildbot/packages:/srv/buildbot/packages:ro - /srv/buildbot/galera_packages:/srv/buildbot/galera_packages:ro - /srv/buildbot/helper_files:/srv/buildbot/helper_files:ro + - /etc/letsencrypt/live:/etc/nginx/ssl + - ./logs/nginx:/var/log/nginx ports: - - "127.0.0.1:8080:80" + - "443:443" + - "80:80" + environment: + - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST + - NGINX_ARTIFACTS_SSL_PATH networks: net_front: + net_back: logging: driver: journald options: @@ -77,7 +86,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=8010 - TITLE - TITLE_URL @@ -88,10 +99,7 @@ services: entrypoint: - /srv/buildbot/master/docker-compose/start-bbm-web.sh networks: - net_front: net_back: - ports: - - "127.0.0.1:8010:8010" depends_on: - mariadb - crossbar @@ -114,7 +122,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=9996 - TITLE - TITLE_URL @@ -154,7 +164,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=9997 - TITLE - TITLE_URL @@ -193,7 +205,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=9998 - TITLE - TITLE_URL @@ -232,7 +246,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=9999 - TITLE - TITLE_URL @@ -271,7 +287,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10000 - TITLE - TITLE_URL @@ -310,7 +328,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10001 - TITLE - TITLE_URL @@ -349,7 +369,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10002 - TITLE - TITLE_URL @@ -388,7 +410,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10003 - TITLE - TITLE_URL @@ -427,7 +451,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10004 - TITLE - TITLE_URL @@ -466,7 +492,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10005 - TITLE - TITLE_URL @@ -505,7 +533,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10006 - TITLE - TITLE_URL @@ -544,7 +574,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10007 - TITLE - TITLE_URL @@ -583,7 +615,9 @@ services: - MASTER_NONLATENT_DOCKERLIBRARY_WORKER - MASTER_PACKAGES_DIR - MQ_ROUTER_URL + - NGINX_ARTIFACTS_SSL_PATH - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST - PORT=10008 - TITLE - TITLE_URL diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index da8ef4b2..a27cbe24 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -72,14 +72,23 @@ hostname: nginx volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./nginx/conf.d/:/etc/nginx/conf.d/:ro + - ./nginx/proxy_params:/etc/nginx/proxy_params:ro + - ./nginx/templates/:/etc/nginx/templates/:ro - /srv/buildbot/packages:/srv/buildbot/packages:ro - /srv/buildbot/galera_packages:/srv/buildbot/galera_packages:ro - /srv/buildbot/helper_files:/srv/buildbot/helper_files:ro + - /etc/letsencrypt/live:/etc/nginx/ssl + - ./logs/nginx:/var/log/nginx ports: - - "127.0.0.1:8080:80" + - "443:443" + - "80:80" + environment: + - NGINX_ARTIFACTS_VHOST + - NGINX_BUILDBOT_VHOST + - NGINX_ARTIFACTS_SSL_PATH networks: net_front: + net_back: logging: driver: journald options: @@ -96,10 +105,7 @@ entrypoint: - /srv/buildbot/master/docker-compose/start-bbm-web.sh networks: - net_front: net_back: - ports: - - "127.0.0.1:8010:8010" depends_on: - mariadb - crossbar diff --git a/docker-compose/nginx/conf.d/ci.conf b/docker-compose/nginx/conf.d/ci.conf deleted file mode 100644 index af748878..00000000 --- a/docker-compose/nginx/conf.d/ci.conf +++ /dev/null @@ -1,26 +0,0 @@ -server { - listen 80; - server_name $NGINX_ARTIFACTS_VHOST; - - root /srv/buildbot/packages/; - autoindex on; - location /helper_files { - alias /srv/buildbot/helper_files; - } - location /galera { - alias /srv/buildbot/galera_packages; - } - - # show mysql error logs directly in browser - # example https://ci.mariadb.org/16646/logs/aarch64-ubuntu-2010/mysqld.2.err.4 - # see https://jira.mariadb.org/browse/MDBF-250 - # location ~ \.err\.\d+$ { - location ~ mysqld\.[0-9]+\.err\.[0-9]+$ { - add_header Content-Type text/plain; - } - - # show some extensions directly in browser - types { - text/plain repo sources txt; - } -} diff --git a/docker-compose/nginx/proxy_params b/docker-compose/nginx/proxy_params new file mode 100644 index 00000000..11c0f2c4 --- /dev/null +++ b/docker-compose/nginx/proxy_params @@ -0,0 +1,4 @@ +proxy_set_header Host $http_host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; \ No newline at end of file diff --git a/docker-compose/nginx/templates/bb.conf.template b/docker-compose/nginx/templates/bb.conf.template new file mode 100644 index 00000000..c5111c9f --- /dev/null +++ b/docker-compose/nginx/templates/bb.conf.template @@ -0,0 +1,101 @@ +server { + listen 80; + listen [::]:80; + server_name ${NGINX_BUILDBOT_VHOST}; + return 301 https://$server_name$request_uri; +} + + +# Default rate limited zone, with 30 requests per minute +limit_req_zone $request_uri zone=default:10m rate=30r/m; +client_max_body_size 10M; + +server { + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + + server_name ${NGINX_BUILDBOT_VHOST}; + + # logging + access_log /var/log/nginx/buildbot.access.log; + error_log /var/log/nginx/buildbot.error.log error; + + # SSL configuration + # ssl on; Deprecated in newer versions of NGINX (yields nginx: [emerg] unknown directive "ssl ) + ssl_certificate /etc/nginx/ssl/${NGINX_BUILDBOT_VHOST}/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/nginx/ssl/${NGINX_BUILDBOT_VHOST}/privkey.pem; # managed by Certbot + # put a one day session timeout for websockets to stay longer + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_protocols TLSv1.1 TLSv1.2; + + # Force https - Enable HSTS + add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;" always; + # # Disable embedding the site + add_header X-Frame-Options "SAMEORIGIN"; + # # Enable XSS protection + add_header X-XSS-Protection "1;mode=block"; + + # Enable gziped format + #gzip on; already on in main conf + # Set level of compression + gzip_comp_level 3; + # Set mime types + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Host $host; + + # Use default zone for rate limiting, allow burst of 10 requests with + # no delay + limit_req zone=default burst=10 nodelay; + + location / { + # Reverse proxy settings + include proxy_params; + proxy_pass http://master-web:8010; + } + + # disable logging for wsgi_dashboards/styles.css since it's generated + # somewhere and mess with fail2ban //TEMP find the root cause! + location ~ /wsgi_dashboards/styles.css* { + access_log off; + } + location = /favicon.ico { + access_log off; + } + location = /robots.txt { + access_log off; + } + + # Server sent event (sse) settings + location /sse { + proxy_buffering off; + proxy_pass http://master-web:8010/sse; + } + + # Websocket settings + location /ws { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://master-web:8010/ws; + proxy_read_timeout 6000s; + } + + + #FIXME: CrossReference not in DEV. ProxyPass for PROD? + # Cross-reference + # location /cr/static { + # alias /srv/cr/static; + # } + + # location /cr/ { + # include proxy_params; + # proxy_pass http://hz-bbw5:8080; + # } +} diff --git a/docker-compose/nginx/templates/ci.conf.template b/docker-compose/nginx/templates/ci.conf.template new file mode 100644 index 00000000..618e7a1d --- /dev/null +++ b/docker-compose/nginx/templates/ci.conf.template @@ -0,0 +1,77 @@ +server { + listen 80; + listen [::]:80; + server_name ${NGINX_ARTIFACTS_VHOST}; + return 301 https://$server_name$request_uri; +} + +# Build artifacts location +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ${NGINX_ARTIFACTS_VHOST}; + + root /srv/buildbot/packages/; + location /helper_files { + alias /srv/buildbot/helper_files; #FIXME - for consistency, on hz-bbm2 let's rename it to helper_files instead of mariadb-shared-packages (current PROD) + } + location /galera { + alias /srv/buildbot/galera_packages; + } + #FIX ME - Still needed? Not present in DEV. + # location /cloud-init { + # alias /srv/buildbot/cloud-init; + # autoindex off; + # } + location = /favicon.ico { + access_log off; + } + + # show mysql error logs directly in browser + # example https://ci.mariadb.org/16646/logs/aarch64-ubuntu-2010/mysqld.2.err.4 + # see https://jira.mariadb.org/browse/MDBF-250 + # location ~ \.err\.\d+$ { + location ~ mysqld\.[0-9]+\.err\.[0-9]+$ { + add_header Content-Type text/plain; + } + + # show some extensions directly in browser + types { + text/plain repo sources txt; + } + + autoindex on; + + # logging + access_log /var/log/nginx/ci.access.log; + error_log /var/log/nginx/ci.error.log error; + + # SSL configuration + # ssl on; + ssl_certificate /etc/nginx/ssl/${NGINX_ARTIFACTS_SSL_PATH}/fullchain.pem; + ssl_certificate_key /etc/nginx/ssl/${NGINX_ARTIFACTS_SSL_PATH}/privkey.pem; + ssl_protocols TLSv1.1 TLSv1.2; + + # Force https - Enable HSTS + add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;" always; + # Disable embedding the site + add_header X-Frame-Options "SAMEORIGIN"; + # Enable XSS protection + add_header X-XSS-Protection "1;mode=block"; + max_ranges 1; + msie_padding off; + + # Enable gziped format + #gzip on; already on in main conf + # Set level of compression + gzip_comp_level 3; + # Set mime types + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + # Use default zone for rate limiting, allow burst of 10 requests with + # no delay + # limit_req zone=default burst=10 nodelay; + + error_page 404 /older_builds$request_uri; +}