Skip to content

Commit

Permalink
MDBF-804 - BB NGINX configuration in GH CI
Browse files Browse the repository at this point in the history
About This Patch:
- Using templates (present since NGINX 1.19) to populate conf.d.
- Templates allow for the use of environment variables defined in the .env files, enabling us to distinguish between PROD and DEV environments, particularly for the server name and certificate paths.
- A proxy_params file is required according to the PROD configuration.
- Mounting /etc/letsencrypt/live for SSL certificates. The base path is the same in both environments.
- NGINX_ARTIFACTS_SSL_PATH variable is necessary because, in DEV, the same certificate is used for both CI and BB.
- Attaching net_back to the NGINX container to facilitate communication with master-web via DNS.
- Removing net_front from master-web; communication will be handled through NGINX.
- NGINX access/error logs are written to the Docker-Compose relative path logs/nginx, which is needed for Zabbix collection.

TODO Before Migration to PROD:
- Address all FIXME comments.
  - Cross-reference proxy pass
  - helper_files directory name on hz-bbm2.
  - location /cloud-init ?

TODO Before Deployment in DEV:
- Disable the HAProxy service.
  • Loading branch information
RazvanLiviuVarzaru committed Oct 10, 2024
1 parent c8775dd commit 91abbe0
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 36 deletions.
2 changes: 2 additions & 0 deletions docker-compose/.env
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ MASTER_PACKAGES_DIR="/mnt/autofs/master_packages"
GALERA_PACKAGES_DIR="/mnt/autofs/galera_packages"
ARTIFACTS_URL="https://ci.mariadb.org"
NGINX_ARTIFACTS_VHOST="ci.mariadb.org"
NGINX_ARTIFACTS_SSL_PATH="ci.mariadb.org"
NGINX_BUILDBOT_VHOST="buildbot.mariadb.org"
ENVIRON="PROD"
BRANCH="main"
MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel8-docker"
Expand Down
2 changes: 2 additions & 0 deletions docker-compose/.env.dev
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ MASTER_PACKAGES_DIR="/mnt/autofs/master_dev_packages"
GALERA_PACKAGES_DIR="/mnt/autofs/galera_dev_packages"
ARTIFACTS_URL="https://ci.dev.mariadb.org"
NGINX_ARTIFACTS_VHOST="ci.dev.mariadb.org"
NGINX_ARTIFACTS_SSL_PATH="buildbot.dev.mariadb.org"
NGINX_BUILDBOT_VHOST="buildbot.dev.mariadb.org"
ENVIRON="DEV"
BRANCH="dev"
MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel9-docker"
Expand Down
44 changes: 39 additions & 5 deletions docker-compose/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,23 @@ services:
hostname: nginx
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./nginx/conf.d/:/etc/nginx/conf.d/:ro
- ./nginx/proxy_params:/etc/nginx/proxy_params:ro
- ./nginx/templates/:/etc/nginx/templates/:ro
- /srv/buildbot/packages:/srv/buildbot/packages:ro
- /srv/buildbot/galera_packages:/srv/buildbot/galera_packages:ro
- /srv/buildbot/helper_files:/srv/buildbot/helper_files:ro
- /etc/letsencrypt/live:/etc/nginx/ssl
- ./logs/nginx:/var/log/nginx
ports:
- "127.0.0.1:8080:80"
- "443:443"
- "80:80"
environment:
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- NGINX_ARTIFACTS_SSL_PATH
networks:
net_front:
net_back:
logging:
driver: journald
options:
Expand All @@ -77,7 +86,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=8010
- TITLE
- TITLE_URL
Expand All @@ -88,10 +99,7 @@ services:
entrypoint:
- /srv/buildbot/master/docker-compose/start-bbm-web.sh
networks:
net_front:
net_back:
ports:
- "127.0.0.1:8010:8010"
depends_on:
- mariadb
- crossbar
Expand All @@ -114,7 +122,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=9996
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -154,7 +164,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=9997
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -193,7 +205,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=9998
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -232,7 +246,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=9999
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -271,7 +287,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10000
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -310,7 +328,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10001
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -349,7 +369,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10002
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -388,7 +410,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10003
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -427,7 +451,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10004
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -466,7 +492,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10005
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -505,7 +533,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10006
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -544,7 +574,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10007
- TITLE
- TITLE_URL
Expand Down Expand Up @@ -583,7 +615,9 @@ services:
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MQ_ROUTER_URL
- NGINX_ARTIFACTS_SSL_PATH
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- PORT=10008
- TITLE
- TITLE_URL
Expand Down
16 changes: 11 additions & 5 deletions docker-compose/generate-config.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,23 @@
hostname: nginx
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./nginx/conf.d/:/etc/nginx/conf.d/:ro
- ./nginx/proxy_params:/etc/nginx/proxy_params:ro
- ./nginx/templates/:/etc/nginx/templates/:ro
- /srv/buildbot/packages:/srv/buildbot/packages:ro
- /srv/buildbot/galera_packages:/srv/buildbot/galera_packages:ro
- /srv/buildbot/helper_files:/srv/buildbot/helper_files:ro
- /etc/letsencrypt/live:/etc/nginx/ssl
- ./logs/nginx:/var/log/nginx
ports:
- "127.0.0.1:8080:80"
- "443:443"
- "80:80"
environment:
- NGINX_ARTIFACTS_VHOST
- NGINX_BUILDBOT_VHOST
- NGINX_ARTIFACTS_SSL_PATH
networks:
net_front:
net_back:
logging:
driver: journald
options:
Expand All @@ -96,10 +105,7 @@
entrypoint:
- /srv/buildbot/master/docker-compose/start-bbm-web.sh
networks:
net_front:
net_back:
ports:
- "127.0.0.1:8010:8010"
depends_on:
- mariadb
- crossbar
Expand Down
26 changes: 0 additions & 26 deletions docker-compose/nginx/conf.d/ci.conf

This file was deleted.

4 changes: 4 additions & 0 deletions docker-compose/nginx/proxy_params
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
101 changes: 101 additions & 0 deletions docker-compose/nginx/templates/bb.conf.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
server {
listen 80;
listen [::]:80;
server_name ${NGINX_BUILDBOT_VHOST};
return 301 https://$server_name$request_uri;
}


# Default rate limited zone, with 30 requests per minute
limit_req_zone $request_uri zone=default:10m rate=30r/m;
client_max_body_size 10M;

server {
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;

server_name ${NGINX_BUILDBOT_VHOST};

# logging
access_log /var/log/nginx/buildbot.access.log;
error_log /var/log/nginx/buildbot.error.log error;

# SSL configuration
# ssl on; Deprecated in newer versions of NGINX (yields nginx: [emerg] unknown directive "ssl )
ssl_certificate /etc/nginx/ssl/${NGINX_BUILDBOT_VHOST}/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/nginx/ssl/${NGINX_BUILDBOT_VHOST}/privkey.pem; # managed by Certbot
# put a one day session timeout for websockets to stay longer
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 1d;
ssl_protocols TLSv1.1 TLSv1.2;

# Force https - Enable HSTS
add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;" always;
# # Disable embedding the site
add_header X-Frame-Options "SAMEORIGIN";
# # Enable XSS protection
add_header X-XSS-Protection "1;mode=block";

# Enable gziped format
#gzip on; already on in main conf
# Set level of compression
gzip_comp_level 3;
# Set mime types
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Forwarded-Host $host;

# Use default zone for rate limiting, allow burst of 10 requests with
# no delay
limit_req zone=default burst=10 nodelay;

location / {
# Reverse proxy settings
include proxy_params;
proxy_pass http://master-web:8010;
}

# disable logging for wsgi_dashboards/styles.css since it's generated
# somewhere and mess with fail2ban //TEMP find the root cause!
location ~ /wsgi_dashboards/styles.css* {
access_log off;
}
location = /favicon.ico {
access_log off;
}
location = /robots.txt {
access_log off;
}

# Server sent event (sse) settings
location /sse {
proxy_buffering off;
proxy_pass http://master-web:8010/sse;
}

# Websocket settings
location /ws {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_pass http://master-web:8010/ws;
proxy_read_timeout 6000s;
}


#FIXME: CrossReference not in DEV. ProxyPass for PROD?
# Cross-reference
# location /cr/static {
# alias /srv/cr/static;
# }

# location /cr/ {
# include proxy_params;
# proxy_pass http://hz-bbw5:8080;
# }
}
Loading

0 comments on commit 91abbe0

Please sign in to comment.