From cee4425a9a9b33e76fcdee32ca3f618e34f6f619 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 29 Aug 2024 19:35:48 +0100 Subject: [PATCH] build: update to prod config using cgimap and background worker services --- docker-compose.yml | 73 ++++++++++++++++++++++++++- nginx/templates/osm-dev.conf.template | 51 ++++++++++++++++++- nginx/templates/osm.conf.template | 34 +++++++++++-- 3 files changed, 153 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6e17363..b4a0342 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ networks: volumes: osm-tmp: osm-storage: + osm-config: osm-db-data: certs: certbot_data: @@ -21,6 +22,8 @@ services: depends_on: osm: condition: service_started + osm-cgi: + condition: service_started ports: - ${OSM_DEV_PORT:-4433}:80 networks: @@ -38,6 +41,8 @@ services: depends_on: osm: condition: service_started + osm-cgi: + condition: service_started certbot: condition: service_completed_successfully volumes: @@ -53,29 +58,87 @@ services: restart: "unless-stopped" osm: - image: ghcr.io/hotosm/osm-sandbox:2024.4.30 + image: "ghcr.io/hotosm/osm-sandbox:2024.4.30" build: . + depends_on: + osm-db: + condition: service_healthy environment: PROTOCOL: http${DOMAIN:+s} + # NOTE for development this must be 127.0.0.1 due to + # OSM oauth config restrictions DOMAIN: ${DOMAIN:-127.0.0.1:4433} ADMIN_EMAIL: ${ADMIN_EMAIL:-admin@hotosm.org} ADMIN_PASS: ${ADMIN_PASS:-Password1234} ID_EDITOR_REDIRECT_URI: http${DOMAIN:+s}://${DOMAIN:-127.0.0.1:4433} volumes: + # Mount a tmp directory that will persist between runs + - osm-tmp:/app/tmp + # Mount a storage directory that will persist between runs + - osm-storage:/app/storage + # Mount config between containers + - osm-config:/app/config + # Mount local setting overrides + # - ./settings.local.yml:/app/config/settings.local.yml:ro + tmpfs: + /tmp/pids/ + networks: + - osm-net + restart: unless-stopped + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/3000' || exit 1 + interval: 5s + retries: 3 + start_period: 5s + timeout: 5s + + osm-jobs: + image: "ghcr.io/hotosm/osm-sandbox:2024.4.30" + depends_on: + osm: + condition: service_healthy volumes: # Mount a tmp directory that will persist between runs - osm-tmp:/app/tmp # Mount a storage directory that will persist between runs - osm-storage:/app/storage + # Mount config between containers + - osm-config:/app/config # Mount local setting overrides # - ./settings.local.yml:/app/config/settings.local.yml:ro tmpfs: /tmp/pids/ networks: - osm-net + restart: unless-stopped + entrypoint: /bin/sh -c + command: + - | + echo "Running background worker" + bundle exec rake jobs:work + + osm-cgi: + image: "ghcr.io/hotosm/osm-sandbox/cgimap:${CGIMAP_VERSION:-v2.0.0.pre}" + build: + context: https://github.com/zerebubuth/openstreetmap-cgimap.git#${CGIMAP_VERSION:-v2.0.0} + dockerfile: docker/debian/Dockerfile_bookworm depends_on: osm-db: condition: service_healthy + environment: + CGIMAP_HOST: osm-db + CGIMAP_DBNAME: openstreetmap + CGIMAP_USERNAME: openstreetmap + CGIMAP_PASSWORD: openstreetmap + CGIMAP_MEMCACHE: memcached + CGIMAP_RATELIMIT: 204800 + CGIMAP_MAXDEBT: 250 + CGIMAP_MODERATOR_RATELIMIT: 1048576 + CGIMAP_MODERATOR_MAXDEBT: 1024 + CGIMAP_PORT: 8000 + CGIMAP_INSTANCES: 3 + networks: + - osm-net restart: unless-stopped osm-db: @@ -107,6 +170,14 @@ services: - osm-net restart: unless-stopped + memcached: + image: "docker.io/memcached:1.6" + # ports: + # - 11211:11211 + networks: + - osm-net + restart: unless-stopped + certbot: image: "ghcr.io/hotosm/osm-sandbox/proxy:certs-init" profiles: [public] diff --git a/nginx/templates/osm-dev.conf.template b/nginx/templates/osm-dev.conf.template index 4264c72..ffad06a 100644 --- a/nginx/templates/osm-dev.conf.template +++ b/nginx/templates/osm-dev.conf.template @@ -3,6 +3,11 @@ upstream openstreetmap { keepalive 32; } +upstream cgimap { + server osm-cgi:8000; + keepalive 32; +} + server { # Default handler for port 80 listen 80 reuseport; @@ -10,8 +15,51 @@ server { client_max_body_size 10M; + # Route specific paths to cgimap + location ~ ^/api/0\.6/map$ { + include /etc/nginx/fastcgi_params; + fastcgi_pass cgimap; + } + + location ~ ^/api/0\.6/(nodes|ways|relations)$ { + include /etc/nginx/fastcgi_params; + fastcgi_pass cgimap; + } + + location ~ ^/api/0\.6/(way|relation)/([^/]+)/full$ { + include /etc/nginx/fastcgi_params; + fastcgi_pass cgimap; + } + + location ~ ^/api/0\.6/(node|way|relation)/([^/]+)$ { + include /etc/nginx/fastcgi_params; + if ($request_method ~ ^(GET|HEAD)$) { + fastcgi_pass cgimap; + } + + + # TODO add handling for other methods needed? + #set $cgimap 0; + + #if ($request_method = GET) { + # set $cgimap 1; + #} + #if ($request_method = HEAD) { + # set $cgimap 1; + #} + + #if ($cgimap) { + # include /etc/nginx/fastcgi_params; + # fastcgi_pass cgimap; + # break; + #} + + #proxy_pass http://openstreetmap; + } + + # Default location block - fallback to openstreetmap location / { - # Requests headers + # Request headers proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; @@ -26,6 +74,7 @@ server { proxy_buffers 8 64k; proxy_busy_buffers_size 64k; + # Pass everything else to the main API server proxy_pass http://openstreetmap; } diff --git a/nginx/templates/osm.conf.template b/nginx/templates/osm.conf.template index 5613eaf..9cb27ce 100644 --- a/nginx/templates/osm.conf.template +++ b/nginx/templates/osm.conf.template @@ -3,15 +3,20 @@ upstream openstreetmap { keepalive 32; } +upstream cgimap { + server osm-cgi:8000; + keepalive 32; +} + server { - # Default handler for port 80 + # Redirect all HTTP traffic to HTTPS listen 80; server_name ${DOMAIN}; return 301 https://$host$request_uri; } server { - # Default handler for port 443 + # Default handler for port 443 (HTTPS) listen 443 ssl reuseport; server_name ${DOMAIN}; @@ -24,8 +29,31 @@ server { add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # Route specific paths to cgimap + location ~ ^/api/0\.6/map$ { + include /etc/nginx/fastcgi_params; + fastcgi_pass cgimap; + } + + location ~ ^/api/0\.6/(nodes|ways|relations)$ { + include /etc/nginx/fastcgi_params; + fastcgi_pass cgimap; + } + + location ~ ^/api/0\.6/(way|relation)/([^/]+)/full$ { + include /etc/nginx/fastcgi_params; + fastcgi_pass cgimap; + } + + location ~ ^/api/0\.6/(node|way|relation)/([^/]+)$ { + include /etc/nginx/fastcgi_params; + if ($request_method ~ ^(GET|HEAD)$) { + fastcgi_pass cgimap; + } + } + location / { - # Requests headers + # Request headers proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Proto $scheme;