build: update to prod config using cgimap and background worker services

docker-compose.yml

@@ -5,6 +5,7 @@ networks:
 volumes:
   osm-tmp:
   osm-storage:
+  osm-config:
   osm-db-data:
   certs:
   certbot_data:
@@ -21,6 +22,8 @@ services:
     depends_on:
       osm:
         condition: service_started
+      osm-cgi:
+        condition: service_started
     ports:
       - ${OSM_DEV_PORT:-4433}:80
     networks:
@@ -38,6 +41,8 @@ services:
     depends_on:
       osm:
         condition: service_started
+      osm-cgi:
+        condition: service_started
       certbot:
         condition: service_completed_successfully
     volumes:
@@ -53,29 +58,87 @@ services:
     restart: "unless-stopped"
 
   osm:
-    image: ghcr.io/hotosm/osm-sandbox:2024.4.30
+    image: "ghcr.io/hotosm/osm-sandbox:2024.4.30"
     build: .
+    depends_on:
+      osm-db:
+        condition: service_healthy
     environment:
       PROTOCOL: http${DOMAIN:+s}
+      # NOTE for development this must be 127.0.0.1 due to
+      # OSM oauth config restrictions
       DOMAIN: ${DOMAIN:-127.0.0.1:4433}
       ADMIN_EMAIL: ${ADMIN_EMAIL:[email protected]}
       ADMIN_PASS: ${ADMIN_PASS:-Password1234}
       ID_EDITOR_REDIRECT_URI: http${DOMAIN:+s}://${DOMAIN:-127.0.0.1:4433}
     volumes:
+      # Mount a tmp directory that will persist between runs
+      - osm-tmp:/app/tmp
+      # Mount a storage directory that will persist between runs
+      - osm-storage:/app/storage
+      # Mount config between containers
+      - osm-config:/app/config
+      # Mount local setting overrides
+      # - ./settings.local.yml:/app/config/settings.local.yml:ro
+    tmpfs:
+      /tmp/pids/
+    networks:
+      - osm-net
+    restart: unless-stopped
+    healthcheck:
+      test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/3000' || exit 1
+      interval: 5s
+      retries: 3
+      start_period: 5s
+      timeout: 5s
+
+  osm-jobs:
+    image: "ghcr.io/hotosm/osm-sandbox:2024.4.30"
+    depends_on:
+      osm:
+        condition: service_healthy
     volumes:
       # Mount a tmp directory that will persist between runs
       - osm-tmp:/app/tmp
       # Mount a storage directory that will persist between runs
       - osm-storage:/app/storage
+      # Mount config between containers
+      - osm-config:/app/config
       # Mount local setting overrides
       # - ./settings.local.yml:/app/config/settings.local.yml:ro
     tmpfs:
       /tmp/pids/
     networks:
       - osm-net
+    restart: unless-stopped
+    entrypoint: /bin/sh -c
+    command:
+      - |
+        echo "Running background worker"
+        bundle exec rake jobs:work
+
+  osm-cgi:
+    image: "ghcr.io/hotosm/osm-sandbox/cgimap:${CGIMAP_VERSION:-v2.0.0.pre}"
+    build:
+      context: https://github.com/zerebubuth/openstreetmap-cgimap.git#${CGIMAP_VERSION:-v2.0.0}
+      dockerfile: docker/debian/Dockerfile_bookworm
     depends_on:
       osm-db:
         condition: service_healthy
+    environment:
+      CGIMAP_HOST: osm-db
+      CGIMAP_DBNAME: openstreetmap
+      CGIMAP_USERNAME: openstreetmap
+      CGIMAP_PASSWORD: openstreetmap
+      CGIMAP_MEMCACHE: memcached
+      CGIMAP_RATELIMIT: 204800
+      CGIMAP_MAXDEBT: 250
+      CGIMAP_MODERATOR_RATELIMIT: 1048576
+      CGIMAP_MODERATOR_MAXDEBT: 1024
+      CGIMAP_PORT: 8000
+      CGIMAP_INSTANCES: 3
+    networks:
+      - osm-net
     restart: unless-stopped
 
   osm-db:
@@ -107,6 +170,14 @@ services:
       - osm-net
     restart: unless-stopped
 
+  memcached:
+    image: "docker.io/memcached:1.6"
+    # ports:
+    #   - 11211:11211
+    networks:
+      - osm-net
+    restart: unless-stopped
+
   certbot:
     image: "ghcr.io/hotosm/osm-sandbox/proxy:certs-init"
     profiles: [public]

nginx/templates/osm-dev.conf.template

@@ -3,15 +3,63 @@ upstream openstreetmap {
     keepalive 32;
 }
 
+upstream cgimap {
+    server osm-cgi:8000;
+    keepalive 32;
+}
+
 server {
     # Default handler for port 80
     listen 80 reuseport;
     server_name osm.localhost;
 
     client_max_body_size 10M;
 
+    # Route specific paths to cgimap
+    location ~ ^/api/0\.6/map$ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass cgimap;
+    }
+
+    location ~ ^/api/0\.6/(nodes|ways|relations)$ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass cgimap;
+    }
+
+    location ~ ^/api/0\.6/(way|relation)/([^/]+)/full$ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass cgimap;
+    }
+
+    location ~ ^/api/0\.6/(node|way|relation)/([^/]+)$ {
+        include /etc/nginx/fastcgi_params;
+        if ($request_method ~ ^(GET|HEAD)$) {
+            fastcgi_pass cgimap;
+        }
+
+
+        # TODO add handling for other methods needed?
+        #set $cgimap 0;
+
+        #if ($request_method = GET) {
+        #    set $cgimap 1;
+        #}
+        #if ($request_method = HEAD) {
+        #    set $cgimap 1;
+        #}
+
+        #if ($cgimap) {
+        #    include /etc/nginx/fastcgi_params;
+        #    fastcgi_pass cgimap;
+        #    break;
+        #}
+
+        #proxy_pass http://openstreetmap;
+    }
+
+    # Default location block - fallback to openstreetmap
     location / {
-        # Requests headers
+        # Request headers
         proxy_set_header Host                $http_host;
         proxy_set_header X-Real-IP           $remote_addr;
         proxy_set_header X-Forwarded-Proto   $scheme;
@@ -26,6 +74,7 @@ server {
         proxy_buffers 8 64k;
         proxy_busy_buffers_size 64k;
 
+        # Pass everything else to the main API server
         proxy_pass http://openstreetmap;
     }
 

nginx/templates/osm.conf.template

@@ -3,15 +3,20 @@ upstream openstreetmap {
     keepalive 32;
 }
 
+upstream cgimap {
+    server osm-cgi:8000;
+    keepalive 32;
+}
+
 server {
-    # Default handler for port 80
+    # Redirect all HTTP traffic to HTTPS
     listen 80;
     server_name ${DOMAIN};
     return 301 https://$host$request_uri;
 }
 
 server {
-    # Default handler for port 443
+    # Default handler for port 443 (HTTPS)
     listen 443 ssl reuseport;
     server_name ${DOMAIN};
 
@@ -24,8 +29,31 @@ server {
 
     add_header 'Content-Security-Policy' 'upgrade-insecure-requests';
 
+    # Route specific paths to cgimap
+    location ~ ^/api/0\.6/map$ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass cgimap;
+    }
+
+    location ~ ^/api/0\.6/(nodes|ways|relations)$ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass cgimap;
+    }
+
+    location ~ ^/api/0\.6/(way|relation)/([^/]+)/full$ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass cgimap;
+    }
+
+    location ~ ^/api/0\.6/(node|way|relation)/([^/]+)$ {
+        include /etc/nginx/fastcgi_params;
+        if ($request_method ~ ^(GET|HEAD)$) {
+            fastcgi_pass cgimap;
+        }
+    }
+
     location / {
-        # Requests headers
+        # Request headers
         proxy_set_header Host                $http_host;
         proxy_set_header X-Real-IP           $remote_addr;
         proxy_set_header X-Forwarded-Proto   $scheme;