Merge pull request #281 from inbo/fix-timeout-export

fix-export-timeout

entrypoint.sh

@@ -34,11 +34,11 @@ echo "Load waarnemingen observation data via: python manage.py load_waarnemingen
 # Start Gunicorn
 echo "Starting Gunicorn..."
 gunicorn --workers 3 \
-         --timeout 300 \
+         --timeout 1800 \
          --keep-alive 65 \
          --bind 0.0.0.0:8000 \
          vespadb.wsgi:application &
-
+         
 # Wait for Gunicorn to start
 sleep 5
 

nginx.conf

@@ -1,4 +1,4 @@
-worker_processes auto;
+worker_processes auto;  # Changed from 1 to auto for better performance
 
 events {
     worker_connections 4096;
@@ -19,7 +19,7 @@ http {
     # Global timeout settings
     proxy_connect_timeout 600;
     proxy_send_timeout 600;
-    proxy_read_timeout 600;
+    proxy_read_timeout 1800;
     send_timeout 600;
     keepalive_timeout 650;
 
@@ -63,7 +63,7 @@ http {
             # Timeouts
             proxy_connect_timeout 600s;  
             proxy_send_timeout 600s;
-            proxy_read_timeout 600s;
+            proxy_read_timeout 1800s;
 
             # Buffer settings for large files
             proxy_buffering on;          
@@ -103,7 +103,7 @@ http {
             # Timeouts
             proxy_connect_timeout 600s;
             proxy_send_timeout 600s;
-            proxy_read_timeout 600s;
+            proxy_read_timeout 1800s;
 
             # Buffer settings for large files
             proxy_buffering on;         

vespadb/observations/views.py

@@ -906,7 +906,7 @@ def export(self, request: HttpRequest) -> Union[FileResponse, JsonResponse]:
 
         try:
             # Create temporary file
-            temp_file = tempfile.NamedTemporaryFile(mode='w+', delete=False)
+            temp_file = tempfile.NamedTemporaryFile(mode='w+', delete=False, encoding='utf-8-sig')
             temp_file_path = temp_file.name
 
             writer = csv.writer(temp_file)
@@ -925,12 +925,12 @@ def export(self, request: HttpRequest) -> Union[FileResponse, JsonResponse]:
                 self.get_queryset().select_related('province', 'municipality', 'reserved_by')
             )
 
-            # Set a smaller chunk size for better memory management
-            chunk_size = 500
+            # Use much smaller chunk size
+            chunk_size = 100
             total_count = queryset.count()
             processed = 0
 
-            # Process in chunks
+            # Process in chunks with periodic flushes
             for start in range(0, total_count, chunk_size):
                 chunk = queryset[start:start + chunk_size]
 
@@ -946,6 +946,10 @@ def export(self, request: HttpRequest) -> Union[FileResponse, JsonResponse]:
                         logger.error(f"Error processing observation {observation.id}: {str(e)}")
                         continue
 
+                # Flush after each chunk
+                temp_file.flush()
+                os.fsync(temp_file.fileno())
+
                 processed += len(chunk)
                 logger.info(f"Export progress: {(processed/total_count)*100:.1f}%")
 
@@ -955,24 +959,23 @@ def export(self, request: HttpRequest) -> Union[FileResponse, JsonResponse]:
             temp_file.close()
 
             # Open the file for reading and create response
-            filename=f"observations_export_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
-
             response = FileResponse(
                 open(temp_file_path, 'rb'),
-                content_type='text/csv',
-                as_attachment=True,
-                filename=filename
+                content_type='text/csv'
             )
-            # Set headers more explicitly
+
+            # Set explicit headers
+            filename = f"observations_export_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
             response['Content-Disposition'] = f'attachment; filename="{filename}"; filename*=UTF-8\'\'{filename}'
             response['Content-Type'] = 'text/csv; charset=utf-8'
             response['Content-Length'] = os.path.getsize(temp_file_path)
-            response['Cache-Control'] = 'no-cache'
+            response['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+            response['Pragma'] = 'no-cache'
+            response['Expires'] = '0'
             response['X-Accel-Buffering'] = 'no'
 
             # Schedule file cleanup after response is sent
-            def cleanup_temp_file(response: FileResponse) -> Any:
-                """."""
+            def cleanup_temp_file(response):
                 try:
                     os.unlink(temp_file_path)
                 except:
@@ -994,7 +997,7 @@ def cleanup_temp_file(response: FileResponse) -> Any:
                 except:
                     pass
             return JsonResponse(
-                {"error": "Export failed. Please try again or contact support."}, 
+                {"error": f"Export failed: {str(e)}. Please try again or contact support."}, 
                 status=500
             )