Check if crawling has already been stopping

dlt-hub · Mar 1, 2024 · 3ae0f1e · 3ae0f1e
1 parent 2c4fd60
commit 3ae0f1e
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/sources/scraping/runner.py b/sources/scraping/runner.py
@@ -29,8 +29,9 @@ def on_item_scraped(self, item: Item) -> None:
                 "Queue is closed, stopping",
                 extra={"pipeline_name": self.pipeline_name},
             )
-            self.stopping = True
-            self.crawler.stop()
+            if not self.stopping:
+                self.stopping = True
+                self.crawler.stop()
 
     def on_engine_stopped(self) -> None:
         logger.info(f"Crawling engine stopped for pipeline={self.pipeline_name}")