支持python3.12

feapder/core/scheduler.py

@@ -17,23 +17,24 @@
 from feapder.buffer.request_buffer import RequestBuffer
 from feapder.core.base_parser import BaseParser
 from feapder.core.collector import Collector
-from feapder.core.handle_failed_requests import HandleFailedRequests
 from feapder.core.handle_failed_items import HandleFailedItems
+from feapder.core.handle_failed_requests import HandleFailedRequests
 from feapder.core.parser_control import ParserControl
 from feapder.db.redisdb import RedisDB
 from feapder.network.item import Item
 from feapder.network.request import Request
 from feapder.utils import metrics
 from feapder.utils.log import log
 from feapder.utils.redis_lock import RedisLock
+from feapder.utils.tail_thread import TailThread
 
 SPIDER_START_TIME_KEY = "spider_start_time"
 SPIDER_END_TIME_KEY = "spider_end_time"
 SPIDER_LAST_TASK_COUNT_RECORD_TIME_KEY = "last_task_count_record_time"
 HEARTBEAT_TIME_KEY = "heartbeat_time"
 
 
-class Scheduler(threading.Thread):
+class Scheduler(TailThread):
     __custom_setting__ = {}
 
     def __init__(

feapder/core/spiders/air_spider.py

@@ -8,8 +8,6 @@
 @email: [email protected]
 """
 
-from threading import Thread
-
 import feapder.setting as setting
 import feapder.utils.tools as tools
 from feapder.buffer.item_buffer import ItemBuffer
@@ -20,9 +18,10 @@
 from feapder.network.request import Request
 from feapder.utils import metrics
 from feapder.utils.log import log
+from feapder.utils.tail_thread import TailThread
 
 
-class AirSpider(BaseParser, Thread):
+class AirSpider(BaseParser, TailThread):
     __custom_setting__ = {}
 
     def __init__(self, thread_count=None):

feapder/utils/tail_thread.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2024/3/19 20:00
+---------
+@summary:
+---------
+@author: Boris
+@email: [email protected]
+"""
+import sys
+import threading
+
+
+class TailThread(threading.Thread):
+    """
+    所有子线程结束后，主线程才会退出
+    """
+
+    def start(self) -> None:
+        """
+        解决python3.12 RuntimeError: cannot join thread before it is started的报错
+        """
+        super().start()
+
+        if sys.version_info.minor >= 12 and sys.version_info.major >= 3:
+            for thread in threading.enumerate():
+                if (
+                    thread.daemon
+                    or thread is threading.current_thread()
+                    or not thread.is_alive()
+                ):
+                    continue
+                thread.join()