diff --git a/feapder/core/scheduler.py b/feapder/core/scheduler.py index 4ef854d6..0177d185 100644 --- a/feapder/core/scheduler.py +++ b/feapder/core/scheduler.py @@ -17,8 +17,8 @@ from feapder.buffer.request_buffer import RequestBuffer from feapder.core.base_parser import BaseParser from feapder.core.collector import Collector -from feapder.core.handle_failed_requests import HandleFailedRequests from feapder.core.handle_failed_items import HandleFailedItems +from feapder.core.handle_failed_requests import HandleFailedRequests from feapder.core.parser_control import ParserControl from feapder.db.redisdb import RedisDB from feapder.network.item import Item @@ -26,6 +26,7 @@ from feapder.utils import metrics from feapder.utils.log import log from feapder.utils.redis_lock import RedisLock +from feapder.utils.tail_thread import TailThread SPIDER_START_TIME_KEY = "spider_start_time" SPIDER_END_TIME_KEY = "spider_end_time" @@ -33,7 +34,7 @@ HEARTBEAT_TIME_KEY = "heartbeat_time" -class Scheduler(threading.Thread): +class Scheduler(TailThread): __custom_setting__ = {} def __init__( diff --git a/feapder/core/spiders/air_spider.py b/feapder/core/spiders/air_spider.py index 33070fb7..70c30112 100644 --- a/feapder/core/spiders/air_spider.py +++ b/feapder/core/spiders/air_spider.py @@ -8,8 +8,6 @@ @email: boris_liu@foxmail.com """ -from threading import Thread - import feapder.setting as setting import feapder.utils.tools as tools from feapder.buffer.item_buffer import ItemBuffer @@ -20,9 +18,10 @@ from feapder.network.request import Request from feapder.utils import metrics from feapder.utils.log import log +from feapder.utils.tail_thread import TailThread -class AirSpider(BaseParser, Thread): +class AirSpider(BaseParser, TailThread): __custom_setting__ = {} def __init__(self, thread_count=None): diff --git a/feapder/utils/tail_thread.py b/feapder/utils/tail_thread.py new file mode 100644 index 00000000..eda266d5 --- /dev/null +++ b/feapder/utils/tail_thread.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Created on 2024/3/19 20:00 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +import sys +import threading + + +class TailThread(threading.Thread): + """ + 所有子线程结束后,主线程才会退出 + """ + + def start(self) -> None: + """ + 解决python3.12 RuntimeError: cannot join thread before it is started的报错 + """ + super().start() + + if sys.version_info.minor >= 12 and sys.version_info.major >= 3: + for thread in threading.enumerate(): + if ( + thread.daemon + or thread is threading.current_thread() + or not thread.is_alive() + ): + continue + thread.join()