支持邮件报警

Boris-code · Mar 26, 2021 · 12a2c57 · 12a2c57
1 parent 210076e
commit 12a2c57
Show file tree

Hide file tree

Showing 10 changed files with 261 additions and 28 deletions.
diff --git a/docs/_sidebar.md b/docs/_sidebar.md
@@ -29,6 +29,7 @@
   * [RedisDB](source_code/RedisDB.md)
   * [工具库-tools](source_code/tools.md)
   * [海量数据去重-dedup](source_code/dedup.md)
+  * [报警及监控](source_code/报警及监控.md)
 
 * 常见问题
   * [安装问题](question/安装问题.md)

diff --git a/docs/source_code/报警及监控.md b/docs/source_code/报警及监控.md
@@ -0,0 +1,64 @@
+# 报警及监控
+
+## 钉钉报警
+
+条件：需要有钉钉群，需要获取钉钉机器人的Webhook地址
+
+获取方式参考官方文档：https://developers.dingtalk.com/document/app/custom-robot-access
+
+安全设置选择自定义关键词，填入**feapder**
+
+![-w547](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167753030324.jpg)
+
+相关配置：
+
+```python
+# 钉钉报警
+DINGDING_WARNING_URL = ""  # 钉钉机器人api
+DINGDING_WARNING_PHONE = ""  # 报警人 支持列表，可指定多个
+```
+
+## 邮件报警
+
+相关配置：
+
+```
+# 邮件报警
+EAMIL_SENDER = ""  # 发件人
+EAMIL_PASSWORD = ""  # 授权码
+EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个
+```
+
+邮件报警目前支持163邮箱作为发送者，`EAMIL_SENDER`为邮箱账号，如`[email protected]`, `EAMIL_PASSWORD`为授权码，不是登录密码，获取授权码的流程如下：
+
+1. 设置 -> POP3/SMTP/IMAP
+
+    ![-w258](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167719328720.jpg)
+
+2. 开启SMTP服务
+
+    ![-w444](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167719490656.jpg)
+
+    开启后，会弹出授权码，该授权码即为EAMIL_PASSWORD
+
+3. 设置反垃圾规则为高级
+
+    ![-w1112](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167719655644.jpg)
+
+## 报警间隔及报警级别
+
+框架会对相同的报警进行过滤，防止刷屏，默认的报警时间间隔为1小时，可通过以下配置修改：
+
+```python
+WARNING_INTERVAL = 3600  # 相同报警的报警时间间隔，防止刷屏
+WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / ERROR
+```
+
+DEBUG级别的报警包含一些运行信息，ERROR级别的报警都是有问题的报警，需要及时处理
+
+
+## 可视化监控
+
+所需环境：Influxdb + Grafana
+
+未完待续，敬请期待...
diff --git a/docs/source_code/配置文件.md b/docs/source_code/配置文件.md
@@ -81,10 +81,17 @@ USE_SESSION = False
 ITEM_FILTER_ENABLE = False  # item 去重
 REQUEST_FILTER_ENABLE = False  # request 去重
 
-# 报警
+# 报警 支持钉钉及邮件，二选一即可
+# 钉钉报警
 DINGDING_WARNING_URL = ""  # 钉钉机器人api
-DINGDING_WARNING_PHONE = ""  # 报警人
-LINGXI_TOKEN = ""  # 灵犀报警token
+DINGDING_WARNING_PHONE = ""  # 报警人 支持列表，可指定多个
+# 邮件报警
+EAMIL_SENDER = ""  # 发件人
+EAMIL_PASSWORD = ""  # 授权码
+EMAIL_RECEIVER = ""  # 收件人 支持列表，可指定多个
+# 时间间隔
+WARNING_INTERVAL = 3600  # 相同报警的报警时间间隔，防止刷屏
+WARNING_LEVEL = "DEBUG"  # 报警级别， DEBUG / ERROR
 
 LOG_NAME = os.path.basename(os.getcwd())
 LOG_PATH = "log/%s.log" % LOG_NAME  # log存储路径

diff --git a/feapder/VERSION b/feapder/VERSION
@@ -1 +1 @@
-1.3.8-beta1
+1.3.8
diff --git a/feapder/core/scheduler.py b/feapder/core/scheduler.py
@@ -433,16 +433,18 @@ def _stop_all_thread(self):
             parser_control.stop()
 
     def send_msg(self, msg, level="debug", message_prefix=""):
-        """
-        @summary: 叮叮 发送消息
-        ---------
-        @param msg: 消息
-        @param developers: 开发者姓名
-        ---------
-        @result:
-        """
+        if setting.WARNING_LEVEL == "ERROR":
+            if level != "error":
+                return
+
+        if setting.DINGDING_WARNING_PHONE:
+            keyword = "feapder报警系统\n"
+            tools.dingding_warning(keyword + msg, message_prefix=message_prefix)
 
-        tools.dingding_warning(msg, rate_limit=3600, message_prefix=message_prefix)
+        if setting.EMAIL_RECEIVER:
+            tools.email_warning(
+                msg, message_prefix=message_prefix, title=self._spider_name
+            )
 
     def spider_begin(self):
         """
@@ -466,7 +468,7 @@ def spider_begin(self):
             )
 
             # 发送消息
-            # self.send_msg('《%s》爬虫开始'%self._spider_name)
+            self.send_msg("《%s》爬虫开始" % self._spider_name)
 
     def spider_end(self):
         self.record_end_time()

diff --git a/feapder/core/spiders/batch_spider.py b/feapder/core/spiders/batch_spider.py
@@ -1018,7 +1018,7 @@ def run(self):
         except Exception as e:
             msg = "《%s》主线程异常 爬虫结束 exception: %s" % (self._batch_name, e)
             log.error(msg)
-            self.send_msg(msg)
+            self.send_msg(msg, level="error")
 
             os._exit(137)  # 使退出码为35072 方便爬虫管理器重启
 

diff --git a/feapder/setting.py b/feapder/setting.py
@@ -96,10 +96,17 @@
 ITEM_FILTER_ENABLE = False  # item 去重
 REQUEST_FILTER_ENABLE = False  # request 去重
 
-# 报警
+# 报警 支持钉钉及邮件，二选一即可
+# 钉钉报警
 DINGDING_WARNING_URL = ""  # 钉钉机器人api
-DINGDING_WARNING_PHONE = ""  # 报警人
-LINGXI_TOKEN = ""  # 灵犀报警token
+DINGDING_WARNING_PHONE = ""  # 报警人 支持列表，可指定多个
+# 邮件报警
+EAMIL_SENDER = ""  # 发件人
+EAMIL_PASSWORD = ""  # 授权码
+EMAIL_RECEIVER = ""  # 收件人 支持列表，可指定多个
+# 时间间隔
+WARNING_INTERVAL = 3600  # 相同报警的报警时间间隔，防止刷屏
+WARNING_LEVEL = "DEBUG"  # 报警级别， DEBUG / ERROR
 
 LOG_NAME = os.path.basename(os.getcwd())
 LOG_PATH = "log/%s.log" % LOG_NAME  # log存储路径

diff --git a/feapder/templates/project_template/setting.py b/feapder/templates/project_template/setting.py
@@ -74,9 +74,17 @@
 # ITEM_FILTER_ENABLE = False  # item 去重
 # REQUEST_FILTER_ENABLE = False  # request 去重
 #
-# # 报警
+# # 报警 支持钉钉及邮件，二选一即可
+# # 钉钉报警
 # DINGDING_WARNING_URL = ""  # 钉钉机器人api
-# DINGDING_WARNING_PHONE = ""  # 报警人
+# DINGDING_WARNING_PHONE = ""  # 报警人 支持列表，可指定多个
+# # 邮件报警
+# EAMIL_SENDER = ""  # 发件人
+# EAMIL_PASSWORD = ""  # 授权码
+# EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个
+# # 报警时间间隔及级别
+# WARNING_INTERVAL = 3600  # 相同报警的报警时间间隔，防止刷屏
+# WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / ERROR
 #
 # LOG_NAME = os.path.basename(os.getcwd())
 # LOG_PATH = "log/%s.log" % LOG_NAME  # log存储路径

diff --git a/feapder/utils/email_sender.py b/feapder/utils/email_sender.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2020/2/19 12:57 PM
+---------
+@summary:
+---------
+@author: Boris
+@email: [email protected]
+"""
+
+import os
+import smtplib
+from email.header import Header
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import formataddr
+
+from utils.log import log
+
+
+class EmailSender(object):
+    SENDER = "feapder报警系统"
+    SMTPSERVER = "smtp.163.com"
+
+    def __init__(self, username, password):
+        self.username = username
+        self.password = password
+
+        self.smtpserver = EmailSender.SMTPSERVER
+        self.sender = EmailSender.SENDER
+
+        self.smtp_client = smtplib.SMTP_SSL()
+
+    def __enter__(self):
+        self.login()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.quit()
+
+    def quit(self):
+        self.smtp_client.quit()
+
+    def login(self):
+        self.smtp_client.connect(self.smtpserver)
+        self.smtp_client.login(self.username, self.password)
+
+    def send(
+        self,
+        receivers: list,
+        title: str,
+        content: str,
+        content_type: str = "plain",
+        filepath: str = None,
+    ):
+        """
+
+        Args:
+            receivers:
+            title:
+            content:
+            content_type: html / plain
+            filepath:
+
+        Returns:
+
+        """
+        # 创建一个带附件的实例
+        message = MIMEMultipart()
+        message["From"] = formataddr(
+            (self.sender, self.username)
+        )  # 括号里的对应发件人邮箱昵称、发件人邮箱账号
+        message["To"] = formataddr((receivers[0], receivers[0]))  # ",".join(receivers)
+
+        message["Subject"] = Header(title, "utf-8")
+
+        content = MIMEText(content, content_type, "utf-8")
+        message.attach(content)
+
+        # 构造附件
+        if filepath:
+            attach = MIMEText(open(filepath, "rb").read(), "base64", "utf-8")
+            attach.add_header(
+                "content-disposition",
+                "attachment",
+                filename=("utf-8", "", os.path.basename(filepath)),
+            )
+            message.attach(attach)
+
+        msg = message.as_string()
+        # 此处直接发送多个邮箱有问题，改成一个个发送
+        for receiver in receivers:
+            log.debug("发送邮件到 {}".format(receiver))
+            self.smtp_client.sendmail(self.username, receiver, msg)
+        log.debug("邮件发送成功！！！")
+        return True
diff --git a/feapder/utils/tools.py b/feapder/utils/tools.py
@@ -43,6 +43,7 @@
 from w3lib.url import canonicalize_url as _canonicalize_url
 
 import feapder.setting as setting
+from feapder.utils.email_sender import EmailSender
 from feapder.utils.log import log
 
 os.environ["EXECJS_RUNTIME"] = "Node"  # 设置使用node执行js
@@ -2197,34 +2198,81 @@ def is_in_rate_limit(rate_limit, *key):
 
 def dingding_warning(
     message,
-    rate_limit=3600,
     message_prefix=None,
+    rate_limit=setting.WARNING_INTERVAL,
     url=setting.DINGDING_WARNING_URL,
     user_phone=setting.DINGDING_WARNING_PHONE,
 ):
-    if not url:
-        log.info("未设置叮叮的地址，不支持报警")
+    if not all([url, user_phone, message]):
         return
 
     if is_in_rate_limit(rate_limit, url, user_phone, message_prefix or message):
         log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message))
         return
 
+    if isinstance(user_phone, str):
+        user_phone = [user_phone]
+
     data = {
         "msgtype": "text",
         "text": {"content": message},
-        "at": {"atMobiles": [user_phone], "isAtAll": False},
+        "at": {"atMobiles": user_phone, "isAtAll": False},
     }
 
     headers = {"Content-Type": "application/json"}
 
-    response = requests.post(url, headers=headers, data=json.dumps(data).encode("utf8"))
-    return response
+    try:
+        response = requests.post(
+            url, headers=headers, data=json.dumps(data).encode("utf8")
+        )
+        result = response.json()
+        response.close()
+        if result.get("errcode") == 0:
+            return True
+        else:
+            raise Exception(result.get("errmsg"))
+    except Exception as e:
+        log.error("报警发送失败。 报警内容 {}, error: {}".format(message, e))
+        return False
 
 
-def linkedsee_warning(
-    message, rate_limit=3600, message_prefix=None, token=setting.LINGXI_TOKEN
+def email_warning(
+    message,
+    title,
+    message_prefix=None,
+    eamil_sender=setting.EAMIL_SENDER,
+    eamil_password=setting.EAMIL_PASSWORD,
+    email_receiver=setting.EMAIL_RECEIVER,
+    rate_limit=setting.WARNING_INTERVAL,
 ):
+    if not all([message, eamil_sender, eamil_password, email_receiver]):
+        return
+
+    if is_in_rate_limit(
+        rate_limit, email_receiver, eamil_sender, message_prefix or message
+    ):
+        log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message))
+        return
+
+    if isinstance(email_receiver, str):
+        email_receiver = [email_receiver]
+
+    with EmailSender(username=eamil_sender, password=eamil_password) as email:
+        return email.send(receivers=email_receiver, title=title, content=message)
+
+
+def linkedsee_warning(message, rate_limit=3600, message_prefix=None, token=None):
+    """
+    灵犀电话报警
+    Args:
+        message:
+        rate_limit:
+        message_prefix:
+        token:
+
+    Returns:
+
+    """
     if not token:
         log.info("未设置灵犀token，不支持报警")
         return