Merge pull request #96 from radiolover/master

Release 0.5.0

CHANGES.md

@@ -2,6 +2,13 @@
 
 The list of the most significant changes made in Netis Packet Agent over time.
 
+
+## Netis Packet Agent 0.5.0
+
+### Features
+* Support GRE direction in key.
+
+
 ## Netis Packet Agent 0.3.6
 
 ### Features

CMakeLists.txt

@@ -14,8 +14,8 @@ endif ()
 
 # set PKTMINERG_MAJOR_VERSION, PKTMINERG_MINOR_VERSION, etc.
 set(PKTMINERG_MAJOR_VERSION "0")
-set(PKTMINERG_MINOR_VERSION "3")
-set(PKTMINERG_PATCH_VERSION "6")
+set(PKTMINERG_MINOR_VERSION "5")
+set(PKTMINERG_PATCH_VERSION "0")
 set(PKTMINERG_VERSION_STRING "${PKTMINERG_MAJOR_VERSION}.${PKTMINERG_MINOR_VERSION}.${PKTMINERG_PATCH_VERSION}")
 
 if(WIN32)

INSTALL.md

@@ -12,8 +12,8 @@ Note: The default libpcap with TPACKET_V3 enabled has some performance issue. If
 2. Download and install the RPM package. Find the latest package from [Releases Page](https://github.com/Netis/packet-agent/releases).
 
 ```shell
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm
-rpm -ivh netis-packet-agent-0.3.6.el6.x86_64.rpm
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm
+rpm -ivh netis-packet-agent-0.5.0.el6.x86_64.rpm
 ```
 
 
@@ -26,8 +26,8 @@ sudo apt-get install libpcap-dev wget
 
 2. Download and install the DEB package. Find the latest package from [Releases Page](https://github.com/Netis/packet-agent/releases).
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6_amd64.deb
-sudo dpkg -i netis-packet-agent-0.3.6_amd64.deb
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0_amd64.deb
+sudo dpkg -i netis-packet-agent-0.5.0_amd64.deb
 ```
 
 3. If libpcap.so.1 not found when running pktminerg, create softlink for libpcap.so.1 in suitable directory.
@@ -49,8 +49,8 @@ yum install libpcap wget
 2. Download and install the RPM package. Find the latest package from [Releases Page](https://github.com/Netis/packet-agent/releases).
 
 ```shell
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm
-rpm -ivh netis-packet-agent-0.3.6.el6.x86_64.rpm
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm
+rpm -ivh netis-packet-agent-0.5.0.el6.x86_64.rpm
 ```
 
 

LICENSE.md

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2018, Netis Technologies Co., Ltd. All rights reserved.
+Copyright (c) 2018-2021, Netis Technologies Co., Ltd. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 

README-zh-Hans.md

@@ -2,9 +2,9 @@
 [English](README.md)  ∙  简体中文
 
 ![packet agent's title](./img/title.jpg)
-# Netis Packet Agent 0.3.6
+# Netis Packet Agent 0.5.0
 
-[![Stable release](https://img.shields.io/badge/version-0.3.6-green.svg)](https://github.com/Netis/packet-agent/releases/tag/0.3.6)
+[![Stable release](https://img.shields.io/badge/version-0.5.0-green.svg)](https://github.com/Netis/packet-agent/releases/tag/0.5.0)
 [![Software License](https://img.shields.io/badge/license-BSD3-green.svg)](./LICENSE.md)
 
 
@@ -35,14 +35,14 @@ yum install libpcap wget
 
 2. 下载并安装RPM包。您可以从[这个地址](https://github.com/Netis/packet-agent/releases)获取最新版本的软件包。
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm
-rpm -ivh netis-packet-agent-0.3.6.el6.x86_64.rpm
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm
+rpm -ivh netis-packet-agent-0.5.0.el6.x86_64.rpm
 ```
 #### SUSE12
 1. 下载并安装RPM包。您可以从[这个地址](https://github.com/Netis/packet-agent/releases)获取最新版本的软件包。
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm
-rpm -ivh netis-packet-agent-0.3.6.el6.x86_64.rpm
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm
+rpm -ivh netis-packet-agent-0.5.0.el6.x86_64.rpm
 ```
 
 #### Ubuntu 18.04LTS
@@ -53,8 +53,8 @@ sudo apt-get install libpcap-dev wget
 
 2. 下载并安装DEB包。您可以从[这个地址](https://github.com/Netis/packet-agent/releases)获取最新版本的软件包。
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6_amd64.deb
-sudo dpkg -i netis-packet-agent-0.3.6_amd64.deb
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0_amd64.deb
+sudo dpkg -i netis-packet-agent-0.5.0_amd64.deb
 ```
 
 3. 如果提示libpcap.so找不到，到libpcap.so所在目录下创建libpcap.so.1软链接。

README.md

@@ -2,9 +2,9 @@
 English  ∙  [简体中文](README-zh-Hans.md) 
 
 ![packet agent's title](./img/title.jpg)
-# Netis Packet Agent 0.3.6
+# Netis Packet Agent 0.5.0
 
-[![Stable release](https://img.shields.io/badge/version-0.3.6-green.svg)](https://github.com/Netis/packet-agent/releases/tag/0.3.6)
+[![Stable release](https://img.shields.io/badge/version-0.5.0-green.svg)](https://github.com/Netis/packet-agent/releases/tag/0.5.0)
 [![Software License](https://img.shields.io/badge/license-BSD3-green.svg)](./LICENSE.md)
 
 ## What is Netis Packet Agent?
@@ -35,15 +35,15 @@ yum install libpcap wget zeromq
 
 2. Download and install the RPM package. Find the latest package from [Releases Page](https://github.com/Netis/packet-agent/releases).
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm
-rpm -ivh netis-packet-agent-0.3.6.el6.x86_64.rpm
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm
+rpm -ivh netis-packet-agent-0.5.0.el6.x86_64.rpm
 ```
 
 #### SUSE 12
 1. Download and install the RPM package. Find the latest package from [Releases Page](https://github.com/Netis/packet-agent/releases).
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm
-rpm -ivh netis-packet-agent-0.3.6.el6.x86_64.rpm
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm
+rpm -ivh netis-packet-agent-0.5.0.el6.x86_64.rpm
 ```
 
 
@@ -55,8 +55,8 @@ sudo apt-get install libpcap-dev wget
 
 2. Download and install the DEB package. Find the latest package from [Releases Page](https://github.com/Netis/packet-agent/releases).
 ```bash
-wget https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6_amd64.deb
-sudo dpkg -i netis-packet-agent-0.3.6_amd64.deb
+wget https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0_amd64.deb
+sudo dpkg -i netis-packet-agent-0.5.0_amd64.deb
 ```
 
 3. If libpcap.so.1 not found when running pktminerg, create softlink for libpcap.so.1 in suitable directory.

USAGE.md

@@ -64,7 +64,7 @@ Send GRE packets from this binded device. Sending will be failed when this devic
 * remoteip, keybit<br>
 Parameters of GRE channel:
 remoteip：GRE channel remote IP addresss (required)
-keybit：GRE protocol keybit parameter to distinguish the channel to remote IP
+keybit：GRE protocol keybit parameter to distinguish the channel to remote IP, high 4 bit is reserved for direction.
 <br>
 
 * zmq_port, zmq_hwm<br>

scripts/ansible_pktg.md

@@ -1,6 +1,6 @@
 # 说明
 1. 在主控的中心服务器上需要安装有 ansible 程序。如果没有，可以使用类似 yum install ansible 的命令安装。</br>
 2. 将需要安装 packet-agent 的机器加入 ansible 的 hosts 文件：比如添加组 [servers_to_install_pktg] 到 /etc/ansible/hosts，并在该组下加入若干机器IP。</br>
-3. 拷贝待安装的 packet-agent rpm/deb 到 ansible_pktg.yaml 同一目录下。如果待安装的 rpm/deb 不是 yaml 中指定的 netis-packet-agent-0.3.6.el6.x86_64.rpm / netis-packet-agent-0.3.6_amd64.deb，则将 ansible_pktg.yaml 中 rpm_file/deb_file 修改为你需要的文件名。</br>
+3. 拷贝待安装的 packet-agent rpm/deb 到 ansible_pktg.yaml 同一目录下。如果待安装的 rpm/deb 不是 yaml 中指定的 netis-packet-agent-0.5.0.el6.x86_64.rpm / netis-packet-agent-0.5.0_amd64.deb，则将 ansible_pktg.yaml 中 rpm_file/deb_file 修改为你需要的文件名。</br>
 4. 执行 ansible-playbook ansible_pktg.yaml 命令，将 packet-agent 程序安装到各个目标机器上。</br>
 

scripts/ansible_pktg.yaml

@@ -2,8 +2,8 @@
 - name: install packet-agent and its depends
   hosts: servers_to_install_pktg
   vars:
-    rpm_file: netis-packet-agent-0.3.6.el6.x86_64.rpm
-    deb_file: netis-packet-agent-0.3.6_amd64.deb
+    rpm_file: netis-packet-agent-0.5.0.el6.x86_64.rpm
+    deb_file: netis-packet-agent-0.5.0_amd64.deb
   remote_user: root
   gather_facts: True
 

scripts/puppet_packet_agent.pp

@@ -3,30 +3,30 @@
   'Suse':     {
                   $lib_pcap = 'libpcap';
                   $cur_dir = '/tmp/';
-                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm';
+                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm';
                   $package_provider = 'rpm'; 
-                  $install_file = '/tmp/netis-packet-agent-0.3.6.el6.x86_64.rpm'
+                  $install_file = '/tmp/netis-packet-agent-0.5.0.el6.x86_64.rpm'
               }
   'RedHat':   { 
                   $lib_pcap = 'libpcap';
                   $cur_dir = '/tmp/';
-                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm';
+                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm';
                   $package_provider = 'rpm'; 
-                  $install_file = '/tmp/netis-packet-agent-0.3.6.el6.x86_64.rpm'
+                  $install_file = '/tmp/netis-packet-agent-0.5.0.el6.x86_64.rpm'
               }
   'Debian':   {
                   $lib_pcap = 'libpcap-dev';
                   $cur_dir = '/tmp/';
-                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6_amd64.deb';
+                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0_amd64.deb';
                   $package_provider = 'dpkg'; 
-                  $install_file = '/tmp/netis-packet-agent-0.3.6_amd64.deb' 
+                  $install_file = '/tmp/netis-packet-agent-0.5.0_amd64.deb' 
               }
   default:    {
                   $lib_pcap = 'libpcap';
                   $cur_dir = '/tmp/';
-                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.3.6/netis-packet-agent-0.3.6.el6.x86_64.rpm';
+                  $download_link = 'https://github.com/Netis/packet-agent/releases/download/v0.5.0/netis-packet-agent-0.5.0.el6.x86_64.rpm';
                   $package_provider = 'rpm'; 
-                  $install_file = '/tmp/netis-packet-agent-0.3.6.el6.x86_64.rpm'
+                  $install_file = '/tmp/netis-packet-agent-0.5.0.el6.x86_64.rpm'
               }
 }
 

src/pcapexport.h

@@ -3,6 +3,11 @@
 
 #include <pcap/pcap.h>
 
+#define PKTD_UNKNOWN -1
+#define PKTD_IC 1
+#define PKTD_OG 2
+#define PKTD_NONCHECK 0
+
 enum class exporttype : uint8_t {
     gre = 0,
     file = 1,
@@ -17,7 +22,7 @@ class PcapExportBase {
         return _type;
     }
     virtual int initExport() = 0;
-    virtual int exportPacket(const struct pcap_pkthdr *header, const uint8_t *pkt_data) = 0;
+    virtual int exportPacket(const struct pcap_pkthdr *header, const uint8_t *pkt_data, int direct) = 0;
     virtual int closeExport() = 0;
 };
 

src/pcaphandler.cpp

@@ -1,7 +1,14 @@
-#include "pcaphandler.h"
 #include <iostream>
 #include <algorithm>
 #include <boost/filesystem.hpp>
+#include <boost/algorithm/string.hpp>
+#include <netinet/ether.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <ifaddrs.h>
+#include <arpa/inet.h>
+
+#include "pcaphandler.h"
 #include "scopeguard.h"
 #include "agent_status.h"
 
@@ -49,12 +56,34 @@ void PcapHandler::closePcap() {
 }
 
 void PcapHandler::packetHandler(const struct pcap_pkthdr* header, const uint8_t* pkt_data) {
+    ether_header* eth;
+    iphdr* ip;
+    ip6_hdr* ipv6;
+    uint16_t eth_type;
+    int direct;
+
+    if(header->caplen < sizeof(ether_header))
+        return;
+    eth = (ether_header*)pkt_data;
+    eth_type = ntohs(eth->ether_type);
+    direct = PKTD_UNKNOWN;
+
+    switch(eth_type) {
+        case ETHERTYPE_IP:
+            ip = (iphdr*)(pkt_data + sizeof(ether_header));
+            direct = checkPktDirectionV4((const in_addr*)&ip->saddr, (const in_addr*)&ip->daddr);
+            break;
+        case ETHERTYPE_IPV6:
+            ipv6 = (ip6_hdr*)(pkt_data + sizeof(ether_header));
+            direct = checkPktDirectionV6(&ipv6->ip6_src, &ipv6->ip6_dst);
+            break;
+        default:
+            break;
+    }
+
     std::for_each(_exports.begin(), _exports.end(),
-                  [header, pkt_data, this](std::shared_ptr<PcapExportBase> pcapExport) {
-//        if (header->caplen > 1472) {
-//            std::cout << "pkt " << _gre_count << ", len: " << header->len << ", caplen: " << header->caplen << std::endl;
-//        }
-                      int ret = pcapExport->exportPacket(header, pkt_data);
+                  [header, pkt_data, this, direct](std::shared_ptr<PcapExportBase> pcapExport) {
+                      int ret = pcapExport->exportPacket(header, pkt_data, direct);
                       if (pcapExport->getExportType() == exporttype::gre) {
                           if (ret == 0) {
                               this->_gre_count++;
@@ -73,8 +102,8 @@ void PcapHandler::packetHandler(const struct pcap_pkthdr* header, const uint8_t*
     _statislog->logSendStatis((uint64_t) (header->ts.tv_sec), header->caplen, _gre_count, _gre_drop_count, 0,
                               _pcap_handle);
     if (_need_update_status) {
-        AgentStatus::get_instance()->update_capture_status((uint64_t) (header->ts.tv_sec), header->caplen, 
-                              _gre_count, _gre_drop_count, _pcap_handle);
+        AgentStatus::get_instance()->update_capture_status((uint64_t) (header->ts.tv_sec), header->caplen,
+                                                           _gre_count, _gre_drop_count, _pcap_handle);
     }
 }
 
@@ -108,6 +137,34 @@ void PcapHandler::stopPcapLoop() {
     pcap_breakloop(_pcap_handle);
 }
 
+int PcapHandler::checkPktDirectionV4(const in_addr *sip, const in_addr *dip) {
+    for(auto& ipv4 : _ipv4s)
+    {
+        if(ipv4.s_addr == sip->s_addr)
+            return PKTD_OG;
+        else if(ipv4.s_addr == dip->s_addr)
+            return PKTD_IC;
+    }
+    return PKTD_UNKNOWN;
+}
+
+int PcapHandler::checkPktDirectionV6(const in6_addr *sip, const in6_addr *dip) {
+    for(auto& ipv6 : _ipv6s)
+    {
+        if(ipv6.s6_addr32[0] == sip->s6_addr32[0] &&
+           ipv6.s6_addr32[1] == sip->s6_addr32[1] &&
+           ipv6.s6_addr32[2] == sip->s6_addr32[2] &&
+           ipv6.s6_addr32[3] == sip->s6_addr32[3])
+            return PKTD_OG;
+        else if(ipv6.s6_addr32[0] == dip->s6_addr32[0] &&
+                ipv6.s6_addr32[1] == dip->s6_addr32[1] &&
+                ipv6.s6_addr32[2] == dip->s6_addr32[2] &&
+                ipv6.s6_addr32[3] == dip->s6_addr32[3])
+            return PKTD_IC;
+    }
+    return PKTD_UNKNOWN;
+}
+
 int PcapOfflineHandler::openPcap(const std::string& dev, const pcap_init_t& param, const std::string& expression,
                                  bool dumpfile) {
     pcap_t* pcap_handle = pcap_open_offline(dev.c_str(), _errbuf);
@@ -140,6 +197,29 @@ int PcapLiveHandler::openPcap(const std::string& dev, const pcap_init_t& param,
     bpf_u_int32 net = 0;
     _need_update_status = param.need_update_status;
 
+    {
+        struct ifaddrs* ifaddr;
+
+        _ipv4s.clear();
+        _ipv6s.clear();
+        if (::getifaddrs(&ifaddr) < 0) {
+            return -1;
+        }
+
+        for (auto ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
+            if(!ifa->ifa_name || dev != ifa->ifa_name || !ifa->ifa_addr)
+                continue;
+            if(ifa->ifa_addr->sa_family == AF_INET)
+            {
+                _ipv4s.push_back(((sockaddr_in*)ifa->ifa_addr)->sin_addr);
+            }
+            else if(ifa->ifa_addr->sa_family == AF_INET6)
+            {
+                _ipv6s.push_back(((sockaddr_in6*)ifa->ifa_addr)->sin6_addr);
+            }
+        }
+        freeifaddrs(ifaddr);
+    }
     pcap_t* pcap_handle = pcap_create(dev.c_str(), _errbuf);
     if (!pcap_handle) {
         std::cerr << StatisLogContext::getTimeString() << "Call pcap_create failed, error is " << _errbuf << "."

src/pcaphandler.h

@@ -4,6 +4,8 @@
 #include <string>
 #include <vector>
 #include <memory>
+#include <netinet/in.h>
+
 #include "pcapexport.h"
 #include "statislog.h"
 
@@ -25,9 +27,16 @@ class PcapHandler {
     uint64_t _gre_count;
     uint64_t _gre_drop_count;
     int _need_update_status;
+
+    std::vector<in_addr> _ipv4s;
+    std::vector<in6_addr> _ipv6s;
+
 protected:
     int openPcapDumper(pcap_t *pcap_handle);
     void closePcapDumper();
+
+    int checkPktDirectionV4(const in_addr* sip, const in_addr* dip);
+    int checkPktDirectionV6(const in6_addr* sip, const in6_addr* dip);
 public:
     PcapHandler();
     virtual ~PcapHandler();