Merge branch 'development' of https://github.com/mmguero-dev/Malcolm …

…into v24.07.0_merge_cisagov

filebeat/scripts/clean-processed-folder.py

@@ -104,16 +104,22 @@ def pruneFiles():
         return
 
     # look for regular Zeek files in the processed/ directory
-    zeekFoundFiles = [
-        (os.path.join(root, filename))
-        for root, dirnames, filenames in os.walk(zeekProcessedDir)
-        for filename in filenames
-    ]
+    zeekFoundFiles = (
+        [
+            (os.path.join(root, filename))
+            for root, dirnames, filenames in os.walk(zeekProcessedDir)
+            for filename in filenames
+        ]
+        if os.path.isdir(zeekProcessedDir)
+        else []
+    )
 
     # look for rotated files from live zeek instance
-    zeekRotatedFiles = [
-        (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(zeekLiveDir) for filename in filenames
-    ]
+    zeekRotatedFiles = (
+        [(os.path.join(root, filename)) for root, dirnames, filenames in os.walk(zeekLiveDir) for filename in filenames]
+        if os.path.isdir(zeekLiveDir)
+        else []
+    )
 
     # look up the filebeat registry file and try to read it
     fbReg = None
@@ -128,18 +134,20 @@ def pruneFiles():
         checkFile(file, filebeatReg=None, checkLogs=False, checkArchives=True)
 
     # clean up any broken symlinks in the Zeek current/ directory
-    for current in os.listdir(zeekCurrentDir):
-        currentFileSpec = os.path.join(zeekCurrentDir, current)
-        if os.path.islink(currentFileSpec) and not os.path.exists(currentFileSpec):
-            print(f'removing dead symlink "{currentFileSpec}"')
-            silentRemove(currentFileSpec)
+    if os.path.isdir(zeekCurrentDir):
+        for current in os.listdir(zeekCurrentDir):
+            currentFileSpec = os.path.join(zeekCurrentDir, current)
+            if os.path.islink(currentFileSpec) and not os.path.exists(currentFileSpec):
+                print(f'removing dead symlink "{currentFileSpec}"')
+                silentRemove(currentFileSpec)
 
     # clean up any old and empty directories in Zeek processed/ directory
     cleanDirSeconds = min(i for i in (cleanLogSeconds, cleanZipSeconds) if i > 0)
     candidateDirs = []
-    for root, dirs, files in os.walk(zeekProcessedDir, topdown=False):
-        if root and dirs:
-            candidateDirs += [os.path.join(root, tmpDir) for tmpDir in dirs]
+    if os.path.isdir(zeekProcessedDir):
+        for root, dirs, files in os.walk(zeekProcessedDir, topdown=False):
+            if root and dirs:
+                candidateDirs += [os.path.join(root, tmpDir) for tmpDir in dirs]
     candidateDirs = list(set(candidateDirs))
     candidateDirs.sort(reverse=True)
     candidateDirs.sort(key=len, reverse=True)
@@ -155,10 +163,11 @@ def pruneFiles():
 
     # check the suricata logs (live and otherwise) as well
     for surDir in [suricataDir, suricataLiveDir]:
-        for eve in os.listdir(surDir):
-            eveFile = os.path.join(surDir, eve)
-            if os.path.isfile(eveFile):
-                checkFile(eveFile, filebeatReg=fbReg, checkLogs=True, checkArchives=False)
+        if os.path.isdir(surDir):
+            for eve in os.listdir(surDir):
+                eveFile = os.path.join(surDir, eve)
+                if os.path.isfile(eveFile):
+                    checkFile(eveFile, filebeatReg=fbReg, checkLogs=True, checkArchives=False)
 
 
 def main():

scripts/malcolm_utils.py

@@ -352,6 +352,23 @@ def isipaddress(value):
     return result
 
 
+###################################################################################################
+# check a string or list to see if something is a private IP address or subnet
+def isprivateip(value):
+    result = True
+    try:
+        if isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set):
+            for v in value:
+                result = result and ipaddress.ip_network(value).is_private
+                if not result:
+                    break
+        else:
+            result = ipaddress.ip_network(value).is_private
+    except Exception:
+        result = False
+    return result
+
+
 ###################################################################################################
 # return the primary IP (the one with a default route) on the local box
 def get_primary_ip():

shared/bin/zeek_threat_feed_utils.py

@@ -37,7 +37,7 @@
 import requests
 import urllib3
 
-from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson
+from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson, isprivateip
 
 # keys for dict returned by map_stix_indicator_to_zeek for Zeek intel file fields
 ZEEK_INTEL_INDICATOR = 'indicator'
@@ -316,9 +316,14 @@ def map_stix_indicator_to_zeek(
             parsed = urlparse(ioc_value)
             scheme = f"{parsed.scheme}://"
             ioc_value = parsed.geturl().replace(scheme, "", 1)
-        elif zeek_type == "ADDR" and re.match(".+/.+", ioc_value):
-            # elevate to subnet if possible
-            zeek_type = "SUBNET"
+        elif zeek_type == "ADDR":
+            if not isprivateip(ioc_value):
+                if re.match(".+/.+", ioc_value):
+                    # elevate to subnet if possible
+                    zeek_type = "SUBNET"
+            else:
+                # ignore private IP-space ADDR avlues
+                continue
 
         # ... "fields containing only a hyphen are considered to be null values"
         zeekItem = defaultdict(lambda: '-')
@@ -395,9 +400,14 @@ def map_misp_attribute_to_zeek(
             parsed = urlparse(attribute_value)
             scheme = f"{parsed.scheme}://"
             attribute_value = parsed.geturl().replace(scheme, "", 1)
-        elif zeek_type == "ADDR" and re.match(".+/.+", attribute_value):
-            # elevate to subnet if possible
-            zeek_type = "SUBNET"
+        elif zeek_type == "ADDR":
+            if not isprivateip(attribute_value):
+                if re.match(".+/.+", attribute_value):
+                    # elevate to subnet if possible
+                    zeek_type = "SUBNET"
+            else:
+                # ignore private IP-space ADDR avlues
+                continue
 
         # ... "fields containing only a hyphen are considered to be null values"
         zeekItem = defaultdict(lambda: '-')