Create multiCleaner.py

multiCleaner.py

@@ -0,0 +1,63 @@
+import os
+import pymongo
+import gridfs
+import shutil
+from bson.objectid import ObjectId
+from datetime import datetime, timedelta
+from multiprocessing import Process
+
+# db.calls.createIndex({"pid": 1})
+
+MAX_MALSCORE = 4
+MAX_RESULTS = 100
+MY_DIR = "/home/analyst/cuckoo-advanced/storage/analyses/"
+DEBUG = False 
+
+db = pymongo.MongoClient("1.2.3.4", 27017)["cuckoo"]
+
+now = datetime.now()
+delta = timedelta(seconds=-14400)
+refDateTime = now+delta
+
+def countSamples():
+    return db.analysis.count( {"$and" : [{"malscore": {"$gt" : -1}}, {"malscore": {"$lt" : MAX_MALSCORE}}], "info.ended": {"$lt" : refDateTime.strftime("%Y-%m-%d %H:%M:%S")}, "virustotal_summary": {"$exists": False } })
+
+def removeSamples(childnum, sample_list):
+    import urllib2
+    import httplib
+    pid = os.getpid()
+    #print "starting new child %d with pid: %s" % (childnum, pid)
+    for obId in sample_list:
+            try:
+              #print ("http://127.0.0.1/analysis/remove/%s/" % analysis["info"]["id"])
+              output=urllib2.urlopen("http://127.0.0.1/analysis/remove/%s/" % obId).read()
+            except httplib.IncompleteRead, e:
+              output=e.partial
+            ## Try to remove tree; if failed show an error using try...except on screen
+            try:
+            #print  MY_DIR + ("%s"  % analysis["info"]["id"]) 
+              curdir = "%s%s" % (MY_DIR, obId)
+              print curdir
+              shutil.rmtree(curdir)
+              if DEBUG: print "deleted: %s" % obId
+            except OSError, e:
+              print e
+
+    if DEBUG: print "child %d with pid: %s ended" % (childnum, pid)
+
+if __name__ == '__main__':
+    print "%s starting" % datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    fs = gridfs.GridFS(db)
+
+    sample_count = countSamples()
+
+    print "found %s samples" % sample_count
+
+    if sample_count > 0:
+        for x in range(0, (sample_count/MAX_RESULTS) +1 ):
+            sample_list = db.analysis.find( {"$and" : [{"malscore": {"$gt" : -1}}, {"malscore": {"$lt" : MAX_MALSCORE}}], "info.ended": {"$lt" : refDateTime.strftime("%Y-%m-%d %H:%M:%S")}, "virustotal_summary": {"$exists": False } } , {"info.id": 1} ).sort([("info.ended", 1)]).limit( MAX_RESULTS ).skip( MAX_RESULTS * x )
+            toChild = []
+            for sample in sample_list:
+                toChild.append(sample["info"]["id"])
+            p = Process(target=removeSamples, args=(x+1, toChild,))
+            p.start()