lookingGlassCyber_scraper.py

import websocket
import json
import urllib.request
from datetime import datetime
 
def getUTCTimestamp():
    return datetime.utcnow()
 
def doDump():
    prevDate = getUTCTimestamp().date()
    f = open('lookingGlassCyber_dump_' + str(prevDate) + '.csv', 'a')
    m = urllib.request.urlopen("https://map.lookingglasscyber.com/socket.io/?EIO=3&transport=polling").read()
    sid = str(m).split('{')[1].split('"')[3]
    ws = websocket.create_connection("wss://map.lookingglasscyber.com/socket.io/?EIO=3&transport=websocket&sid=%s" %sid)
    ws.send('2probe')
    ws.send('5')
     
    count = 0
    try:
        while ws.connected:
            try:
                if count == 20:
                    ws.send('2')
                    ws.recv()
                    count = 0

                result =  ws.recv()                
                dump = json.loads(result[2:])
                dump2 = json.loads(dump[1])
                timestamp = getUTCTimestamp()
                # botnet_name , latitude , longitude , variant (may be null) , asn (may be null and may contain commas - so replaced by ;) , 
                # organization (may be null and may contain commas - so replaced by ;) , city (may be null) , country code , timestamp (UTC)
                data_string = (str(dump2['botnet']) + " , " + str(dump2['location']['geo']['latitude']) + " , " + str(dump2['location']['geo']['longitude']) +
                    " , " + str(dump2['variant']) + " , " + str(dump2['asn']).replace(',',';') + " , " + str(dump2['organization']).replace(',',';') + " , " + 
                    str(dump2['city']) + " , " + str(dump2['countrycode']) + " , " + str(getUTCTimestamp()))
                if timestamp.date() > prevDate:
                    f.close()
                    f = open('lookingGlassCyber_dump_' + str(timestamp.date()) + '.csv', 'a')
                    prevDate = timestamp.date()
                f.write(data_string + "\n")
                print(data_string)
                count += 1
            except Exception as e:
                pass
    finally:
        print('Closing file...')
        f.close() 
                 
if __name__ == "__main__":
    while True:
        try:
            doDump()
        except:
            pass