diff --git a/scrapers/scrapers.py b/scrapers/scrapers.py index deb437e..c4f4996 100644 --- a/scrapers/scrapers.py +++ b/scrapers/scrapers.py @@ -1,9 +1,11 @@ -import typer import re -from src import utils + +import typer + from src import craigslist as cl -from src import facebook as fb from src import database as db +from src import facebook as fb +from src import utils app = typer.Typer() diff --git a/scrapers/src/craigslist.py b/scrapers/src/craigslist.py index 64ee544..d4b3358 100644 --- a/scrapers/src/craigslist.py +++ b/scrapers/src/craigslist.py @@ -1,7 +1,10 @@ -from bs4 import BeautifulSoup import time + +from bs4 import BeautifulSoup + from . import utils + def loadPageResources(driver): scroll = 100 diff --git a/scrapers/src/database.py b/scrapers/src/database.py index 5cbd3a2..8143c2a 100644 --- a/scrapers/src/database.py +++ b/scrapers/src/database.py @@ -1,8 +1,9 @@ -from dotenv import load_dotenv -import pymongo import os from datetime import date +import pymongo +from dotenv import load_dotenv + db = "scrape" collection = "scraped_raw" diff --git a/scrapers/src/facebook.py b/scrapers/src/facebook.py index c40694e..31752d8 100644 --- a/scrapers/src/facebook.py +++ b/scrapers/src/facebook.py @@ -1,5 +1,7 @@ -from bs4 import BeautifulSoup import time + +from bs4 import BeautifulSoup + from . import utils postClass = "x9f619 x78zum5 x1r8uery xdt5ytf x1iyjqo2 xs83m0k x1e558r4 x150jy0e x1iorvi4 xjkvuk6 xnpuxes x291uyu x1uepa24" diff --git a/scrapers/src/utils.py b/scrapers/src/utils.py index 8651c74..17dbaef 100644 --- a/scrapers/src/utils.py +++ b/scrapers/src/utils.py @@ -1,9 +1,11 @@ from selenium import webdriver from selenium.webdriver.chrome.options import Options -from . import database as db + from . import craigslist +from . import database as db from . import facebook + def scrollTo(x, driver): driver.execute_script(f"window.scrollTo({{top: {x}, left: 100, behavior: 'smooth'}})")