6/craigslist scraper

.gitignore

@@ -130,4 +130,7 @@ dist
 .pnp.*
 
 # misc
-*.DS_STORE
+*.DS_STORE
+
+# python
+*.pyc

README.md

@@ -5,4 +5,37 @@ Senior Design Repository for the Statefarm Automotive Fraud Project
 Make a copy of the ``.env.example`` file and make the following changes.
 1. remove ``.example`` from the extension
 2. Paste the username and password provided in MongoDB Atlas (if you should have access but do not, please contact @waseem-polus)
-3. Paste the connection URL provided provided in MongoDB Atlas. Include the password and username fields using ``${VARIABLE}`` syntax to embed the value of the variable
+3. Paste the connection URL provided provided in MongoDB Atlas. Include the password and username fields using ``${VARIABLE}`` syntax to embed the value of the variable
+
+## Run Scrapers locally
+**Prerequisites**
+- python3
+- pipenv
+
+**Installing dependencies**  
+Navigate to ``scrapers/`` and open the virtual environment using
+```bash
+pipenv shell
+```
+Then install dependencies using
+```bash
+pipenv install
+```
+
+**Scraper Usage**  
+To create build a Docker Image use
+```bash
+pipenv run build
+```
+to run a docker container "smarecontainer" use
+```bash
+pipenv run cont
+```
+then
+```bash
+# Scrape Craigsist homepage
+pipenv run craigslist
+
+# Scrape Facebook Marketplace homepage
+pipenv run facebook
+```

scrapers/Dockerfile

@@ -0,0 +1,22 @@
+FROM public.ecr.aws/lambda/python@sha256:f0c3116a56d167eba8021a5d7c595f969835fbe78826303326f80de00d044733 as build
+RUN yum install -y unzip && \
+    curl -Lo "/tmp/chromedriver-linux64.zip" "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/119.0.6045.105/linux64/chromedriver-linux64.zip" && \
+    curl -Lo "/tmp/chrome-linux64.zip" "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/119.0.6045.105/linux64/chrome-linux64.zip" && \
+    unzip /tmp/chromedriver-linux64.zip -d /opt/ && \
+    unzip /tmp/chrome-linux64.zip -d /opt/
+
+FROM public.ecr.aws/lambda/python@sha256:f0c3116a56d167eba8021a5d7c595f969835fbe78826303326f80de00d044733
+RUN yum install atk cups-libs gtk3 libXcomposite alsa-lib \
+    libXcursor libXdamage libXext libXi libXrandr libXScrnSaver \
+    libXtst pango at-spi2-atk libXt xorg-x11-server-Xvfb \
+    xorg-x11-xauth dbus-glib dbus-glib-devel -y
+COPY --from=build /opt/chrome-linux64 /opt/chrome
+COPY --from=build /opt/chromedriver-linux64 /opt/
+
+COPY scrapers.py ./
+COPY src ./src
+COPY requirements.txt ./
+
+RUN pip install -r requirements.txt
+
+CMD [ "scrapers.craigslist" ]

scrapers/Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[scripts]
+build = "docker build --platform linux/amd64 -t smare ."
+cont = "docker run --name smarecontainer -d smare:latest"
+exec = "docker exec -it smarecontainer"
+craigslist = "pipenv run exec python3 -c 'import scrapers; scrapers.craigslist(\"\",\"\")'"
+facebook = "pipenv run exec python3 -c 'import scrapers; scrapers.facebook(\"\",\"\")'"
+
+[packages]
+selenium = "*"
+bs4 = "*"
+pymongo = "*"
+typer = "*"
+python-dotenv = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"