Worker setup

Dockerfile

@@ -0,0 +1,30 @@
+FROM python
+
+# Install dependencies.
+RUN apt-get update && apt-get install -y \
+  openssl \
+  postgresql
+
+# Set env vars for postgres.
+ENV LDFLAGS $(pg_config --ldflags)
+
+# Install pipenv.
+RUN pip install pipenv
+
+# Install our project dependencies.
+# Docs: https://pipenv-fork.readthedocs.io/en/latest/basics.html#pipenv-install
+# --dev — Install both develop and default packages from Pipfile.
+# --system — Use the system pip command rather than the one from your
+#            virtualenv.
+# --deploy — Make sure the packages are properly locked in Pipfile.lock, and
+#            abort if the lock file is out-of-date.
+# --ignore-pipfile — Ignore the Pipfile and install from the Pipfile.lock.
+# --skip-lock — Ignore the Pipfile.lock and install from the Pipfile. In
+#               addition, do not write out a Pipfile.lock reflecting changes to
+#               the Pipfile.
+COPY Pipfile Pipfile
+COPY Pipfile.lock Pipfile.lock
+RUN pipenv install --system --dev
+
+# Setup docker image to run as an executable.
+ENTRYPOINT [ "pipenv", "run" ]

Pipfile

@@ -0,0 +1,21 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pytest = "*"
+ipython = "*"
+pavotebymail = {path = ".",editable = true}
+
+[packages]
+firebase-admin = "*"
+names = "*"
+pyyaml = "*"
+sqlalchemy = "*"
+psycopg2 = "*"
+arrow = "*"
+click = "*"
+
+[requires]
+python_version = "3.7"

README.md

@@ -5,3 +5,45 @@ More than [550,000 Americans had their mail-in ballots rejected](https://www.npr
 **37,119** of those rejected ballots were from Pennsylvania.
 
 The goal of this site is to teach people in PA how to vote successfully by mail so that their vote is actually counted.
+
+## Developing pavotebymail
+
+### Install dependencies
+
+```bash
+$ brew install postgresql openssl
+$ export LDFLAGS=$(pg_config --ldflags)
+$ pipenv install
+```
+
+### Start docker-compose for databases
+
+Before you can import data you'll need to be running the databases
+
+```bash
+$ docker-compose -f docker/dev-compose.yml up
+```
+
+### Run tests
+
+```bash
+pipenv run pytest
+```
+
+### Import database data so you can explore it on your local machine
+
+```bash
+$ pipenv run pavotebymail data-import --import-config phildelphia --postgres-password postgres /path/to/data_to_import.py
+```
+
+### Populate google firestore database
+
+```bash
+# tbd
+```
+
+### Run the worker and fake send postcards
+
+```bash
+# tbd
+```

docker/dev-compose.yml

@@ -0,0 +1,24 @@
+version: '3'
+
+services:
+  db:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: postgres
+    ports:
+      - "5432:5432"
+
+  dynamodb:
+    image: amazon/dynamodb-local
+    restart: always
+    ports:
+      - "9091:8000"
+
+  firestore:
+    image: mtlynch/firestore-emulator
+    environment:
+      - FIRESTORE_PROJECT_ID=dummy-project-id
+      - PORT=8200
+    ports:
+      - "8200:8200"

exporters/docker/data-explorer-docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3'
+
+services:
+  db:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: postgres
+    ports:
+      - "5432:5432"
+
+  dynamodb:
+    image: amazon/dynamodb-local
+    restart: always
+    ports:
+      - "9091:8000"
+
+  firestore:
+    image: mtlynch/firestore-emulator
+    environment:
+      - FIRESTORE_PROJECT_ID=dummy-project-id
+      - PORT=8200
+    ports:
+      - "8200:8200"

pavotebymail/cli.py

@@ -0,0 +1,45 @@
+"""The main CLI for the application/worker
+"""
+import sys
+import logging 
+import click
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from pavotebymail.dataimport import voter_data_import
+from pavotebymail.dataimport.db.db import ensure_db
+
+root = logging.getLogger()
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+
+@click.group()
+@click.option('--log-level', default='info')
+def cli(log_level):
+    root.setLevel(getattr(logging, log_level.upper()))
+
+
+@cli.command()
+@click.argument('voter_data_path', type=click.Path(exists=True, 
+                dir_okay=False, resolve_path=True))
+@click.option('--postgres-host', default='127.0.0.1')
+@click.option('--postgres-port', type=click.INT, default=5432)
+@click.option('--postgres-user', default='postgres')
+@click.option('--postgres-password', default='postgres')
+@click.option('--postgres-db', default='voters')
+@click.option('--import-config', default='philadelphia')
+def data_import(voter_data_path, postgres_host, postgres_port, 
+                      postgres_user, postgres_password, postgres_db,
+                      import_config):
+    ensure_db(postgres_user, postgres_password, postgres_host, postgres_port, postgres_db)
+
+    engine = create_engine('postgresql://%s:%s@%s:%d/%s' % (
+        postgres_user,
+        postgres_password,
+        postgres_host,
+        postgres_port,
+        postgres_db
+    ))
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    voter_data_import(engine, session, voter_data_path, import_config)

pavotebymail/dataimport/__init__.py

@@ -0,0 +1,44 @@
+import os
+import importlib
+import yaml
+from .loader import *
+from .db import *
+from .schemas import *
+from sqlalchemy.orm import session, Session
+from sqlalchemy.engine import Engine
+
+CURR_FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+
+def voter_data_import(engine: Engine, session: Session, voter_data_path: str, config_name: str, commit_size=10000, base_class=Base):
+    FieldParser, schema = load_config(config_name)
+
+    voter_cls = create_class_from_schema('%sVoter' % config_name.capitalize(), config_name, 
+                                         schema, base_class=base_class)
+    base_class.metadata.create_all(engine)
+
+    items = DataLoader.load(schema, voter_data_path, FieldParser)
+
+    count = 0
+
+    for item in items:
+        if count > commit_size:
+            session.commit()
+            count = 0
+        voter = voter_cls(**item)
+        session.add(voter)
+        count += 1
+    session.commit()
+
+
+def load_config(config_name: str) -> (BaseFieldParser, Schema):
+    FieldParser = importlib.import_module('.configs.%s' % config_name, __name__).FieldParser
+
+    # Load the schema
+    schema = load_schema(config_name)
+    return (FieldParser(), schema)
+
+def load_schema(config_name: str) -> Schema:
+    schema_path = os.path.join(CURR_FILE_DIR, 'configs', config_name, 'schema.yml')
+
+    return Schema.load_raw(yaml.safe_load(open(schema_path)))
+

pavotebymail/dataimport/configs/philadelphia/__init__.py

@@ -0,0 +1,7 @@
+from typing import List
+from pavotebymail.dataimport.loader import BaseFieldParser
+
+class FieldParser(BaseFieldParser):
+    def parse_for_fields(self, blob) -> List[str]:
+        split_by_tabs = blob.split('\t')
+        return list(map(lambda a: a.strip('"'), split_by_tabs))