From 506cb7de96b142aeb647d92f51a681749e8a22d5 Mon Sep 17 00:00:00 2001 From: Reuven V Gonzales Date: Fri, 4 Sep 2020 03:38:55 -0700 Subject: [PATCH 1/8] Lots of prototypical work --- Pipfile | 14 + Pipfile.lock | 458 ++++++++++++++++++ .../docker/data-explorer-docker-compose.yml | 24 + exporters/pennsylvania/schema.sql | 0 worker/__init__.py | 0 worker/db.py | 69 +++ worker/fixtures.py | 31 ++ worker/model.py | 56 +++ worker/report.py | 24 + worker/worker.py | 22 + 10 files changed, 698 insertions(+) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 exporters/docker/data-explorer-docker-compose.yml create mode 100644 exporters/pennsylvania/schema.sql create mode 100644 worker/__init__.py create mode 100644 worker/db.py create mode 100644 worker/fixtures.py create mode 100644 worker/model.py create mode 100644 worker/report.py create mode 100644 worker/worker.py diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..3124f9c --- /dev/null +++ b/Pipfile @@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +firebase-admin = "*" +ipython = "*" +names = "*" + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..0fd1dfa --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,458 @@ +{ + "_meta": { + "hash": { + "sha256": "3b1f48d2ca770b1b03ce43cd3e1db46a057b90d12b32f6eea071852cc69f1c62" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "appnope": { + "hashes": [ + "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", + "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.0" + }, + "backcall": { + "hashes": [ + "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", + "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255" + ], + "version": "==0.2.0" + }, + "cachecontrol": { + "hashes": [ + "sha256:10d056fa27f8563a271b345207402a6dcce8efab7e5b377e270329c62471b10d", + "sha256:be9aa45477a134aee56c8fac518627e1154df063e85f67d4f83ce0ccc23688e8" + ], + "version": "==0.12.6" + }, + "cachetools": { + "hashes": [ + "sha256:513d4ff98dd27f85743a8dc0e92f55ddb1b49e060c2d5961512855cda2c01a98", + "sha256:bbaa39c3dede00175df2dc2b03d0cf18dd2d32a7de7beb68072d13043c9edb20" + ], + "version": "==4.1.1" + }, + "certifi": { + "hashes": [ + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + ], + "version": "==2020.6.20" + }, + "cffi": { + "hashes": [ + "sha256:0da50dcbccd7cb7e6c741ab7912b2eff48e85af217d72b57f80ebc616257125e", + "sha256:12a453e03124069b6896107ee133ae3ab04c624bb10683e1ed1c1663df17c13c", + "sha256:15419020b0e812b40d96ec9d369b2bc8109cc3295eac6e013d3261343580cc7e", + "sha256:15a5f59a4808f82d8ec7364cbace851df591c2d43bc76bcbe5c4543a7ddd1bf1", + "sha256:23e44937d7695c27c66a54d793dd4b45889a81b35c0751ba91040fe825ec59c4", + "sha256:29c4688ace466a365b85a51dcc5e3c853c1d283f293dfcc12f7a77e498f160d2", + "sha256:57214fa5430399dffd54f4be37b56fe22cedb2b98862550d43cc085fb698dc2c", + "sha256:577791f948d34d569acb2d1add5831731c59d5a0c50a6d9f629ae1cefd9ca4a0", + "sha256:6539314d84c4d36f28d73adc1b45e9f4ee2a89cdc7e5d2b0a6dbacba31906798", + "sha256:65867d63f0fd1b500fa343d7798fa64e9e681b594e0a07dc934c13e76ee28fb1", + "sha256:672b539db20fef6b03d6f7a14b5825d57c98e4026401fce838849f8de73fe4d4", + "sha256:6843db0343e12e3f52cc58430ad559d850a53684f5b352540ca3f1bc56df0731", + "sha256:7057613efefd36cacabbdbcef010e0a9c20a88fc07eb3e616019ea1692fa5df4", + "sha256:76ada88d62eb24de7051c5157a1a78fd853cca9b91c0713c2e973e4196271d0c", + "sha256:837398c2ec00228679513802e3744d1e8e3cb1204aa6ad408b6aff081e99a487", + "sha256:8662aabfeab00cea149a3d1c2999b0731e70c6b5bac596d95d13f643e76d3d4e", + "sha256:95e9094162fa712f18b4f60896e34b621df99147c2cee216cfa8f022294e8e9f", + "sha256:99cc66b33c418cd579c0f03b77b94263c305c389cb0c6972dac420f24b3bf123", + "sha256:9b219511d8b64d3fa14261963933be34028ea0e57455baf6781fe399c2c3206c", + "sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b", + "sha256:b9aa9d8818c2e917fa2c105ad538e222a5bce59777133840b93134022a7ce650", + "sha256:bf44a9a0141a082e89c90e8d785b212a872db793a0080c20f6ae6e2a0ebf82ad", + "sha256:c0b48b98d79cf795b0916c57bebbc6d16bb43b9fc9b8c9f57f4cf05881904c75", + "sha256:da9d3c506f43e220336433dffe643fbfa40096d408cb9b7f2477892f369d5f82", + "sha256:e4082d832e36e7f9b2278bc774886ca8207346b99f278e54c9de4834f17232f7", + "sha256:e4b9b7af398c32e408c00eb4e0d33ced2f9121fd9fb978e6c1b57edd014a7d15", + "sha256:e613514a82539fc48291d01933951a13ae93b6b444a88782480be32245ed4afa", + "sha256:f5033952def24172e60493b68717792e3aebb387a8d186c43c020d9363ee7281" + ], + "version": "==1.14.2" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "decorator": { + "hashes": [ + "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", + "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" + ], + "version": "==4.4.2" + }, + "firebase-admin": { + "hashes": [ + "sha256:23deaede741f6c4785939c1b7e1385561431887d8b065202f27372d8ea73cf01", + "sha256:7cf4ce710fa06e69308b58e7d55d5f2aebff559f07272578186c5ef3c3461730" + ], + "index": "pypi", + "version": "==4.3.0" + }, + "google-api-core": { + "extras": [ + "grpc" + ], + "hashes": [ + "sha256:67e33a852dcca7cb7eff49abc35c8cc2c0bb8ab11397dc8306d911505cae2990", + "sha256:779107f17e0fef8169c5239d56a8fbff03f9f72a3893c0c9e5842ec29dfedd54" + ], + "markers": "platform_python_implementation != 'PyPy'", + "version": "==1.22.2" + }, + "google-api-python-client": { + "hashes": [ + "sha256:4f596894f702736da84cf89490a810b55ca02a81f0cddeacb3022e2900b11ec6", + "sha256:caf4015800ef1a18d06d117f47f0219c0c0641f21978f6b1bb5ede7912fab97b" + ], + "version": "==1.11.0" + }, + "google-auth": { + "hashes": [ + "sha256:bcbd9f970e7144fe933908aa286d7a12c44b7deb6d78a76871f0377a29d09789", + "sha256:f4d5093f13b1b1c0a434ab1dc851cd26a983f86a4d75c95239974e33ed406a87" + ], + "version": "==1.21.1" + }, + "google-auth-httplib2": { + "hashes": [ + "sha256:8d092cc60fb16517b12057ec0bba9185a96e3b7169d86ae12eae98e645b7bc39", + "sha256:aeaff501738b289717fac1980db9711d77908a6c227f60e4aa1923410b43e2ee" + ], + "version": "==0.0.4" + }, + "google-cloud-core": { + "hashes": [ + "sha256:4c9e457fcfc026fdde2e492228f04417d4c717fb0f29f070122fb0ab89e34ebd", + "sha256:613e56f164b6bee487dd34f606083a0130f66f42f7b10f99730afdf1630df507" + ], + "version": "==1.4.1" + }, + "google-cloud-firestore": { + "hashes": [ + "sha256:2b2985180591433f9343b5b4cafb9b0dbe077ced95b3ac5c57ef850a0339a4ce", + "sha256:d8a56919a3a32c7271d1253542ec24cb13f384a726fed354fdeb2a2269f25d1c" + ], + "markers": "platform_python_implementation != 'PyPy'", + "version": "==1.9.0" + }, + "google-cloud-storage": { + "hashes": [ + "sha256:34fb8f7e8a2a633cbfb09d8dec38b3450c4029af1a328a67bca64f6226a1f4a5", + "sha256:4f51c7700242a9d54c07117f25fec5d110ab85435b3ce60ac28cc553f8ea938b" + ], + "version": "==1.31.0" + }, + "google-crc32c": { + "hashes": [ + "sha256:00b34d4c9ac565b2be553f81f58e5861e51d43af2043ed7cbfe1853ee2f54671", + "sha256:17223ac9135eab28e874ff1e221810190d109a1abd482451d0776dc388be14de", + "sha256:176cef33c9ad2a56977efd084646b378e50ab14b43a7c0a16e956bc3e3ec130a", + "sha256:1a613f43534c9a345cc86fc6531bda477e2473cb876b6e26aee22b8060917069", + "sha256:337566ce49d7ea7493f95bd6bc89ab08640caa91b6105cea0be57ed026980e74", + "sha256:41fb6c22cd72ae3db4d98d28dbb768d53397c8fc3cb8ab945fd434e842e622d4", + "sha256:438d6c314a52d50a9523460024e655a3d27774adde47d72eebccc89dc9eec992", + "sha256:6fd5d861421c37786b9c1a87dc7b0d8349a426151a461d5724b76c5a07f6ae9b", + "sha256:7b5ccdc7697ca54351d2965d4241f907d53f26f5288710bed505f8c3776ed235", + "sha256:7f44c5259f6b2f8b2b6f668dbaa954693a10e97811345c193e46b933c2dd5165", + "sha256:9439b960b6ecd847557675d130fc3626d762bf535da595c20a6949a705fb3eae", + "sha256:b6fad0842a02abd270f8b660db082d37d197ab80aa4db6a2ddbfcf472eade9e7", + "sha256:b7ee33659231c8205bb05559781ac61a325f31b06b917b3e997bea5c2c49ff4d", + "sha256:cda3a6829e8b5bf6058615e53387430d004590c9b0ad808e53fea5bec35bbe44", + "sha256:cf373207380e54c42da6c88baf1f7a31c2d9f29b87c9c922d5147d219eed55aa", + "sha256:ec4d91c9236b0576d9d2b23c7eb85c6a6372b88afe2d0c64681cf11629586f74", + "sha256:f3b859200c3bc73925b1719ed8b1f6d8d73b6620b42dbc121c4df58423045e34", + "sha256:f54c90058e3f56e55fa0f699c6f4ceaaa825ea7f17ef2adbf07b2b06b27455e7" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.0" + }, + "google-resumable-media": { + "hashes": [ + "sha256:173acc6bade1480a529fa29c6c2717543ae2dc09d42e9461fdb86f39502efcf2", + "sha256:99b5ac33a75ddb25d5e6aad487b37ecb4fa18b1fbf3d1ad726e032c3d6fc9aff" + ], + "version": "==1.0.0" + }, + "googleapis-common-protos": { + "hashes": [ + "sha256:560716c807117394da12cecb0a54da5a451b5cf9866f1d37e9a5e2329a665351", + "sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24" + ], + "version": "==1.52.0" + }, + "grpcio": { + "hashes": [ + "sha256:013287f99c99b201aa8a5f6bc7918f616739b9be031db132d9e3b8453e95e151", + "sha256:0397616355760cd8282ed5ea34d51830ae4cb6613b7e5f66bed3be5d041b8b9a", + "sha256:074871a184483d5cd0746fd01e7d214d3ee9d36e67e32a5786b0a21f29fb8304", + "sha256:08a9b648dbe8852ff94b73a1c96da126834c3057ba2301d13e8c4adff334c482", + "sha256:0fa86ac4452602c79774783aa68979a1a7625ebb7eaabee2b6550b975b9d61e6", + "sha256:220c46b1fc9c9a6fcca4caac398f08f0ed43cdd63c45b7458983c4a1575ef6df", + "sha256:259240aab2603891553e17ad5b2655693df79e02a9b887ff605bdeb2fcd3dcc9", + "sha256:292635f05b6ce33f87116951d0b3d8d330bdfc5cac74f739370d60981e8c256c", + "sha256:344b50865914cc8e6d023457bffee9a640abb18f75d0f2bb519041961c748da9", + "sha256:3c2aa6d7a5e5bf73fdb1715eee777efe06dd39df03383f1cc095b2fdb34883e6", + "sha256:43d44548ad6ee738b941abd9f09e3b83a5c13f3e1410321023c3c148ba50e796", + "sha256:5043440c45c0a031f387e7f48527541c65d672005fb24cf18ef6857483557d39", + "sha256:58d7121f48cb94535a4cedcce32921d0d0a78563c7372a143dedeec196d1c637", + "sha256:5d7faa89992e015d245750ca9ac916c161bbf72777b2c60abc61da3fae41339e", + "sha256:5fb0923b16590bac338e92d98c7d8effb3cfad1d2e18c71bf86bde32c49cd6dd", + "sha256:63ee8e02d04272c3d103f44b4bce5d43ea757dd288673cea212d2f7da27967d2", + "sha256:64077e3a9a7cf2f59e6c76d503c8de1f18a76428f41a5b000dc53c48a0b772ff", + "sha256:739a72abffbd36083ff7adbb862cf1afc1e311c35834bed9c0361d8e68b063e1", + "sha256:75e383053dccb610590aa53eed5278db5c09bf498d3b5105ce6c776478f59352", + "sha256:7a11b1ebb3210f34913b8be6995936bf9ebc541a65ab69e75db5ce1fe5047e8f", + "sha256:8002a89ea91c0078c15d3c0daf423fd4968946be78f08545e807ea9a5ff8054a", + "sha256:8b42f0ac76be07a5fa31117a3388d754ad35ef05e2e34be185ca9ccbcfac2069", + "sha256:8ca26b489b5dc1e3d31807d329c23d6cb06fe40fbae25b0649b718947936e26a", + "sha256:92e54ab65e782f227e751c7555918afaba8d1229601687e89b80c2b65d2f6642", + "sha256:a9a7ae74cb3108e6457cf15532d4c300324b48fbcf3ef290bcd2835745f20510", + "sha256:ba3e43cb984399064ffaa3c0997576e46a1e268f9da05f97cd9b272f0b59ee71", + "sha256:baaa036540d7ace433bdf38a3fe5e41cf9f84cdf10a88bac805f678a7ca8ddcc", + "sha256:bf00ab06ea4f89976288f4d6224d4aa120780e30c955d4f85c3214ada29b3ddf", + "sha256:bf39977282a79dc1b2765cc3402c0ada571c29a491caec6ed12c0993c1ec115e", + "sha256:c22b19abba63562a5a200e586b5bde39d26c8ec30c92e26d209d81182371693b", + "sha256:c9016ab1eaf4e054099303287195f3746bd4e69f2631d040f9dca43e910a5408", + "sha256:d2c5e05c257859febd03f5d81b5015e1946d6bcf475c7bf63ee99cea8ab0d590", + "sha256:e64bddd09842ef508d72ca354319b0eb126205d951e8ac3128fe9869bd563552", + "sha256:e8c3264b0fd728aadf3f0324471843f65bd3b38872bdab2a477e31ffb685dd5b", + "sha256:ea849210e7362559f326cbe603d5b8d8bb1e556e86a7393b5a8847057de5b084", + "sha256:ebb2ca09fa17537e35508a29dcb05575d4d9401138a68e83d1c605d65e8a1770", + "sha256:ef9fce98b6fe03874c2a6576b02aec1a0df25742cd67d1d7b75a49e30aa74225", + "sha256:f04c59d186af3157dc8811114130aaeae92e90a65283733f41de94eed484e1f7", + "sha256:f5b0870b733bcb7b6bf05a02035e7aaf20f599d3802b390282d4c2309f825f1d" + ], + "version": "==1.31.0" + }, + "httplib2": { + "hashes": [ + "sha256:8af66c1c52c7ffe1aa5dc4bcd7c769885254b0756e6e69f953c7f0ab49a70ba3", + "sha256:ca2914b015b6247791c4866782fa6042f495b94401a0f0bd3e1d6e0ba2236782" + ], + "version": "==0.18.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "version": "==2.10" + }, + "ipython": { + "hashes": [ + "sha256:2e22c1f74477b5106a6fb301c342ab8c64bb75d702e350f05a649e8cb40a0fb8", + "sha256:a331e78086001931de9424940699691ad49dfb457cea31f5471eae7b78222d5e" + ], + "index": "pypi", + "version": "==7.18.1" + }, + "ipython-genutils": { + "hashes": [ + "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", + "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" + ], + "version": "==0.2.0" + }, + "jedi": { + "hashes": [ + "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20", + "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5" + ], + "version": "==0.17.2" + }, + "msgpack": { + "hashes": [ + "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408", + "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8", + "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84", + "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d", + "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a", + "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322", + "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2", + "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e", + "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97", + "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0", + "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be", + "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf", + "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab", + "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08", + "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e", + "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272", + "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1", + "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140" + ], + "version": "==1.0.0" + }, + "names": { + "hashes": [ + "sha256:726e46254f2ed03f1ffb5d941dae3bc67c35123941c29becd02d48d0caa2a671" + ], + "index": "pypi", + "version": "==0.3.0" + }, + "parso": { + "hashes": [ + "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea", + "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9" + ], + "version": "==0.7.1" + }, + "pexpect": { + "hashes": [ + "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", + "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" + ], + "markers": "sys_platform != 'win32'", + "version": "==4.8.0" + }, + "pickleshare": { + "hashes": [ + "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", + "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" + ], + "version": "==0.7.5" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489", + "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950" + ], + "version": "==3.0.7" + }, + "protobuf": { + "hashes": [ + "sha256:0bba42f439bf45c0f600c3c5993666fcb88e8441d011fad80a11df6f324eef33", + "sha256:1e834076dfef9e585815757a2c7e4560c7ccc5962b9d09f831214c693a91b463", + "sha256:339c3a003e3c797bc84499fa32e0aac83c768e67b3de4a5d7a5a9aa3b0da634c", + "sha256:361acd76f0ad38c6e38f14d08775514fbd241316cce08deb2ce914c7dfa1184a", + "sha256:3dee442884a18c16d023e52e32dd34a8930a889e511af493f6dc7d4d9bf12e4f", + "sha256:4d1174c9ed303070ad59553f435846a2f877598f59f9afc1b89757bdf846f2a7", + "sha256:5db9d3e12b6ede5e601b8d8684a7f9d90581882925c96acf8495957b4f1b204b", + "sha256:6a82e0c8bb2bf58f606040cc5814e07715b2094caeba281e2e7d0b0e2e397db5", + "sha256:8c35bcbed1c0d29b127c886790e9d37e845ffc2725cc1db4bd06d70f4e8359f4", + "sha256:91c2d897da84c62816e2f473ece60ebfeab024a16c1751aaf31100127ccd93ec", + "sha256:9c2e63c1743cba12737169c447374fab3dfeb18111a460a8c1a000e35836b18c", + "sha256:9edfdc679a3669988ec55a989ff62449f670dfa7018df6ad7f04e8dbacb10630", + "sha256:c0c5ab9c4b1eac0a9b838f1e46038c3175a95b0f2d944385884af72876bd6bc7", + "sha256:c8abd7605185836f6f11f97b21200f8a864f9cb078a193fe3c9e235711d3ff1e", + "sha256:d69697acac76d9f250ab745b46c725edf3e98ac24763990b24d58c16c642947a", + "sha256:df3932e1834a64b46ebc262e951cd82c3cf0fa936a154f0a42231140d8237060", + "sha256:e7662437ca1e0c51b93cadb988f9b353fa6b8013c0385d63a70c8a77d84da5f9", + "sha256:f68eb9d03c7d84bd01c790948320b768de8559761897763731294e3bc316decb" + ], + "version": "==3.13.0" + }, + "ptyprocess": { + "hashes": [ + "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", + "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" + ], + "version": "==0.6.0" + }, + "pyasn1": { + "hashes": [ + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" + ], + "version": "==0.4.8" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", + "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74" + ], + "version": "==0.2.8" + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "version": "==2.20" + }, + "pygments": { + "hashes": [ + "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", + "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" + ], + "version": "==2.6.1" + }, + "pytz": { + "hashes": [ + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + ], + "version": "==2020.1" + }, + "requests": { + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "version": "==2.24.0" + }, + "rsa": { + "hashes": [ + "sha256:109ea5a66744dd859bf16fe904b8d8b627adafb9408753161e766a92e7d681fa", + "sha256:6166864e23d6b5195a5cfed6cd9fed0fe774e226d8f854fcb23b7bbef0350233" + ], + "markers": "python_version >= '3.5'", + "version": "==4.6" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "version": "==1.15.0" + }, + "traitlets": { + "hashes": [ + "sha256:8bdadb17a04c844f444cdefaa3dee47a12ff14cf6277b9eeda29bfa0659d5987", + "sha256:a2e91709a0330b6c5d497ed470b2feb1ed8da5c9dd807c6daab41f727b9391c9" + ], + "version": "==5.0.3" + }, + "uritemplate": { + "hashes": [ + "sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f", + "sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae" + ], + "version": "==3.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", + "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" + ], + "version": "==1.25.10" + }, + "wcwidth": { + "hashes": [ + "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", + "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" + ], + "version": "==0.2.5" + } + }, + "develop": {} +} diff --git a/exporters/docker/data-explorer-docker-compose.yml b/exporters/docker/data-explorer-docker-compose.yml new file mode 100644 index 0000000..49ebb29 --- /dev/null +++ b/exporters/docker/data-explorer-docker-compose.yml @@ -0,0 +1,24 @@ +version: '3' + +services: + db: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: example + ports: + - "5432:5432" + + dynamodb: + image: amazon/dynamodb-local + restart: always + ports: + - "9091:8000" + + firestore: + image: mtlynch/firestore-emulator + environment: + - FIRESTORE_PROJECT_ID=dummy-project-id + - PORT=8200 + ports: + - "8200:8200" diff --git a/exporters/pennsylvania/schema.sql b/exporters/pennsylvania/schema.sql new file mode 100644 index 0000000..e69de29 diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/worker/db.py b/worker/db.py new file mode 100644 index 0000000..2ca9be2 --- /dev/null +++ b/worker/db.py @@ -0,0 +1,69 @@ +from typing import List +import firebase_admin +from firebase_admin import firestore +from model import Voter + +class VoterDB(object): + @classmethod + def connect(cls, collection_name) -> 'VoterDB': + db = firestore.client() + + return cls(db, collection_name) + + def __init__(self, db, collection_name): + self._collection = db.collection(collection_name) + self._db = db + + def add_voters(self, voters: List[Voter]): + batch = self._db.batch() + + for voter in voters: + voter_doc_ref = self._collection.document(voter.id) + batch.set(voter_doc_ref, voter.to_dict()) + + batch.commit() + + def all_voters_sent_postcards(self) -> List[Voter]: + return self._voters_from_collection_stream( + self._collection.where('was_sent_postcard', '==', True).order_by('priority').stream() + ) + + def all_voters(self, limit=None) -> List[Voter]: + collection = self._collection + if limit: + collection = collection.limit(limit) + return self._voters_from_collection_stream( + collection.order_by('priority').stream(), + ) + + def choose_random_voters(self, limit: int) -> List[Voter]: + # Choose voters based on their priority value + # TODO handle errors + return self._voters_from_collection_stream( + self._collection.where('was_sent_postcard', '==', False).order_by('priority').limit(limit).stream() + ) + + def _voters_from_collection_stream(self, stream) -> List[Voter]: + voters = [] + for voter_doc in stream: + voter_doc_dict = voter_doc.to_dict() + voters.append(Voter( + voter_doc.id, + voter_doc_dict['priority'], + voter_doc_dict['first_name'], + voter_doc_dict['last_name'], + voter_doc_dict['address'], + voter_doc_dict['was_sent_postcard'], + )) + return voters + + + def confirm_postcard_sent_to_voter(self, voter) -> Voter: + # TODO handle errors + voter_doc_ref = self._collection.document(voter.id) + voter_doc_dict = voter_doc_ref.get().to_dict() + + voter_doc_dict['was_sent_postcard'] = True + voter_doc_ref.set(voter_doc_dict) + + return Voter.from_dict(voter_doc_dict) diff --git a/worker/fixtures.py b/worker/fixtures.py new file mode 100644 index 0000000..8878bd5 --- /dev/null +++ b/worker/fixtures.py @@ -0,0 +1,31 @@ +import random +import firebase_admin +import names +import uuid +from model import Voter +from db import VoterDB + +def main(): + firebase_admin.initialize_app() + + # Create a bunch of voters + voters = [] + for i in range(10): + voter = Voter( + str(uuid.uuid4()), + random.randint(10,1000), + names.get_first_name(), + names.get_last_name(), + "1234 %s St, Philadelphia, PA 01919" % names.get_first_name(), + False, + ) + print(voter) + voters.append(voter) + + voter_db = VoterDB.connect('voters') + + voter_db.add_voters(voters) + + +if __name__ == '__main__': + main() diff --git a/worker/model.py b/worker/model.py new file mode 100644 index 0000000..e2f961f --- /dev/null +++ b/worker/model.py @@ -0,0 +1,56 @@ +class Voter(object): + @classmethod + def from_dict(cls, d: dict): + return cls( + d['id'], + d['priority'], + d['first_name'], + d['last_name'], + d['address'], + d['was_sent_postcard'], + ) + + def __init__(self, id: str, priority: int, first_name: str, last_name: str, address: str, was_sent_postcard: bool): + self._id = id + self._priority = priority + self._first_name = first_name + self._last_name = last_name + self._address = address + self._was_sent_postcard = was_sent_postcard + + def to_dict(self) -> dict: + return dict( + id=self.id, + priority=self._priority, + first_name=self.first_name, + last_name=self.last_name, + address=self.address, + was_sent_postcard=self.was_sent_postcard, + ) + + @property + def id(self) -> str: + return self._id + + @property + def first_name(self) -> str: + return self._first_name + + @property + def last_name(self) -> str: + return self._last_name + + @property + def address(self) -> str: + return self._address + + @property + def priority(self) -> int: + return self._priority + + @property + def was_sent_postcard(self) -> str: + return self._was_sent_postcard + + def __repr__(self): + return 'Voter<%s, %s, %d>' % (self.last_name, self.first_name, self._priority) diff --git a/worker/report.py b/worker/report.py new file mode 100644 index 0000000..1982752 --- /dev/null +++ b/worker/report.py @@ -0,0 +1,24 @@ +import firebase_admin +from firebase_admin import firestore + +from db import VoterDB + + +def main(): + firebase_admin.initialize_app() + + voter_db = VoterDB.connect('voters') + + print("All voters:") + for voter in voter_db.all_voters(): + print(voter) + + print("") + + print("All voters sent postcards:") + for voter in voter_db.all_voters_sent_postcards(): + print(voter) + + +if __name__ == '__main__': + main() diff --git a/worker/worker.py b/worker/worker.py new file mode 100644 index 0000000..4bb8343 --- /dev/null +++ b/worker/worker.py @@ -0,0 +1,22 @@ +from typing import List +import firebase_admin +from firebase_admin import firestore + +from db import VoterDB + + +def main(): + # This assumes that your GOOGLE_APPLICATION_CREDENTIALS env var is set at this time + firebase_admin.initialize_app() + db = firestore.client() + + voter_db = VoterDB.connect('voters') + + voters = voter_db.choose_random_voters(2) + + for voter in voters: + voter_db.confirm_postcard_sent_to_voter(voter) + + +if __name__ == '__main__': + main() From 5e64576721df0523d8486b7cf6ee55278d96e1f3 Mon Sep 17 00:00:00 2001 From: Reuven V Gonzales Date: Sun, 6 Sep 2020 03:07:41 -0700 Subject: [PATCH 2/8] Database import working --- Pipfile | 9 +- Pipfile.lock | 356 +++++++++++++----- README.md | 36 ++ docker/dev-compose.yml | 24 ++ {worker => pavotebymail}/__init__.py | 0 pavotebymail/cli.py | 45 +++ pavotebymail/dataimport/__init__.py | 44 +++ pavotebymail/dataimport/configs/__init__.py | 0 .../configs/philadelphia/__init__.py | 7 + .../configs/philadelphia/schema.yml | 154 ++++++++ pavotebymail/dataimport/db/__init__.py | 2 + pavotebymail/dataimport/db/base.py | 2 + pavotebymail/dataimport/db/db.py | 27 ++ pavotebymail/dataimport/db/tables.py | 29 ++ pavotebymail/dataimport/loader/__init__.py | 1 + pavotebymail/dataimport/loader/base.py | 61 +++ pavotebymail/dataimport/schemas/__init__.py | 32 ++ {worker => pavotebymail}/db.py | 0 {worker => pavotebymail}/fixtures.py | 0 pavotebymail/models/__init__.py | 1 + .../model.py => pavotebymail/models/voter.py | 0 {worker => pavotebymail}/report.py | 0 {worker => pavotebymail}/worker.py | 0 setup.py | 11 + tests/__init__.py | 0 tests/fixtures/philly_schema_test.txt | 2 + tests/test_db.py | 75 ++++ tests/test_loader.py | 58 +++ tests/tools.py | 6 + 29 files changed, 897 insertions(+), 85 deletions(-) create mode 100644 docker/dev-compose.yml rename {worker => pavotebymail}/__init__.py (100%) create mode 100644 pavotebymail/cli.py create mode 100644 pavotebymail/dataimport/__init__.py create mode 100644 pavotebymail/dataimport/configs/__init__.py create mode 100644 pavotebymail/dataimport/configs/philadelphia/__init__.py create mode 100644 pavotebymail/dataimport/configs/philadelphia/schema.yml create mode 100644 pavotebymail/dataimport/db/__init__.py create mode 100644 pavotebymail/dataimport/db/base.py create mode 100644 pavotebymail/dataimport/db/db.py create mode 100644 pavotebymail/dataimport/db/tables.py create mode 100644 pavotebymail/dataimport/loader/__init__.py create mode 100644 pavotebymail/dataimport/loader/base.py create mode 100644 pavotebymail/dataimport/schemas/__init__.py rename {worker => pavotebymail}/db.py (100%) rename {worker => pavotebymail}/fixtures.py (100%) create mode 100644 pavotebymail/models/__init__.py rename worker/model.py => pavotebymail/models/voter.py (100%) rename {worker => pavotebymail}/report.py (100%) rename {worker => pavotebymail}/worker.py (100%) create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/philly_schema_test.txt create mode 100644 tests/test_db.py create mode 100644 tests/test_loader.py create mode 100644 tests/tools.py diff --git a/Pipfile b/Pipfile index 3124f9c..0aa997d 100644 --- a/Pipfile +++ b/Pipfile @@ -4,11 +4,18 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +pytest = "*" +ipython = "*" +pavotebymail = {path = ".",editable = true} [packages] firebase-admin = "*" -ipython = "*" names = "*" +pyyaml = "*" +sqlalchemy = "*" +psycopg2 = "*" +arrow = "*" +click = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 0fd1dfa..c426f4b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3b1f48d2ca770b1b03ce43cd3e1db46a057b90d12b32f6eea071852cc69f1c62" + "sha256": "885bee6710d32e3b9b6046f43310342f31ca57349996f63b5c6a190500d4ca48" }, "pipfile-spec": 6, "requires": { @@ -16,20 +16,13 @@ ] }, "default": { - "appnope": { + "arrow": { "hashes": [ - "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", - "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + "sha256:92aac856ea5175c804f7ccb96aca4d714d936f1c867ba59d747a8096ec30e90a", + "sha256:98184d8dd3e5d30b96c2df4596526f7de679ccb467f358b82b0f686436f3a6b8" ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.0" - }, - "backcall": { - "hashes": [ - "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", - "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255" - ], - "version": "==0.2.0" + "index": "pypi", + "version": "==0.16.0" }, "cachecontrol": { "hashes": [ @@ -92,12 +85,13 @@ ], "version": "==3.0.4" }, - "decorator": { + "click": { "hashes": [ - "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", - "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "version": "==4.4.2" + "index": "pypi", + "version": "==7.1.2" }, "firebase-admin": { "hashes": [ @@ -257,28 +251,6 @@ ], "version": "==2.10" }, - "ipython": { - "hashes": [ - "sha256:2e22c1f74477b5106a6fb301c342ab8c64bb75d702e350f05a649e8cb40a0fb8", - "sha256:a331e78086001931de9424940699691ad49dfb457cea31f5471eae7b78222d5e" - ], - "index": "pypi", - "version": "==7.18.1" - }, - "ipython-genutils": { - "hashes": [ - "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", - "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" - ], - "version": "==0.2.0" - }, - "jedi": { - "hashes": [ - "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20", - "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5" - ], - "version": "==0.17.2" - }, "msgpack": { "hashes": [ "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408", @@ -309,35 +281,6 @@ "index": "pypi", "version": "==0.3.0" }, - "parso": { - "hashes": [ - "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea", - "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9" - ], - "version": "==0.7.1" - }, - "pexpect": { - "hashes": [ - "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", - "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" - ], - "markers": "sys_platform != 'win32'", - "version": "==4.8.0" - }, - "pickleshare": { - "hashes": [ - "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", - "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" - ], - "version": "==0.7.5" - }, - "prompt-toolkit": { - "hashes": [ - "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489", - "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950" - ], - "version": "==3.0.7" - }, "protobuf": { "hashes": [ "sha256:0bba42f439bf45c0f600c3c5993666fcb88e8441d011fad80a11df6f324eef33", @@ -361,12 +304,24 @@ ], "version": "==3.13.0" }, - "ptyprocess": { - "hashes": [ - "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", - "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" + "psycopg2": { + "hashes": [ + "sha256:132efc7ee46a763e68a815f4d26223d9c679953cd190f1f218187cb60decf535", + "sha256:2327bf42c1744a434ed8ed0bbaa9168cac7ee5a22a9001f6fc85c33b8a4a14b7", + "sha256:27c633f2d5db0fc27b51f1b08f410715b59fa3802987aec91aeb8f562724e95c", + "sha256:2c0afb40cfb4d53487ee2ebe128649028c9a78d2476d14a67781e45dc287f080", + "sha256:2df2bf1b87305bd95eb3ac666ee1f00a9c83d10927b8144e8e39644218f4cf81", + "sha256:440a3ea2c955e89321a138eb7582aa1d22fe286c7d65e26a2c5411af0a88ae72", + "sha256:6a471d4d2a6f14c97a882e8d3124869bc623f3df6177eefe02994ea41fd45b52", + "sha256:6b306dae53ec7f4f67a10942cf8ac85de930ea90e9903e2df4001f69b7833f7e", + "sha256:a0984ff49e176062fcdc8a5a2a670c9bb1704a2f69548bce8f8a7bad41c661bf", + "sha256:ac5b23d0199c012ad91ed1bbb971b7666da651c6371529b1be8cbe2a7bf3c3a9", + "sha256:acf56d564e443e3dea152efe972b1434058244298a94348fc518d6dd6a9fb0bb", + "sha256:d3b29d717d39d3580efd760a9a46a7418408acebbb784717c90d708c9ed5f055", + "sha256:f7d46240f7a1ae1dd95aab38bd74f7428d46531f69219954266d669da60c0818" ], - "version": "==0.6.0" + "index": "pypi", + "version": "==2.8.5" }, "pyasn1": { "hashes": [ @@ -389,12 +344,12 @@ ], "version": "==2.20" }, - "pygments": { + "python-dateutil": { "hashes": [ - "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", - "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==2.6.1" + "version": "==2.8.1" }, "pytz": { "hashes": [ @@ -403,6 +358,23 @@ ], "version": "==2020.1" }, + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "index": "pypi", + "version": "==5.3.1" + }, "requests": { "hashes": [ "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", @@ -425,12 +397,43 @@ ], "version": "==1.15.0" }, - "traitlets": { - "hashes": [ - "sha256:8bdadb17a04c844f444cdefaa3dee47a12ff14cf6277b9eeda29bfa0659d5987", - "sha256:a2e91709a0330b6c5d497ed470b2feb1ed8da5c9dd807c6daab41f727b9391c9" + "sqlalchemy": { + "hashes": [ + "sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb", + "sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804", + "sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6", + "sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0", + "sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe", + "sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de", + "sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36", + "sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e", + "sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66", + "sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6", + "sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc", + "sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d", + "sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce", + "sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea", + "sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f", + "sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365", + "sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea", + "sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23", + "sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338", + "sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1", + "sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b", + "sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e", + "sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba", + "sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02", + "sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12", + "sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86", + "sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d", + "sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7", + "sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0", + "sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac", + "sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc", + "sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37" ], - "version": "==5.0.3" + "index": "pypi", + "version": "==1.3.19" }, "uritemplate": { "hashes": [ @@ -445,6 +448,185 @@ "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], "version": "==1.25.10" + } + }, + "develop": { + "appnope": { + "hashes": [ + "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", + "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.0" + }, + "attrs": { + "hashes": [ + "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", + "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + ], + "version": "==20.2.0" + }, + "backcall": { + "hashes": [ + "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", + "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255" + ], + "version": "==0.2.0" + }, + "decorator": { + "hashes": [ + "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", + "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" + ], + "version": "==4.4.2" + }, + "importlib-metadata": { + "hashes": [ + "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83", + "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070" + ], + "markers": "python_version < '3.8'", + "version": "==1.7.0" + }, + "iniconfig": { + "hashes": [ + "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437", + "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69" + ], + "version": "==1.0.1" + }, + "ipython": { + "hashes": [ + "sha256:2e22c1f74477b5106a6fb301c342ab8c64bb75d702e350f05a649e8cb40a0fb8", + "sha256:a331e78086001931de9424940699691ad49dfb457cea31f5471eae7b78222d5e" + ], + "index": "pypi", + "version": "==7.18.1" + }, + "ipython-genutils": { + "hashes": [ + "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", + "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" + ], + "version": "==0.2.0" + }, + "jedi": { + "hashes": [ + "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20", + "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5" + ], + "version": "==0.17.2" + }, + "more-itertools": { + "hashes": [ + "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", + "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" + ], + "version": "==8.5.0" + }, + "packaging": { + "hashes": [ + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + ], + "version": "==20.4" + }, + "parso": { + "hashes": [ + "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea", + "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9" + ], + "version": "==0.7.1" + }, + "pavotebymail": { + "editable": true, + "path": "." + }, + "pexpect": { + "hashes": [ + "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", + "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" + ], + "markers": "sys_platform != 'win32'", + "version": "==4.8.0" + }, + "pickleshare": { + "hashes": [ + "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", + "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" + ], + "version": "==0.7.5" + }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "version": "==0.13.1" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489", + "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950" + ], + "version": "==3.0.7" + }, + "ptyprocess": { + "hashes": [ + "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", + "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" + ], + "version": "==0.6.0" + }, + "py": { + "hashes": [ + "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", + "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" + ], + "version": "==1.9.0" + }, + "pygments": { + "hashes": [ + "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", + "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" + ], + "version": "==2.6.1" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "version": "==2.4.7" + }, + "pytest": { + "hashes": [ + "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4", + "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad" + ], + "index": "pypi", + "version": "==6.0.1" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "version": "==1.15.0" + }, + "toml": { + "hashes": [ + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" + ], + "version": "==0.10.1" + }, + "traitlets": { + "hashes": [ + "sha256:8bdadb17a04c844f444cdefaa3dee47a12ff14cf6277b9eeda29bfa0659d5987", + "sha256:a2e91709a0330b6c5d497ed470b2feb1ed8da5c9dd807c6daab41f727b9391c9" + ], + "version": "==5.0.3" }, "wcwidth": { "hashes": [ @@ -452,7 +634,13 @@ "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" ], "version": "==0.2.5" + }, + "zipp": { + "hashes": [ + "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", + "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" + ], + "version": "==3.1.0" } - }, - "develop": {} + } } diff --git a/README.md b/README.md index a0efe01..7451b6d 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,39 @@ More than [550,000 Americans had their mail-in ballots rejected](https://www.npr **37,119** of those rejected ballots were from Pennsylvania. The goal of this site is to teach people in PA how to vote successfully by mail so that their vote is actually counted. + +## Developing pavotebymail + +### Install dependencies + +```bash +$ brew install postgresql openssl +$ export LDFLAGS=$(pg_config --ld_flags) +$ pipenv install +``` + +### Start docker-compose for databases + +Before you can import data you'll need to be running the databases + +```bash +$ docker-compose -f docker/dev-compose.yml up +``` + +### Import database data so you can explore it on your local machine + +```bash +$ pipenv run pavotebymail data-import --import-config phildelphia --postgres-password example /path/to/data_to_import.py +``` + +### Populate google firestore database + +```bash +# tbd +``` + +### Run the worker and fake send postcards + +```bash +# tbd +``` diff --git a/docker/dev-compose.yml b/docker/dev-compose.yml new file mode 100644 index 0000000..49ebb29 --- /dev/null +++ b/docker/dev-compose.yml @@ -0,0 +1,24 @@ +version: '3' + +services: + db: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: example + ports: + - "5432:5432" + + dynamodb: + image: amazon/dynamodb-local + restart: always + ports: + - "9091:8000" + + firestore: + image: mtlynch/firestore-emulator + environment: + - FIRESTORE_PROJECT_ID=dummy-project-id + - PORT=8200 + ports: + - "8200:8200" diff --git a/worker/__init__.py b/pavotebymail/__init__.py similarity index 100% rename from worker/__init__.py rename to pavotebymail/__init__.py diff --git a/pavotebymail/cli.py b/pavotebymail/cli.py new file mode 100644 index 0000000..38f3e22 --- /dev/null +++ b/pavotebymail/cli.py @@ -0,0 +1,45 @@ +"""The main CLI for the application/worker +""" +import sys +import logging +import click +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from pavotebymail.dataimport import voter_data_import +from pavotebymail.dataimport.db.db import ensure_db + +root = logging.getLogger() +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +@click.group() +@click.option('--log-level', default='info') +def cli(log_level): + root.setLevel(getattr(logging, log_level.upper())) + + +@cli.command() +@click.argument('voter_data_path', type=click.Path(exists=True, + dir_okay=False, resolve_path=True)) +@click.option('--postgres-host', default='127.0.0.1') +@click.option('--postgres-port', type=click.INT, default=5432) +@click.option('--postgres-user', default='postgres') +@click.option('--postgres-password', default='postgres') +@click.option('--postgres-db', default='voters') +@click.option('--import-config', default='philadelphia') +def data_import(voter_data_path, postgres_host, postgres_port, + postgres_user, postgres_password, postgres_db, + import_config): + ensure_db(postgres_user, postgres_password, postgres_host, postgres_port, postgres_db) + + engine = create_engine('postgresql://%s:%s@%s:%d/%s' % ( + postgres_user, + postgres_password, + postgres_host, + postgres_port, + postgres_db + )) + Session = sessionmaker(bind=engine) + session = Session() + + voter_data_import(engine, session, voter_data_path, import_config) diff --git a/pavotebymail/dataimport/__init__.py b/pavotebymail/dataimport/__init__.py new file mode 100644 index 0000000..e20b1cf --- /dev/null +++ b/pavotebymail/dataimport/__init__.py @@ -0,0 +1,44 @@ +import os +import importlib +import yaml +from .loader import * +from .db import * +from .schemas import * +from sqlalchemy.orm import session, Session +from sqlalchemy.engine import Engine + +CURR_FILE_DIR = os.path.dirname(os.path.realpath(__file__)) + +def voter_data_import(engine: Engine, session: Session, voter_data_path: str, config_name: str, commit_size=10000, base_class=Base): + FieldParser, schema = load_config(config_name) + + voter_cls = create_class_from_schema('%sVoter' % config_name.capitalize(), config_name, + schema, base_class=base_class) + base_class.metadata.create_all(engine) + + items = DataLoader.load(schema, voter_data_path, FieldParser) + + count = 0 + + for item in items: + if count > commit_size: + session.commit() + count = 0 + voter = voter_cls(**item) + session.add(voter) + count += 1 + session.commit() + + +def load_config(config_name: str) -> (BaseFieldParser, Schema): + FieldParser = importlib.import_module('.configs.%s' % config_name, __name__).FieldParser + + # Load the schema + schema = load_schema(config_name) + return (FieldParser(), schema) + +def load_schema(config_name: str) -> Schema: + schema_path = os.path.join(CURR_FILE_DIR, 'configs', config_name, 'schema.yml') + + return Schema.load_raw(yaml.safe_load(open(schema_path))) + \ No newline at end of file diff --git a/pavotebymail/dataimport/configs/__init__.py b/pavotebymail/dataimport/configs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pavotebymail/dataimport/configs/philadelphia/__init__.py b/pavotebymail/dataimport/configs/philadelphia/__init__.py new file mode 100644 index 0000000..4f73e98 --- /dev/null +++ b/pavotebymail/dataimport/configs/philadelphia/__init__.py @@ -0,0 +1,7 @@ +from typing import List +from pavotebymail.dataimport.loader import BaseFieldParser + +class FieldParser(BaseFieldParser): + def parse_for_fields(self, blob) -> List[str]: + split_by_tabs = blob.split('\t') + return list(map(lambda a: a.strip('"'), split_by_tabs)) diff --git a/pavotebymail/dataimport/configs/philadelphia/schema.yml b/pavotebymail/dataimport/configs/philadelphia/schema.yml new file mode 100644 index 0000000..04c0b2f --- /dev/null +++ b/pavotebymail/dataimport/configs/philadelphia/schema.yml @@ -0,0 +1,154 @@ +fields: +- { "field": "ID Number", "type": "String", "is_primary_key": True } +- { "field": "Title", "type": "String" } +- { "field": "Last Name", "type": "String" } +- { "field": "First Name", "type": "String" } +- { "field": "Middle Name", "type": "String" } +- { "field": "Suffix", "type": "String" } +- { "field": "Gender", "type": "String" } +- { "field": "DOB", "type": "Date" } +- { "field": "Registration Date", "type": "Date" } +- { "field": "Voter Status", "type": "String" } +- { "field": "Status Change Date", "type": "Date" } +- { "field": "Party Code", "type": "String" } +- { "field": "House Number", "type": "String" } +- { "field": "House Number Suffix", "type": "String" } +- { "field": "Street Name", "type": "String" } +- { "field": "Apartment Number", "type": "String" } +- { "field": "Address Line 2", "type": "String" } +- { "field": "City", "type": "String" } +- { "field": "State", "type": "String" } +- { "field": "Zip", "type": "String" } +- { "field": "Mail Address 1", "type": "String" } +- { "field": "Mail Address 2", "type": "String" } +- { "field": "Mail City", "type": "String" } +- { "field": "Mail State", "type": "String" } +- { "field": "Mail Zip", "type": "String" } +- { "field": "Last Vote Date", "type": "Date" } +- { "field": "Precinct Code", "type": "String" } +- { "field": "Precinct Split ID", "type": "String" } +- { "field": "Date Last Changed", "type": "Date" } +- { "field": "Custom Data 1", "type": "String" } +- { "field": "District 1", "type": "String" } +- { "field": "District 2", "type": "String" } +- { "field": "District 3", "type": "String" } +- { "field": "District 4", "type": "String" } +- { "field": "District 5", "type": "String" } +- { "field": "District 6", "type": "String" } +- { "field": "District 7", "type": "String" } +- { "field": "District 8", "type": "String" } +- { "field": "District 9", "type": "String" } +- { "field": "District 10", "type": "String" } +- { "field": "District 11", "type": "String" } +- { "field": "District 12", "type": "String" } +- { "field": "District 13", "type": "String" } +- { "field": "District 14", "type": "String" } +- { "field": "District 15", "type": "String" } +- { "field": "District 16", "type": "String" } +- { "field": "District 17", "type": "String" } +- { "field": "District 18", "type": "String" } +- { "field": "District 19", "type": "String" } +- { "field": "District 20", "type": "String" } +- { "field": "District 21", "type": "String" } +- { "field": "District 22", "type": "String" } +- { "field": "District 23", "type": "String" } +- { "field": "District 24", "type": "String" } +- { "field": "District 25", "type": "String" } +- { "field": "District 26", "type": "String" } +- { "field": "District 27", "type": "String" } +- { "field": "District 28", "type": "String" } +- { "field": "District 29", "type": "String" } +- { "field": "District 30", "type": "String" } +- { "field": "District 31", "type": "String" } +- { "field": "District 32", "type": "String" } +- { "field": "District 33", "type": "String" } +- { "field": "District 34", "type": "String" } +- { "field": "District 35", "type": "String" } +- { "field": "District 36", "type": "String" } +- { "field": "District 37", "type": "String" } +- { "field": "District 38", "type": "String" } +- { "field": "District 39", "type": "String" } +- { "field": "District 40", "type": "String" } +- { "field": "Election 1 Vote Method", "type": "String" } +- { "field": "Election 1 Party", "type": "String" } +- { "field": "Election 2 Vote Method", "type": "String" } +- { "field": "Election 2 Party", "type": "String" } +- { "field": "Election 3 Vote Method", "type": "String" } +- { "field": "Election 3 Party", "type": "String" } +- { "field": "Election 4 Vote Method", "type": "String" } +- { "field": "Election 4 Party", "type": "String" } +- { "field": "Election 5 Vote Method", "type": "String" } +- { "field": "Election 5 Party", "type": "String" } +- { "field": "Election 6 Vote Method", "type": "String" } +- { "field": "Election 6 Party", "type": "String" } +- { "field": "Election 7 Vote Method", "type": "String" } +- { "field": "Election 7 Party", "type": "String" } +- { "field": "Election 8 Vote Method", "type": "String" } +- { "field": "Election 8 Party", "type": "String" } +- { "field": "Election 9 Vote Method", "type": "String" } +- { "field": "Election 9 Party", "type": "String" } +- { "field": "Election 10 Vote Method", "type": "String" } +- { "field": "Election 10 Party", "type": "String" } +- { "field": "Election 11 Vote Method", "type": "String" } +- { "field": "Election 11 Party", "type": "String" } +- { "field": "Election 12 Vote Method", "type": "String" } +- { "field": "Election 12 Party", "type": "String" } +- { "field": "Election 13 Vote Method", "type": "String" } +- { "field": "Election 13 Party", "type": "String" } +- { "field": "Election 14 Vote Method", "type": "String" } +- { "field": "Election 14 Party", "type": "String" } +- { "field": "Election 15 Vote Method", "type": "String" } +- { "field": "Election 15 Party", "type": "String" } +- { "field": "Election 16 Vote Method", "type": "String" } +- { "field": "Election 16 Party", "type": "String" } +- { "field": "Election 17 Vote Method", "type": "String" } +- { "field": "Election 17 Party", "type": "String" } +- { "field": "Election 18 Vote Method", "type": "String" } +- { "field": "Election 18 Party", "type": "String" } +- { "field": "Election 19 Vote Method", "type": "String" } +- { "field": "Election 19 Party", "type": "String" } +- { "field": "Election 20 Vote Method", "type": "String" } +- { "field": "Election 20 Party", "type": "String" } +- { "field": "Election 21 Vote Method", "type": "String" } +- { "field": "Election 21 Party", "type": "String" } +- { "field": "Election 22 Vote Method", "type": "String" } +- { "field": "Election 22 Party", "type": "String" } +- { "field": "Election 23 Vote Method", "type": "String" } +- { "field": "Election 23 Party", "type": "String" } +- { "field": "Election 24 Vote Method", "type": "String" } +- { "field": "Election 24 Party", "type": "String" } +- { "field": "Election 25 Vote Method", "type": "String" } +- { "field": "Election 25 Party", "type": "String" } +- { "field": "Election 26 Vote Method", "type": "String" } +- { "field": "Election 26 Party", "type": "String" } +- { "field": "Election 27 Vote Method", "type": "String" } +- { "field": "Election 27 Party", "type": "String" } +- { "field": "Election 28 Vote Method", "type": "String" } +- { "field": "Election 28 Party", "type": "String" } +- { "field": "Election 29 Vote Method", "type": "String" } +- { "field": "Election 29 Party", "type": "String" } +- { "field": "Election 30 Vote Method", "type": "String" } +- { "field": "Election 30 Party", "type": "String" } +- { "field": "Election 31 Vote Method", "type": "String" } +- { "field": "Election 31 Party", "type": "String" } +- { "field": "Election 32 Vote Method", "type": "String" } +- { "field": "Election 32 Party", "type": "String" } +- { "field": "Election 33 Vote Method", "type": "String" } +- { "field": "Election 33 Party", "type": "String" } +- { "field": "Election 34 Vote Method", "type": "String" } +- { "field": "Election 34 Party", "type": "String" } +- { "field": "Election 35 Vote Method", "type": "String" } +- { "field": "Election 35 Party", "type": "String" } +- { "field": "Election 36 Vote Method", "type": "String" } +- { "field": "Election 36 Party", "type": "String" } +- { "field": "Election 37 Vote Method", "type": "String" } +- { "field": "Election 37 Party", "type": "String" } +- { "field": "Election 38 Vote Method", "type": "String" } +- { "field": "Election 38 Party", "type": "String" } +- { "field": "Election 39 Vote Method", "type": "String" } +- { "field": "Election 39 Party", "type": "String" } +- { "field": "Election 40 Vote Method", "type": "String" } +- { "field": "Election 40 Party", "type": "String" } +- { "field": "Home Phone", "type": "String" } +- { "field": "County", "type": "String" } +- { "field": "Mail Country", "type": "String" } diff --git a/pavotebymail/dataimport/db/__init__.py b/pavotebymail/dataimport/db/__init__.py new file mode 100644 index 0000000..50007c5 --- /dev/null +++ b/pavotebymail/dataimport/db/__init__.py @@ -0,0 +1,2 @@ +from .base import * +from .tables import * diff --git a/pavotebymail/dataimport/db/base.py b/pavotebymail/dataimport/db/base.py new file mode 100644 index 0000000..c64447d --- /dev/null +++ b/pavotebymail/dataimport/db/base.py @@ -0,0 +1,2 @@ +from sqlalchemy.ext.declarative import declarative_base +Base = declarative_base() diff --git a/pavotebymail/dataimport/db/db.py b/pavotebymail/dataimport/db/db.py new file mode 100644 index 0000000..638a9fa --- /dev/null +++ b/pavotebymail/dataimport/db/db.py @@ -0,0 +1,27 @@ +import psycopg2 + +def ensure_db(user, password, host, port, db_name: str): + """Ensures a database exists in the postgresql server""" + + connection = psycopg2.connect( + database='postgres', + user=user, + password=password, + host=host, + port=port + ) + cursor = connection.cursor() + cursor.execute('commit') + + # Check that the database exists + cursor.execute("""SELECT datname FROM pg_database WHERE datistemplate = false""") + exists = False + for table in cursor.fetchall(): + if table[0] == db_name: + exists = True + break + + # Create if it does not + if not exists: + cursor.execute('create database %s' % db_name) + diff --git a/pavotebymail/dataimport/db/tables.py b/pavotebymail/dataimport/db/tables.py new file mode 100644 index 0000000..878995e --- /dev/null +++ b/pavotebymail/dataimport/db/tables.py @@ -0,0 +1,29 @@ +""" +Generates tables from a yaml schema +""" +import os +import yaml +from typing import List + +from sqlalchemy import Table, MetaData, Column, Integer, String, ForeignKey, Date +from sqlalchemy.orm import mapper +from .base import Base +from ..schemas import Schema + + +def create_class_from_schema(class_name: str, table_name: str, schema: Schema, base_class=Base): + schema_fields = schema.fields + class_attrs = { + "__tablename__": table_name, + } + + for field in schema_fields: + column_name = field['field'] + primary_key = column_name == schema.primary_key_name + column_type_str = field['type'] + column_type = String(100) + if column_type_str == 'Date': + column_type = Date() + class_attrs[field['field']] = Column(column_name, column_type, primary_key=primary_key) + + return type(class_name, (base_class, ), class_attrs) diff --git a/pavotebymail/dataimport/loader/__init__.py b/pavotebymail/dataimport/loader/__init__.py new file mode 100644 index 0000000..9b5ed21 --- /dev/null +++ b/pavotebymail/dataimport/loader/__init__.py @@ -0,0 +1 @@ +from .base import * diff --git a/pavotebymail/dataimport/loader/base.py b/pavotebymail/dataimport/loader/base.py new file mode 100644 index 0000000..5bc1ebb --- /dev/null +++ b/pavotebymail/dataimport/loader/base.py @@ -0,0 +1,61 @@ +import logging +from typing import List +import yaml +import arrow +from ..schemas import Schema + +logger = logging.getLogger(__name__) + +class BaseFieldParser(object): + """Parse a line or blob of text for fields. We'll assume things are a line + for now.""" + def parse_for_fields(self, blob: str) -> List[str]: + return blob.split(',') + + +class DataLoader(object): + """Loads data from a file into a dictionary""" + + @classmethod + def load(cls, schema: Schema, data_path: str, field_parser: BaseFieldParser): + loader = cls(schema, data_path, field_parser) + return loader.process() + + def __init__(self, schema, data_path: str, field_parser: BaseFieldParser): + self._field_parser = field_parser + self._schema = schema + self._data_path = data_path + + def process(self): + """Default data processing opens a file and processes each line of the + file as an item""" + + with open(self._data_path) as data: + count = 0 + for line in data: + item = dict() + parsed_fields = self._field_parser.parse_for_fields(line) + + if len(parsed_fields) != len(self._schema.fields): + logger.warn("Skipping item. Does not match expected schema") + continue + + for i in range(len(parsed_fields)): + field_def = self.field_def(i) + item[field_def['field']] = self.type_translate(field_def['type'], parsed_fields[i]) + + count += 1 + logger.info("loaded %s lines" % count) + yield item + + def type_translate(self, type: str, value: str): + if value == '': + return None + if type == 'String': + return value + if type == 'Date': + return arrow.get(value, ['MM/DD/YYYY']).date() + + + def field_def(self, index): + return self._schema.fields[index] diff --git a/pavotebymail/dataimport/schemas/__init__.py b/pavotebymail/dataimport/schemas/__init__.py new file mode 100644 index 0000000..2730e7c --- /dev/null +++ b/pavotebymail/dataimport/schemas/__init__.py @@ -0,0 +1,32 @@ +class Schema(object): + @classmethod + def load_raw(cls, raw_schema) -> 'Schema': + fields = [] + + for field_def in raw_schema['fields']: + field = field_def['field'].lower().replace(' ', '_') + fields.append(dict( + field=field, + type=field_def['type'], + is_primary_key=field_def.get('is_primary_key', False) + )) + + + return cls(fields) + + def __init__(self, fields): + self._fields = fields + + @property + def fields(self): + return self._fields + + @property + def primary_key_name(self): + primary_key = list(filter(lambda a: a.get('is_primary_key', False), self.fields)) + if len(primary_key) != 1: + raise Exception('Schema does not have a primary key') + + return primary_key[0]['field'] + + diff --git a/worker/db.py b/pavotebymail/db.py similarity index 100% rename from worker/db.py rename to pavotebymail/db.py diff --git a/worker/fixtures.py b/pavotebymail/fixtures.py similarity index 100% rename from worker/fixtures.py rename to pavotebymail/fixtures.py diff --git a/pavotebymail/models/__init__.py b/pavotebymail/models/__init__.py new file mode 100644 index 0000000..59967e5 --- /dev/null +++ b/pavotebymail/models/__init__.py @@ -0,0 +1 @@ +from .voter import Voter diff --git a/worker/model.py b/pavotebymail/models/voter.py similarity index 100% rename from worker/model.py rename to pavotebymail/models/voter.py diff --git a/worker/report.py b/pavotebymail/report.py similarity index 100% rename from worker/report.py rename to pavotebymail/report.py diff --git a/worker/worker.py b/pavotebymail/worker.py similarity index 100% rename from worker/worker.py rename to pavotebymail/worker.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d5e0a31 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup, find_packages + +setup( + name="pavotebymail", + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'pavotebymail=pavotebymail.cli:cli' + ] + } +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/philly_schema_test.txt b/tests/fixtures/philly_schema_test.txt new file mode 100644 index 0000000..47af188 --- /dev/null +++ b/tests/fixtures/philly_schema_test.txt @@ -0,0 +1,2 @@ +"111111111-51" "" "APPLE" "BILL" "" "" "M" 09/30/1951 09/30/2020 "A" 08/30/2020 "D" "1122" "" "PAVOTE ST" "" "" "PHILADELPHIA" "PA" "19111" "" "" "" "" "" "5555" "5555-1" 08/30/2020 "" "5555" "WD56" "" "MN01" "" "STH172" "STS05" "USC02" "CTC10" "CPR" "CPD" "RCD13" "5614-1" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "5555555555" "PHILADELPHIA" "" +"111111111-52" "" "HORSE" "STAPLER" "B" "JR" "M" 09/30/1999 09/30/2020 "A" 08/30/2020 "LN" "1123" "" "PAVOTE AVE" "" "" "PHILADELPHIA" "PA" "19111" "" "" "" "" "" "1210" "1210-1" 08/30/2020 "" "1210" "WD12" "" "MN01" "" "STH198" "STS04" "USC03" "CTC08" "CPR" "CPD" "RCD02" "1210-1" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "5555555555" "PHILADELPHIA" "" diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..6510512 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,75 @@ +from .tools import fixture_path + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, instrumentation +from sqlalchemy.orm.session import close_all_sessions +from sqlalchemy.ext.declarative import declarative_base +from pavotebymail.dataimport import load_config +from pavotebymail.dataimport.loader import DataLoader +from pavotebymail.dataimport.db.tables import create_class_from_schema +from pavotebymail.dataimport.db import Base + + +def test_philly_sql_table(): + _, schema = load_config('philadelphia') + PhillyVoter = create_class_from_schema('PhillyVoter', 'philly_voters', schema) + assert PhillyVoter.__dict__['id_number'] is not None + assert PhillyVoter.__dict__['home_phone'] is not None + + +class TestPostgres(object): + def setup(self): + root_engine = create_engine('postgresql://postgres:example@127.0.0.1:5432/postgres') + root_conn = root_engine.connect() + root_conn.execute('commit') + + try: + root_conn.execute('drop database test_db') + except: + pass + root_conn.execute('commit') + + root_conn.execute('create database test_db') + root_conn.close() + + self.field_parser, self.schema = load_config('philadelphia') + + self.engine = create_engine('postgresql://postgres:example@127.0.0.1:5432/test_db') + self.base_class = declarative_base() + self.Session = sessionmaker(bind=self.engine) + self.session = self.Session() + self.PhillyVoter = create_class_from_schema('PhillyVoter', 'philly_voters', self.schema, base_class=self.base_class) + + self.base_class.metadata.create_all(self.engine) + + def teardown(self): + # Close the current connection + close_all_sessions() + self.engine.dispose() + + root_engine = create_engine('postgresql://postgres:example@127.0.0.1:5432/postgres') + root_conn = root_engine.connect() + root_conn.execute('commit') + root_conn.execute('drop database test_db') + root_conn.close() + + def test_add_to_database(self): + voter = self.PhillyVoter(id_number='11123334-41') + + self.session.add(voter) + self.session.commit() + + def test_adding_fixtures_to_database(self): + philly_test_path = fixture_path('philly_schema_test.txt') + field_parser, schema = load_config('philadelphia') + + items = DataLoader.load(schema, philly_test_path, field_parser) + + for item in items: + voter = self.PhillyVoter(**item) + self.session.add(voter) + self.session.commit() + + voters = list(self.session.query(self.PhillyVoter).order_by(self.PhillyVoter.id_number)) + assert voters[0].first_name == 'BILL' + assert voters[1].first_name == 'STAPLER' diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..db52a75 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,58 @@ +import os + +from pavotebymail.dataimport import load_config +from pavotebymail.dataimport.loader import DataLoader +from .tools import fixture_path + + +def test_philly_parse(): + philly_test_path = fixture_path('philly_schema_test.txt') + field_parser, schema = load_config('philadelphia') + + items = list(DataLoader.load(schema, philly_test_path, field_parser)) + + assert items[0]['first_name'] == 'BILL' + assert items[0]['middle_name'] == None + assert items[0]['last_name'] == 'APPLE' + assert items[0]['suffix'] == None + assert items[0]['id_number'] == '111111111-51' + assert items[0]['party_code'] == 'D' + assert items[0]['home_phone'] == '5555555555' + assert items[0]['house_number'] == '1122' + assert items[0]['house_number_suffix'] == None + assert items[0]['street_name'] == 'PAVOTE ST' + assert items[0]['apartment_number'] == None + assert items[0]['address_line_2'] == None + assert items[0]['city'] == 'PHILADELPHIA' + assert items[0]['state'] == 'PA' + assert items[0]['zip'] == '19111' + assert items[0]['mail_address_1'] == None + assert items[0]['mail_address_2'] == None + assert items[0]['mail_city'] == None + assert items[0]['mail_state'] == None + assert items[0]['mail_zip'] == None + assert items[0]['last_vote_date'] == None + assert items[0]['election_1_vote_method'] == None + assert items[0]['election_1_party'] == None + + assert items[1]['first_name'] == 'STAPLER' + assert items[1]['middle_name'] == 'B' + assert items[1]['last_name'] == 'HORSE' + assert items[1]['suffix'] == 'JR' + assert items[1]['id_number'] == '111111111-52' + assert items[1]['party_code'] == 'LN' + assert items[1]['home_phone'] == '5555555555' + assert items[1]['house_number'] == '1123' + assert items[1]['house_number_suffix'] == None + assert items[1]['street_name'] == 'PAVOTE AVE' + assert items[1]['apartment_number'] == None + assert items[1]['address_line_2'] == None + assert items[1]['city'] == 'PHILADELPHIA' + assert items[1]['state'] == 'PA' + assert items[1]['zip'] == '19111' + assert items[1]['mail_address_1'] == None + assert items[1]['mail_address_2'] == None + assert items[1]['mail_city'] == None + assert items[1]['mail_state'] == None + assert items[1]['mail_zip'] == None + assert items[1]['last_vote_date'] == None diff --git a/tests/tools.py b/tests/tools.py new file mode 100644 index 0000000..a3af07a --- /dev/null +++ b/tests/tools.py @@ -0,0 +1,6 @@ +import os + +CURR_FILE_DIR = os.path.dirname(os.path.realpath(__file__)) + +def fixture_path(*args): + return os.path.join(CURR_FILE_DIR, 'fixtures', *args) From b60e6208ae5bb8f01f244c42c66331b76392f072 Mon Sep 17 00:00:00 2001 From: Reuven V Gonzales Date: Sun, 6 Sep 2020 03:10:00 -0700 Subject: [PATCH 3/8] Add instructions on running tests --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 7451b6d..3b98158 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,12 @@ Before you can import data you'll need to be running the databases $ docker-compose -f docker/dev-compose.yml up ``` +### Run tests + +```bash +pipenv run pytest +``` + ### Import database data so you can explore it on your local machine ```bash From 3bc40833585352878be8e6d49890d12569ed0ce9 Mon Sep 17 00:00:00 2001 From: "Reuven V. Gonzales" Date: Sun, 6 Sep 2020 03:14:46 -0700 Subject: [PATCH 4/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b98158..de6d7d8 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ pipenv run pytest ### Import database data so you can explore it on your local machine ```bash -$ pipenv run pavotebymail data-import --import-config phildelphia --postgres-password example /path/to/data_to_import.py +$ pipenv run pavotebymail data-import --import-config phildelphia --postgres-password postgres /path/to/data_to_import.py ``` ### Populate google firestore database From e4ed53c2f93334f943fc2cffe1e3c0b1ea6bbbd5 Mon Sep 17 00:00:00 2001 From: "Reuven V. Gonzales" Date: Sun, 6 Sep 2020 03:15:03 -0700 Subject: [PATCH 5/8] Update exporters/docker/data-explorer-docker-compose.yml --- exporters/docker/data-explorer-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/docker/data-explorer-docker-compose.yml b/exporters/docker/data-explorer-docker-compose.yml index 49ebb29..f8d5b77 100644 --- a/exporters/docker/data-explorer-docker-compose.yml +++ b/exporters/docker/data-explorer-docker-compose.yml @@ -5,7 +5,7 @@ services: image: postgres restart: always environment: - POSTGRES_PASSWORD: example + POSTGRES_PASSWORD: postgres ports: - "5432:5432" From af069e83d3e1f63eee81d82147ff27e8e678cdcf Mon Sep 17 00:00:00 2001 From: "Reuven V. Gonzales" Date: Sun, 6 Sep 2020 03:15:31 -0700 Subject: [PATCH 6/8] Update docker/dev-compose.yml --- docker/dev-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/dev-compose.yml b/docker/dev-compose.yml index 49ebb29..f8d5b77 100644 --- a/docker/dev-compose.yml +++ b/docker/dev-compose.yml @@ -5,7 +5,7 @@ services: image: postgres restart: always environment: - POSTGRES_PASSWORD: example + POSTGRES_PASSWORD: postgres ports: - "5432:5432" From a838a6cce25d0a41b83287fc7fab1f7643c4ead5 Mon Sep 17 00:00:00 2001 From: Conor Gilsenan Date: Sun, 6 Sep 2020 23:43:33 -0400 Subject: [PATCH 7/8] Fix typo in flag name --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de6d7d8..f8c5819 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ The goal of this site is to teach people in PA how to vote successfully by mail ```bash $ brew install postgresql openssl -$ export LDFLAGS=$(pg_config --ld_flags) -$ pipenv install +$ export LDFLAGS=$(pg_config --ldflags) +$ pipenv install ``` ### Start docker-compose for databases From 37f4795f581b388008857f4d2e04c946233eb341 Mon Sep 17 00:00:00 2001 From: Conor Gilsenan Date: Sun, 6 Sep 2020 23:47:13 -0400 Subject: [PATCH 8/8] WIP. Initial Dockerfile commit. --- Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8424cab --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM python + +# Install dependencies. +RUN apt-get update && apt-get install -y \ + openssl \ + postgresql + +# Set env vars for postgres. +ENV LDFLAGS $(pg_config --ldflags) + +# Install pipenv. +RUN pip install pipenv + +# Install our project dependencies. +# Docs: https://pipenv-fork.readthedocs.io/en/latest/basics.html#pipenv-install +# --dev — Install both develop and default packages from Pipfile. +# --system — Use the system pip command rather than the one from your +# virtualenv. +# --deploy — Make sure the packages are properly locked in Pipfile.lock, and +# abort if the lock file is out-of-date. +# --ignore-pipfile — Ignore the Pipfile and install from the Pipfile.lock. +# --skip-lock — Ignore the Pipfile.lock and install from the Pipfile. In +# addition, do not write out a Pipfile.lock reflecting changes to +# the Pipfile. +COPY Pipfile Pipfile +COPY Pipfile.lock Pipfile.lock +RUN pipenv install --system --dev + +# Setup docker image to run as an executable. +ENTRYPOINT [ "pipenv", "run" ]