From 2677b596cb37a991634bf31eac4a38faa25aefbc Mon Sep 17 00:00:00 2001 From: palewire Date: Mon, 17 Jul 2023 11:17:11 -0400 Subject: [PATCH] Pre-commit fixes --- .flake8 | 2 + .github/workflows/docs.yaml | 6 +-- .pre-commit-config.yaml | 56 ++++++++++++++++++++++ Pipfile | 1 + Pipfile.lock | 93 ++++++++++++++++++++++++++++++++----- README.md | 2 +- docs/_static/css/custom.css | 2 +- docs/command-line.md | 2 +- docs/conf.py | 27 +++++------ docs/index.md | 2 +- docs/prerequisites.md | 2 +- docs/python.md | 38 +++++++-------- docs/requirements.txt | 2 +- docs/web-scraping.md | 4 +- scrapers/crime/scrape.py | 15 +++--- 15 files changed, 191 insertions(+), 63 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..457e6b8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +extend-ignore = D100,D104,E203,E501,SIM115 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 40948f9..6cb5b0b 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -10,11 +10,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - + - uses: ammaraskar/sphinx-action@master with: docs-folder: "docs/" - + - uses: actions/upload-artifact@v1 with: name: documentation-html @@ -26,4 +26,4 @@ jobs: aws_secret_access_key: ${{ secrets.PALEWIRE_DOCS_AWS_SECRET_ACCESS_KEY }} aws_bucket: ${{ secrets.PALEWIRE_DOCS_AWS_BUCKET }} source_dir: docs/_build/html/ - destination_dir: first-github-scraper \ No newline at end of file + destination_dir: first-github-scraper diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..af6d50e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,56 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-byte-order-marker + - id: check-case-conflict + - id: check-json + - id: mixed-line-ending + - id: check-ast + - id: check-merge-conflict + +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + +- repo: https://github.com/asottile/blacken-docs + rev: 1.15.0 + hooks: + - id: blacken-docs + additional_dependencies: [black] + +- repo: https://github.com/timothycrosley/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] + +- repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-docstrings + - flake8-simplify + - pep8-naming + +- repo: https://github.com/asottile/pyupgrade + rev: v3.9.0 + hooks: + - id: pyupgrade + args: [--py37-plus] + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.4.1 + hooks: + - id: mypy + additional_dependencies: + - types-requests diff --git a/Pipfile b/Pipfile index 459c1b2..4390119 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ bs4 = "*" sphinx = "*" sphinx-autobuild = "*" myst-parser = "*" +pre-commit = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 40cacc9..091569c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0364c9ef06eea5e79cb19967b6eea6896633390e8750d51e54a2ee9204933bbc" + "sha256": "8559f177ed437a82c9025cc6ba946c6d930ed1312171e676d762c19494f2b0f6" }, "pipfile-spec": 6, "requires": { @@ -29,7 +29,7 @@ "sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610", "sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==2.12.1" }, "beautifulsoup4": { @@ -55,6 +55,14 @@ "markers": "python_version >= '3.6'", "version": "==2023.5.7" }, + "cfgv": { + "hashes": [ + "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426", + "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==3.3.1" + }, "charset-normalizer": { "hashes": [ "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96", @@ -133,7 +141,7 @@ "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac", "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.2.0" }, "colorama": { @@ -144,14 +152,37 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", "version": "==0.4.6" }, + "distlib": { + "hashes": [ + "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057", + "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8" + ], + "version": "==0.3.7" + }, "docutils": { "hashes": [ "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6", "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==0.20.1" }, + "filelock": { + "hashes": [ + "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81", + "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.12.2" + }, + "identify": { + "hashes": [ + "sha256:0aac67d5b4812498056d28a9a512a483f5085cc28640b02b258a59dac34301d4", + "sha256:986dbfb38b1140e763e413e6feb44cd731faf72d1909543178aa79b0e258265d" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==2.5.24" + }, "idna": { "hashes": [ "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", @@ -173,7 +204,7 @@ "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.1.2" }, "livereload": { @@ -244,7 +275,7 @@ "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==2.1.3" }, "mdit-py-plugins": { @@ -260,7 +291,7 @@ "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==0.1.2" }, "myst-parser": { @@ -271,20 +302,44 @@ "index": "pypi", "version": "==2.0.0" }, + "nodeenv": { + "hashes": [ + "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2", + "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==1.8.0" + }, "packaging": { "hashes": [ "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==23.1" }, + "platformdirs": { + "hashes": [ + "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421", + "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.9.1" + }, + "pre-commit": { + "hashes": [ + "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb", + "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023" + ], + "index": "pypi", + "version": "==3.3.3" + }, "pygments": { "hashes": [ "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c", "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==2.15.1" }, "pyyaml": { @@ -341,6 +396,14 @@ "index": "pypi", "version": "==2.31.0" }, + "setuptools": { + "hashes": [ + "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", + "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==68.0.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -361,7 +424,7 @@ "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8", "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==2.4.1" }, "sphinx": { @@ -450,8 +513,16 @@ "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1", "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==2.0.3" + }, + "virtualenv": { + "hashes": [ + "sha256:18d1b37fc75cc2670625702d76849a91ebd383768b4e91382a8d51be3246049e", + "sha256:e2a7cef9da880d693b933db7654367754f14e20650dc60e8ee7385571f8593a3" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==20.24.0" } }, "develop": {} diff --git a/README.md b/README.md index 9e95b44..7fa8ba4 100644 --- a/README.md +++ b/README.md @@ -24,4 +24,4 @@ to the restructured text files in the ``docs`` directory. $ make docs ``` -Open ``http://localhost:8000`` in your browser and start making changes. \ No newline at end of file +Open ``http://localhost:8000`` in your browser and start making changes. diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index cb6f12f..5254d70 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -295,4 +295,4 @@ table.align-default { pre, tt, code { background: none; font-family: 'IBM Plex Mono', 'Droid Sans Mono', monospace; -} \ No newline at end of file +} diff --git a/docs/command-line.md b/docs/command-line.md index 232f3ec..741b87d 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -143,4 +143,4 @@ del test.py **This must be done with caution**. Files you delete from the command line do not go into the recycle bin. They are gone. **Forever**. ``` -And that's it! You've learned all the basic command-line tricks necessary to move on. \ No newline at end of file +And that's it! You've learned all the basic command-line tricks necessary to move on. diff --git a/docs/conf.py b/docs/conf.py index ae0f979..9be56aa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,27 +1,24 @@ from datetime import datetime - -extensions = [ - "myst_parser" -] +extensions = ["myst_parser"] templates_path = ["_templates"] source_suffix = ".rst" master_doc = "index" -project = 'First Web Scraper' +project = "First Web Scraper" year = datetime.now().year -copyright = f'{year} Chase Davis, Jackie Kazil, Ken Schwencke, Sisi Wei, Ben Welsh and Matt Wynn' +copyright = f"{year} Chase Davis, Jackie Kazil, Ken Schwencke, Sisi Wei, Ben Welsh and Matt Wynn" exclude_patterns = ["_build"] html_theme = "alabaster" html_sidebars = { - '**': [ - 'about.html', - 'navigation.html', - 'relations.html', - 'searchbox.html', - 'donate.html', + "**": [ + "about.html", + "navigation.html", + "relations.html", + "searchbox.html", + "donate.html", ] } html_theme_options = { @@ -30,9 +27,9 @@ "show_relbar_bottom": True, } -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = [ - 'css/custom.css', + "css/custom.css", ] -pygments_style = 'sphinx' \ No newline at end of file +pygments_style = "sphinx" diff --git a/docs/index.md b/docs/index.md index 453ee05..e4e2d1e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,4 +25,4 @@ This course is free. The materials assume the reader has little experience with ./python.md ./web-scraping.md ./about.md -``` \ No newline at end of file +``` diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 455de1b..9a86457 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -104,4 +104,4 @@ Verify pip is installed with the following. ```bash pip -V -``` \ No newline at end of file +``` diff --git a/docs/python.md b/docs/python.md index 919b521..0c035a3 100644 --- a/docs/python.md +++ b/docs/python.md @@ -68,7 +68,7 @@ the terminal because that value is stored in the variable. ```python print(greeting) -'Hello world!' +"Hello world!" ``` ### Data types @@ -86,7 +86,7 @@ are declared within single or double quotes like so: ```python greeting = "Hello, world!" goodbye = "Seeya later, dude." -favorite_animal = 'Donkey' +favorite_animal = "Donkey" ``` #### Integers @@ -123,9 +123,9 @@ lists. Here are several examples: ```python list_of_numbers = [1, 2, 3, 4, 5] -list_of_strings = ['a', 'b', 'c', 'd'] -list_of_both = [1, 'a', 2, 'b'] -list_of_lists = [[1, 2, 3], [4, 5, 6], ['a', 'b', 'c']] +list_of_strings = ["a", "b", "c", "d"] +list_of_both = [1, "a", 2, "b"] +list_of_lists = [[1, 2, 3], [4, 5, 6], ["a", "b", "c"]] ``` Lists also have another neat feature: The ability to retrieve individual @@ -134,11 +134,11 @@ items. In order to get a specific item out of a list, you just pass in its posit All lists in Python are **zero-indexed**, which means the first item in them sits at position 0. ```python -my_list = ['a', 'b', 'c', 'd'] +my_list = ["a", "b", "c", "d"] my_list[0] -'a' +"a" my_list[2] -'c' +"c" ``` You can also extract a range of values by specifiying the first and last @@ -146,21 +146,21 @@ positions you want to retrieve with a colon in between them, like this: ```python my_list[0:2] -['a', 'b'] +["a", "b"] ``` To specify the last value in a list, you use `-1` ```python my_list[-1] -'d' +"d" ``` So if you want ever item in a list besides the last, you would do: ```python my_list[-1] -'d' +"d" ``` #### Tuples @@ -169,7 +169,7 @@ Tuples are a special type of list that cannot be changed once they are created. ```python tuple_of_numbers = (1, 2, 3, 4, 5) -tuple_of_strings = ('a', 'b', 'c', 'd') +tuple_of_strings = ("a", "b", "c", "d") ``` #### Dictionaries @@ -179,7 +179,7 @@ Dictionaries are probably the most difficult data type to explain, but also amon An example will make this a little more clear. ```python -my_phonebook = {'Mom': '713-555-5555', 'Chinese Takeout': '573-555-5555'} +my_phonebook = {"Mom": "713-555-5555", "Chinese Takeout": "573-555-5555"} ``` In this example, the keys are the names "Mom" and "Chinese takeout", @@ -191,8 +191,8 @@ dictionary values in practice can be any data type. If you wanted to get Mom's phone number from the dictionary, here's how: ```python -my_phonebook['Mom'] -713-555-5555 +my_phonebook["Mom"] +713 - 555 - 5555 ``` There's a lot more to dictionaries, but that's all you need to know for now. @@ -220,8 +220,6 @@ Let's start with a simple example. number = 10 if number > 5: print("Wow, that's a big number!") - -Wow, that's a big number! ``` Our little program in this case starts with a variable, which we've called `number`, being set to 10. That's pretty simple, and a concept you should be familiar with by this point. @@ -320,7 +318,7 @@ Many times during your programming career, you'll find it helps to run through a That's where for loops come in. Let's start by having Python say the ABC's: ```python -list_of_letters = ['a', 'b', 'c'] +list_of_letters = ["a", "b", "c"] for letter in list_of_letters: print(letter) @@ -451,6 +449,7 @@ with Python: ```python import urllib.request + urllib.request.urlopen("http://www.python.org/").read() ``` @@ -458,6 +457,7 @@ You can also import specific tools from inside a toolkit by working in the `from ```python from urllib.request import urlopen + urlopen("http://www.python.org/").read() ``` @@ -465,4 +465,4 @@ In practice, you'll use both of these methods. ```{note} There's no rule but most Python programmers try to keep things manageable by lining up all `import` statements at the top of each script. -``` \ No newline at end of file +``` diff --git a/docs/requirements.txt b/docs/requirements.txt index 2a6c22d..f436e55 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ sphinx sphinx-autobuild -myst-parser \ No newline at end of file +myst-parser diff --git a/docs/web-scraping.md b/docs/web-scraping.md index 6ce1f1a..1b38b10 100644 --- a/docs/web-scraping.md +++ b/docs/web-scraping.md @@ -103,8 +103,8 @@ Then open your text editor and save an empty file into the directory name `scrap ```python import requests -url = 'https://report.boonecountymo.org/mrcjava/servlet/SH01_MP.I00290s' -response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}) +url = "https://report.boonecountymo.org/mrcjava/servlet/SH01_MP.I00290s" +response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}) html = response.content print(html) ``` diff --git a/scrapers/crime/scrape.py b/scrapers/crime/scrape.py index a37b8f5..2a11a6f 100644 --- a/scrapers/crime/scrape.py +++ b/scrapers/crime/scrape.py @@ -1,23 +1,24 @@ import csv + import requests from BeautifulSoup import BeautifulSoup -url = 'http://www.showmeboone.com/sheriff/JailResidents/JailResidents.asp' +url = "http://www.showmeboone.com/sheriff/JailResidents/JailResidents.asp" response = requests.get(url) html = response.content soup = BeautifulSoup(html) -table = soup.find('table', attrs={'class': 'resultsTable'}) +table = soup.find("table", attrs={"class": "resultsTable"}) list_of_rows = [] -for row in table.findAll('tr')[1:]: +for row in table.findAll("tr")[1:]: list_of_cells = [] - for cell in row.findAll('td'): - text = cell.text.replace(' ', '') + for cell in row.findAll("td"): + text = cell.text.replace(" ", "") list_of_cells.append(text) list_of_rows.append(list_of_cells) -outfile = open("./inmates.csv", "wb") +outfile = open("./inmates.csv", "w") writer = csv.writer(outfile) writer.writerow(["Last", "First", "Middle", "Gender", "Race", "Age", "City", "State"]) -writer.writerows(list_of_rows) \ No newline at end of file +writer.writerows(list_of_rows)