OD-1730: Use Ruff for linting and formatting (#74)

otwcode · Nov 4, 2023 · bdc5d9d · bdc5d9d
1 parent c4b766a
commit bdc5d9d
Show file tree

Hide file tree

Showing 29 changed files with 1,892 additions and 1,331 deletions.
diff --git a/.github/workflows/python-app-linux.yml b/.github/workflows/python-app-linux.yml
@@ -35,14 +35,14 @@ jobs:
         shell: bash
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest
+          pip install pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Lint with flake8
+      - name: Lint with ruff
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          ruff check .
+      - name: Check formatting with ruff
+        run: |
+          ruff format --check .
       - name: Test with pytest
         run: |
           pytest
diff --git a/.github/workflows/python-app-macos-windows.yml b/.github/workflows/python-app-macos-windows.yml
@@ -27,14 +27,14 @@ jobs:
         shell: bash
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest
+          pip install pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Lint with flake8
+      - name: Lint with ruff
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          ruff check .
+      - name: Check formatting with ruff
+        run: |
+          ruff format --check .
       - name: Test with pytest - exclude MySQL integration tests
         run: |
           pytest --ignore "test/test_multitag_mapping.py" --ignore "test/test_percent_symbol.py" --ignore "test/test_tags_length.py" --ignore "test/test_step_05.py"
diff --git a/01-Load-Automated-Archive-into-Mysql.py b/01-Load-Automated-Archive-into-Mysql.py
@@ -3,12 +3,15 @@
 from automated_archive import aa
 
 if __name__ == "__main__":
-  args_obj = Args()
-  args = args_obj.args_for_01()
-  log = args_obj.logger_with_filename()
-  sql = Sql(args, log)
-
-  # eg: python 01-Load-Automated-Archive-into-Mysql.py -dh localhost -du root -dt dsa -dd temp_python -a AA -f /path/to/ARCHIVE_DB.pl -o .
-  log.info('Loading Automated Archive file "{0}" into database "{1}"'.format(args.db_input_file, args.temp_db_database))
-  aa.clean_and_load_data(args, log)
+    args_obj = Args()
+    args = args_obj.args_for_01()
+    log = args_obj.logger_with_filename()
+    sql = Sql(args, log)
 
+    # eg: python 01-Load-Automated-Archive-into-Mysql.py -dh localhost -du root -dt dsa -dd temp_python -a AA -f /path/to/ARCHIVE_DB.pl -o .
+    log.info(
+        'Loading Automated Archive file "{0}" into database "{1}"'.format(
+            args.db_input_file, args.temp_db_database
+        )
+    )
+    aa.clean_and_load_data(args, log)
diff --git a/02a-Load-Chapters-to-Working-Table.py b/02a-Load-Chapters-to-Working-Table.py
@@ -8,20 +8,19 @@
 
 # Given an existing final chapter table, this will use the URL field and chapter location to load the chapter contents
 def __current_table(table_name, db):
-  query = "SELECT * FROM `{0}`.`{1}`".format(args.output_database, table_name)
-  dict_cursor = db.cursor(cursors.DictCursor)
-  dict_cursor.execute(query)
-  return dict_cursor.fetchall()
+    query = "SELECT * FROM `{0}`.`{1}`".format(args.output_database, table_name)
+    dict_cursor = db.cursor(cursors.DictCursor)
+    dict_cursor.execute(query)
+    return dict_cursor.fetchall()
 
 
 if __name__ == "__main__":
-  # TODO the eFiction process now loads chapters into MySQL before further processing in ODAP, all others should too
-  args_obj = Args()
-  args = args_obj.args_for_07()
-  log = args_obj.logger_with_filename()
-  sql = Sql(args, log)
-  chaps = Chapters(args, sql, log)
+    # TODO the eFiction process now loads chapters into MySQL before further processing in ODAP, all others should too
+    args_obj = Args()
+    args = args_obj.args_for_07()
+    log = args_obj.logger_with_filename()
+    sql = Sql(args, log)
+    chaps = Chapters(args, sql, log)
 
-
-  log.info("Loading chapters from {0}...".format(args.chapters_path))
-  chaps.populate_chapters()
+    log.info("Loading chapters from {0}...".format(args.chapters_path))
+    chaps.populate_chapters()
diff --git a/02b-Extract-Tags-From-Stories.py b/02b-Extract-Tags-From-Stories.py
@@ -6,43 +6,65 @@
 
 
 if __name__ == "__main__":
-  """
+    """
   Only for non-eFiction archives.
   This script creates a table called tags in the temporary database and denormalises all the tags for each story.
   This table is the basis for the Tag Wrangling sheet and is used to map the tags back to the story when the final
   tables are created.
   """
-  args_obj = Args()
-  args = args_obj.args_for_02()
-  log = args_obj.logger_with_filename()
-  sql = Sql(args, log)
-  tags = Tags(args, sql, log)
-  log.info('Processing tags from stories and bookmarks table in {0}'.format(args.temp_db_database))
-  tags.create_tags_table()
-
-  tag_col_list = {}
-  stories_id_name = ""
-  stories_table_name = ""
-
-  # AUTOMATED ARCHIVE
-  if args.archive_type == 'AA':
-
-    story_table_name = input('Story table name (default: "stories"): ')
-    if story_table_name is None or story_table_name == '':
-      story_table_name = 'stories'
-
-    bookmark_table_name = input('Bookmark table name (default: "story_links"): ')
-    if bookmark_table_name is None or bookmark_table_name == '':
-      bookmark_table_name = 'story_links'
-
-    tag_columns = input('Column names containing tags \n   (delimited by commas - default: "rating, tags, warnings, characters, fandoms, relationships"): ')
-    if tag_columns is None or tag_columns == '':
-      tag_columns = "rating, tags, warnings, characters, fandoms, relationships"
-    # fancy footwork to ensure compatibility with eFiction
-    tag_col_list = re.split(r", ?", tag_columns)
-    tag_columns_dict = dict(zip(tag_col_list, tag_col_list))
-    fields_with_fandom = args.fields_with_fandom.split(", ") if args.fields_with_fandom is not None else []
-    tags.populate_tag_table(args.temp_db_database, "id", story_table_name, tag_columns_dict, fields_with_fandom)
-    tags.populate_tag_table(args.temp_db_database, "id", bookmark_table_name, tag_columns_dict, fields_with_fandom, False)
-
-  log.info("Done extracting tags.")
+    args_obj = Args()
+    args = args_obj.args_for_02()
+    log = args_obj.logger_with_filename()
+    sql = Sql(args, log)
+    tags = Tags(args, sql, log)
+    log.info(
+        "Processing tags from stories and bookmarks table in {0}".format(
+            args.temp_db_database
+        )
+    )
+    tags.create_tags_table()
+
+    tag_col_list = {}
+    stories_id_name = ""
+    stories_table_name = ""
+
+    # AUTOMATED ARCHIVE
+    if args.archive_type == "AA":
+        story_table_name = input('Story table name (default: "stories"): ')
+        if story_table_name is None or story_table_name == "":
+            story_table_name = "stories"
+
+        bookmark_table_name = input('Bookmark table name (default: "story_links"): ')
+        if bookmark_table_name is None or bookmark_table_name == "":
+            bookmark_table_name = "story_links"
+
+        tag_columns = input(
+            'Column names containing tags \n   (delimited by commas - default: "rating, tags, warnings, characters, fandoms, relationships"): '
+        )
+        if tag_columns is None or tag_columns == "":
+            tag_columns = "rating, tags, warnings, characters, fandoms, relationships"
+        # fancy footwork to ensure compatibility with eFiction
+        tag_col_list = re.split(r", ?", tag_columns)
+        tag_columns_dict = dict(zip(tag_col_list, tag_col_list))
+        fields_with_fandom = (
+            args.fields_with_fandom.split(", ")
+            if args.fields_with_fandom is not None
+            else []
+        )
+        tags.populate_tag_table(
+            args.temp_db_database,
+            "id",
+            story_table_name,
+            tag_columns_dict,
+            fields_with_fandom,
+        )
+        tags.populate_tag_table(
+            args.temp_db_database,
+            "id",
+            bookmark_table_name,
+            tag_columns_dict,
+            fields_with_fandom,
+            False,
+        )
+
+    log.info("Done extracting tags.")