Merge branch 'check-option-strict-pep8' of https://github.com/un-poga…

…z/calibre

pyproject.toml

@@ -9,7 +9,8 @@ include = ['*.py', '*.recipe']
 exclude = [
     "*_ui.py",
     "bypy/*",
-    "setup/*",
+    "setup/polib.py",
+    "setup/linux-installer.py",
     "src/css_selectors/*",
     "src/polyglot/*",
     "src/templite/*",
@@ -20,7 +21,7 @@ exclude = [
 quote-style = 'single'
 
 [tool.ruff.lint]
-ignore = ['E402', 'E722', 'E741', 'E401']
+ignore = ['E402', 'E722', 'E741']
 select = ['E', 'F', 'I', 'W', 'INT']
 
 [tool.ruff.lint.per-file-ignores]

ruff-strict-pep8.toml

@@ -0,0 +1,81 @@
+line-length = 160
+target-version = 'py38'
+builtins = ['_', 'I', 'P']
+include = ['*.py', '*.recipe']
+exclude = [
+    "*_ui.py",
+    "bypy/*",
+    "setup/polib.py",
+    "setup/linux-installer.py",
+    "src/css_selectors/*",
+    "src/polyglot/*",
+    "src/templite/*",
+    "src/tinycss/*",
+]
+
+preview = true
+
+[format]
+quote-style = 'single'
+
+[lint]
+explicit-preview-rules = true
+ignore = [
+    'E402', 'E722', 'E741',
+    'UP012', 'UP030', 'UP031', 'UP032', 'C413', 'C420', 'PIE790', 'ISC003',
+    'RUF001', 'RUF002', 'RUF003', 'RUF005', 'RUF012', 'RUF013', 'RUF015', 'RUF100',
+    'F841',  # because in preview, unused tuple unpacking variable that not use dummy syntax (prefix '_' underscore)
+             # raise error 'unused-variable', sigh (https://github.com/astral-sh/ruff/issues/8884)
+]
+select = [
+    'E', 'F', 'I', 'W', 'INT',
+    'UP', 'YTT', 'TID', 'Q', 'C4', 'COM818', 'PIE', 'RET501', 'ISC',
+    'RUF',  # nota: RUF can flag many unsolicited errors
+    # preview rules
+    'RUF051', 'RUF056',
+    'RUF039', 'RUF055',  # always use raw-string for regex
+    'E302', 'E303', 'E304', 'E305', 'W391',  # blank-line standard
+    'E111', 'E112', 'E113', 'E117',  # code indentation
+    'E114', 'E115', 'E116', 'E261', 'E262', 'E265',  # comment formating
+    'E201', 'E202', 'E211', 'E251', 'E275', # + partial: 'E203', 'E222', 'E241', 'E271', 'E272' # various whitespace
+]
+unfixable = ['ISC001']
+
+
+[lint.per-file-ignores]
+"recipes/*" = ['UP']
+"manual/plugin_examples/*" = ['UP']
+"src/calibre/customize/__init__.py" = ['RET501']
+"src/calibre/devices/interface.py" = ['RET501']
+"src/calibre/devices/kobo/driver.py" = ['E116']
+"src/calibre/ebooks/unihandecode/*codepoints.py" = ['E501']
+"src/calibre/ebooks/metadata/sources/*" = ['UP']
+"src/calibre/ebooks/metadata/sources/base.py" = ['RET501']
+"src/calibre/ebooks/pdf/reflow.py" = ['E114']
+"src/calibre/gui2/store/stores/*" = ['UP']
+"src/calibre/utils/copy_files.py" = ['UP037']
+"src/calibre/utils/smartypants.py" = ['RUF039', 'RUF055']
+"src/calibre/web/feeds/news.py" = ['RET501']
+"src/qt/*.py" = ['I', 'E302']
+"src/qt/*.pyi" = ['I']
+
+[lint.isort]
+detect-same-package = true
+extra-standard-library = ["aes", "elementmaker", "encodings"]
+known-first-party = ["calibre_extensions", "calibre_plugins", "polyglot"]
+known-third-party = ["odf", "qt", "templite", "tinycss", "css_selectors"]
+relative-imports-order = "closest-to-furthest"
+split-on-trailing-comma = false
+section-order = ['__python__', "future", "standard-library", "third-party", "first-party", "local-folder"]
+
+[lint.isort.sections]
+'__python__' = ['__python__']
+
+[lint.flake8-comprehensions]
+allow-dict-calls-with-keyword-arguments = true
+
+[lint.flake8-quotes]
+avoid-escape = true
+docstring-quotes = 'single'
+inline-quotes = 'single'
+multiline-quotes = 'single'

setup/check.py

@@ -23,44 +23,50 @@ def __str__(self):
         return f'{self.filename}:{self.lineno}: {self.msg}'
 
 
+def get_ruff_config(is_strict_check):
+    if is_strict_check:
+        return ['--config=ruff-strict-pep8.toml']
+    else:
+        return []
+
+
+def files_walker(root_path, ext):
+    for x in os.walk(root_path):
+        for f in x[-1]:
+            y = os.path.join(x[0], f)
+            if f.endswith(ext):
+                yield y
+
+
 def checkable_python_files(SRC):
     for dname in ('odf', 'calibre'):
-        for x in os.walk(os.path.join(SRC, dname)):
-            for f in x[-1]:
-                y = os.path.join(x[0], f)
-                if (f.endswith('.py') and f not in (
-                        'dict_data.py', 'unicodepoints.py', 'krcodepoints.py',
-                        'jacodepoints.py', 'vncodepoints.py', 'zhcodepoints.py') and
-                        'prs500/driver.py' not in y) and not f.endswith('_ui.py'):
-                    yield y
+        for f in files_walker(os.path.join(SRC, dname), '.py'):
+            if not f.endswith('_ui.py'):
+                yield f
 
 
 class Check(Command):
 
     description = 'Check for errors in the calibre source code'
 
     CACHE = 'check.json'
+    CACHE_STRICT = 'check-strict.json'
 
     def add_options(self, parser):
         parser.add_option('--fix', '--auto-fix', default=False, action='store_true',
                 help='Try to automatically fix some of the smallest errors')
         parser.add_option('--pep8', '--pep8-commit', default=False, action='store_true',
                 help='Try to automatically fix some of the smallest errors, then perform a pep8 commit')
+        parser.add_option('--strict', '--strict-pep8', default=False, action='store_true',
+                help='Perform the checking more strictely. See the file "strict-pep8.toml"')
 
     def get_files(self):
         yield from checkable_python_files(self.SRC)
 
-        for x in os.walk(self.j(self.d(self.SRC), 'recipes')):
-            for f in x[-1]:
-                f = self.j(x[0], f)
-                if f.endswith('.recipe'):
-                    yield f
-
-        for x in os.walk(self.j(self.SRC, 'pyj')):
-            for f in x[-1]:
-                f = self.j(x[0], f)
-                if f.endswith('.pyj'):
-                    yield f
+        yield from files_walker(self.j(self.d(self.SRC), 'recipes'), '.recipe')
+
+        yield from files_walker(self.j(self.SRC, 'pyj'), '.pyj')
+
         if self.has_changelog_check:
             yield self.j(self.d(self.SRC), 'Changelog.txt')
 
@@ -80,16 +86,16 @@ def is_cache_valid(self, f, cache):
 
     @property
     def cache_file(self):
-        return self.j(build_cache_dir(), self.CACHE)
+        return self.j(build_cache_dir(), self.CACHE_STRICT if self.is_strict_check else self.CACHE)
 
     def save_cache(self, cache):
         dump_json(cache, self.cache_file)
 
     def file_has_errors(self, f):
         ext = os.path.splitext(f)[1]
         if ext in {'.py', '.recipe'}:
-            p2 = subprocess.Popen(['ruff', 'check', f])
-            return p2.wait() != 0
+            p = subprocess.Popen(['ruff', 'check', f] + get_ruff_config(self.is_strict_check))
+            return p.wait() != 0
         if ext == '.pyj':
             p = subprocess.Popen(['rapydscript', 'lint', f])
             return p.wait() != 0
@@ -113,11 +119,15 @@ def run(self, opts):
         if opts.fix and opts.pep8:
             self.info('setup.py check: error: options --fix and --pep8 are mutually exclusive')
             raise SystemExit(2)
+        if opts.strict and opts.pep8:
+            self.info('setup.py check: error: options --strict and --pep8 are mutually exclusive')
+            raise SystemExit(2)
 
         self.fhash_cache = {}
         self.wn_path = os.path.expanduser('~/work/srv/main/static')
         self.has_changelog_check = os.path.exists(self.wn_path)
         self.auto_fix = opts.fix
+        self.is_strict_check = opts.strict
         if opts.pep8:
             self.run_pep8_commit()
         else:
@@ -167,11 +177,12 @@ def report_errors(self, errors):
             self.info('\t\t', str(err))
 
     def clean(self):
-        try:
-            os.remove(self.cache_file)
-        except OSError as err:
-            if err.errno != errno.ENOENT:
-                raise
+        for cache_file in [self.j(build_cache_dir(), self.CACHE), self.j(build_cache_dir(), self.CACHE_STRICT)]:
+            try:
+                os.remove(cache_file)
+            except OSError as err:
+                if err.errno != errno.ENOENT:
+                    raise
 
 
 class UpgradeSourceCode(Command):