diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..dba2fed --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + strategy: + matrix: + python_version: ["3.8", "3.9", "3.10", "3.11"] + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + python -m pip install tox==4.* tox-gh-actions==3.* + - name: Run tox + run: tox + - name: System test + env: + SETUPTOOLS_SCM_PRETEND_VERSION: "0.0.1" + run: ./tests/system_test/system_test.sh diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..487c3d8 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,19 @@ +# Thanks to: Sean Hammond +# https://www.seanh.cc/2022/05/21/publishing-python-packages-from-github-actions +name: Publish to PyPI.org +on: + release: + types: [published] +jobs: + pypi: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: python3 -m pip install --upgrade build && python3 -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index 68bc17f..2dc53ca 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..840f20e --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +# pytest-litter + +Pytest plugin that, when installed, will fail test cases which +create or delete files. Tests should not modify the file tree, +because it can be a cause of test pollution as well as accidental +committing of files to the repo. + +To use it, simply run +``` +pip install pytest-litter +``` +The only dependency is `pytest` itself. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..100b1be --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,130 @@ +[project] +name = "pytest-litter" +description = "Pytest plugin which verifies that tests do not modify file trees." +readme = "README.md" +license = {file = "LICENSE"} +urls = {repo = "https://github.com/mam-dev/pytest-litter"} +requires-python = ">=3.8" +dependencies = ["pytest >= 6.1"] +dynamic = ["version"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Testing :: Unit", + "Framework :: Pytest", + "License :: OSI Approved :: Apache Software License", +] + +[project.entry-points.pytest11] +pytest-litter = "pytest_litter.plugin.plugin" + +[build-system] +requires = ["setuptools>=51", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false + +[tool.pytest.ini_options] +addopts = "--random-order -p no:pytest-litter -p pytester -vv" +testpaths = ["tests"] +pytester_example_dir = "tests/pytester" + +[tool.coverage.run] +branch = true +source_pkgs = ["pytest_litter"] + +[tool.coverage.report] +show_missing = true +fail_under = 100 + +[tool.mypy] +files = ["src", "tests"] +warn_no_return = true +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true +warn_redundant_casts = true +warn_unreachable = true +check_untyped_defs = true +disallow_any_generics = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +no_implicit_reexport = true +strict_equality = true +strict_concatenate = true + +[tool.ruff] +src = ["src", "tests"] +select = [ + "E", # pycodestyle + "F", # pyflakes + "UP", # pyupgrade + "S", # flake8-bandit + "D", # pydocstyle + "PT", # flake8-pytest-style + "I", # isort + "RUF", # Ruff-specific rules + "PTH", # flake8-use-pathlib + "ERA", # eradicate + "PL", # pylint + "FBT", # flake8-boolean-trap + "B", # flake8-bugbear + "A", # flake8-builtins + "ISC", # flake8-implicit-str-concat + "INP", # flake8-no-pep420 + "SLF", # flake8-self + "SIM", # flake8-simplify + "TID", # flake8-tidy-imports + "ARG", # flake8-unused-arguments + "TRY", # tryceratops + "FLY", # flynt + "RSE", # flake8-raise + "RET", # flake8-return + "FIX", # flake8-fixme + "Q", # flake8-quotes + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez + "T10", # flake8-debugger + "T20", # flake8-print + "TCH", # flake8-type-checking +] +ignore = [ + "D100", "D102", "D103", "D104", "D105", "D107", + "PTH123", + "TRY003", "TRY301", +] + +[tool.ruff.per-file-ignores] +"tests/**/*.py" = [ + "S101", "S105", + "D103", + "FBT001", + "SLF001", + "PLR2004", "PLR0913", + "ARG001", + "INP001", +] + +[tool.ruff.pydocstyle] +convention = "google" + +[tool.ruff.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false +parametrize-names-type = "csv" +parametrize-values-type = "list" + +[tool.ruff.flake8-tidy-imports] +ban-relative-imports = "parents" diff --git a/requirements-lint.txt b/requirements-lint.txt new file mode 100644 index 0000000..e073daa --- /dev/null +++ b/requirements-lint.txt @@ -0,0 +1,3 @@ +ruff +black +mypy diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..f77f785 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +pytest +coverage>=5.3 +pytest-random-order +pytest-integration diff --git a/src/pytest_litter/__init__.py b/src/pytest_litter/__init__.py new file mode 100644 index 0000000..a2eb8b2 --- /dev/null +++ b/src/pytest_litter/__init__.py @@ -0,0 +1 @@ +"""Pytest plugin which verifies that tests do not modify file trees.""" diff --git a/src/pytest_litter/plugin/__init__.py b/src/pytest_litter/plugin/__init__.py new file mode 100644 index 0000000..6efbfe8 --- /dev/null +++ b/src/pytest_litter/plugin/__init__.py @@ -0,0 +1 @@ +"""Pytest plugin code.""" diff --git a/src/pytest_litter/plugin/plugin.py b/src/pytest_litter/plugin/plugin.py new file mode 100644 index 0000000..4e9be9c --- /dev/null +++ b/src/pytest_litter/plugin/plugin.py @@ -0,0 +1,44 @@ +"""Pytest plugin code.""" + +from typing import Optional + +import pytest + +from pytest_litter.plugin.utils import ( + COMPARATOR_KEY, + SNAPSHOT_KEY, + raise_test_error_from_comparison, + run_snapshot_comparison, +) +from pytest_litter.snapshots import ( + DirectoryIgnoreSpec, + IgnoreSpec, + RegexIgnoreSpec, + SnapshotComparator, + TreeSnapshot, +) + + +def pytest_configure(config: pytest.Config) -> None: + """Configure pytest-litter plugin (pytest hook function).""" + ignore_specs: list[IgnoreSpec] = [] + basetemp: Optional[str] = config.getoption("basetemp", None) + if basetemp is not None: + ignore_specs.append( + DirectoryIgnoreSpec( + directory=config.rootpath / basetemp, + ) + ) + ignore_specs.append(RegexIgnoreSpec(regex=r".*/__pycache__.*")) + config.stash[SNAPSHOT_KEY] = TreeSnapshot(root=config.rootpath) + config.stash[COMPARATOR_KEY] = SnapshotComparator(ignore_specs=ignore_specs) + + +@pytest.hookimpl(hookwrapper=True) # type: ignore[misc] +def pytest_runtest_call(item: pytest.Item): # type: ignore[no-untyped-def] + yield + run_snapshot_comparison( + test_name=item.name, + config=item.config, + mismatch_cb=raise_test_error_from_comparison, + ) diff --git a/src/pytest_litter/plugin/utils.py b/src/pytest_litter/plugin/utils.py new file mode 100644 index 0000000..343d145 --- /dev/null +++ b/src/pytest_litter/plugin/utils.py @@ -0,0 +1,59 @@ +from pathlib import Path +from typing import Callable, Iterable + +import pytest + +from pytest_litter.snapshots import SnapshotComparator, SnapshotComparison, TreeSnapshot + +SNAPSHOT_KEY = pytest.StashKey[TreeSnapshot]() +COMPARATOR_KEY = pytest.StashKey[SnapshotComparator]() + + +class ProblematicTestLitterError(Exception): + """Raised when a test causes littering, i.e., modifies file tree.""" + + +def format_test_snapshot_mismatch_message( + test_name: str, paths_added: Iterable[Path], paths_deleted: Iterable[Path] +) -> str: + def _iterable_to_human_readable(iterable: Iterable[Path]) -> str: + return ", ".join(f"'{x}'" for x in iterable) + + message = f"The test '{test_name}'" + if paths_added: + message += f" added {_iterable_to_human_readable(paths_added)}" + if paths_deleted: + message += " and" + if paths_deleted: + message += f" deleted {_iterable_to_human_readable(paths_deleted)}" + return message + + +def raise_test_error_from_comparison( + test_name: str, comparison: SnapshotComparison +) -> None: + """Raise ProblematicTestLitterError for test_name based on comparison.""" + raise ProblematicTestLitterError( + format_test_snapshot_mismatch_message( + test_name=test_name, + paths_added=tuple(p.path for p in comparison.only_b), + paths_deleted=tuple(p.path for p in comparison.only_a), + ) + ) + + +def run_snapshot_comparison( + test_name: str, + config: pytest.Config, + mismatch_cb: Callable[[str, SnapshotComparison], None], +) -> None: + """Compare current and old snapshots and call mismatch_cb if there is a mismatch.""" + original_snapshot: TreeSnapshot = config.stash[SNAPSHOT_KEY] + new_snapshot: TreeSnapshot = TreeSnapshot(root=original_snapshot.root) + config.stash[SNAPSHOT_KEY] = new_snapshot + + comparator: SnapshotComparator = config.stash[COMPARATOR_KEY] + comparison: SnapshotComparison = comparator.compare(original_snapshot, new_snapshot) + + if not comparison.matches: + mismatch_cb(test_name, comparison) diff --git a/src/pytest_litter/snapshots.py b/src/pytest_litter/snapshots.py new file mode 100644 index 0000000..adb3b99 --- /dev/null +++ b/src/pytest_litter/snapshots.py @@ -0,0 +1,171 @@ +"""Module related to taking and comparing snapshots of directory trees.""" +import abc +import re +from collections.abc import Iterable +from pathlib import Path +from typing import Any, Optional, Union + + +class UnexpectedLitterError(Exception): + """Error that should not occur normally, indicative of some programming error.""" + + +class PathSnapshot: + """A snapshot of a path.""" + + __slots__ = ("_path",) + + def __init__(self, path: Path) -> None: + self._path: Path = path + + def __str__(self) -> str: + return str(self._path) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self._path})" + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, PathSnapshot): + return False + return str(self) == str(other) + + def __hash__(self) -> int: + return hash(str(self._path)) + + @property + def path(self) -> Path: + return self._path + + +class TreeSnapshot: + """A snapshot of a directory tree.""" + + __slots__ = ("_root", "_paths") + + def __init__(self, root: Path) -> None: + """Initialize. + + Args: + root: The root directory of the tree. + + """ + if not root.is_dir(): + raise UnexpectedLitterError(f"'{root}' is not a directory.") + self._root: Path = root + self._paths: frozenset[PathSnapshot] = frozenset( + PathSnapshot(path=path) for path in self._root.rglob("*") + ) + + @property + def root(self) -> Path: + """The root directory of the tree.""" + return self._root + + @property + def paths(self) -> frozenset[PathSnapshot]: + """The paths in the snapshot.""" + return self._paths + + +class SnapshotComparison: + """A comparison of two TreeSnapshots.""" + + __slots__ = ("_only_a", "_only_b") + + def __init__( + self, only_a: Iterable[PathSnapshot], only_b: Iterable[PathSnapshot] + ) -> None: + """Initialize. + + Args: + only_a: Paths found only in snapshot A. + only_b: Paths found only in snapshot B. + + """ + self._only_a: frozenset[PathSnapshot] = frozenset(only_a) + self._only_b: frozenset[PathSnapshot] = frozenset(only_b) + + @property + def only_a(self) -> frozenset[PathSnapshot]: + """Paths found only in snapshot A.""" + return self._only_a + + @property + def only_b(self) -> frozenset[PathSnapshot]: + """Paths found only in snapshot B.""" + return self._only_b + + @property + def matches(self) -> bool: + """If snapshots A and B match each other.""" + return not self._only_a and not self._only_b + + +class IgnoreSpec(abc.ABC): + """Specification about paths to ignore in comparisons.""" + + __slots__ = () + + @abc.abstractmethod + def matches(self, path: PathSnapshot) -> bool: + ... # pragma: no cover + + +class DirectoryIgnoreSpec(IgnoreSpec): + """Specification to ignore everything in a given directory.""" + + __slots__ = ("_directory",) + + def __init__(self, directory: Path) -> None: + self._directory = directory + + def matches(self, path: PathSnapshot) -> bool: + return self._directory == path.path or self._directory in path.path.parents + + +class RegexIgnoreSpec(IgnoreSpec): + """Regex-based specification about paths to ignore in comparisons.""" + + __slots__ = ("_regex",) + + def __init__(self, regex: Union[str, re.Pattern[str]]) -> None: + self._regex: re.Pattern[str] = re.compile(regex) + + def matches(self, path: PathSnapshot) -> bool: + return self._regex.fullmatch(str(path)) is not None + + +class SnapshotComparator: + """Compare TreeSnapshots with each other.""" + + __slots__ = ("_ignore_specs",) + + def __init__(self, ignore_specs: Optional[Iterable[IgnoreSpec]] = None) -> None: + """Initialize. + + Args: + ignore_specs: Glob patterns to ignore when doing the comparison. + + """ + self._ignore_specs: frozenset[IgnoreSpec] = frozenset(ignore_specs or []) + + def _should_be_ignored(self, path: PathSnapshot) -> bool: + return any(ignore_spec.matches(path) for ignore_spec in self._ignore_specs) + + def compare( + self, snapshot_a: TreeSnapshot, snapshot_b: TreeSnapshot + ) -> SnapshotComparison: + """Compare snapshot_a and snapshot_b to produce a SnapshotComparison.""" + if snapshot_a.root != snapshot_b.root: + raise UnexpectedLitterError( + f"Comparing a snapshot of {snapshot_a.root} vs one of {snapshot_b.root}" + ) + common_paths: frozenset[PathSnapshot] = snapshot_a.paths.intersection( + snapshot_b.paths + ) + only_in_a: frozenset[PathSnapshot] = snapshot_a.paths - common_paths + only_in_b: frozenset[PathSnapshot] = snapshot_b.paths - common_paths + return SnapshotComparison( + only_a=(path for path in only_in_a if not self._should_be_ignored(path)), + only_b=(path for path in only_in_b if not self._should_be_ignored(path)), + ) diff --git a/tests/pytester/pytest.ini b/tests/pytester/pytest.ini new file mode 100644 index 0000000..56dc52d --- /dev/null +++ b/tests/pytester/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +required_plugins = "pytest-litter" +python_files = "pytester_*.py" +python_functions = "pytester_*" +testpaths = "." +addopts = + "-p pytest-litter" diff --git a/tests/pytester/pytester_tests.py b/tests/pytester/pytester_tests.py new file mode 100644 index 0000000..51184d5 --- /dev/null +++ b/tests/pytester/pytester_tests.py @@ -0,0 +1,24 @@ +"""Tests using pytest-litter intended for testing with pytester.""" +from pathlib import Path + +import pytest + + +def pytester_should_pass() -> None: + """Do not create any litter.""" + + +def pytester_should_fail() -> None: + """Create a file 'litter' in cwd which is not cleaned up.""" + (Path.cwd() / "litter").touch() + + +def pytester_should_also_pass(tmp_path: Path) -> None: + """Create litter in tmp_path only.""" + (tmp_path / "litter").touch() + + +@pytest.mark.xfail +def pytester_should_also_fail() -> None: + """Create a file 'more_litter' in cwd which is not cleaned up.""" + (Path.cwd() / "more_litter").touch() diff --git a/tests/system_test/system_test.sh b/tests/system_test/system_test.sh new file mode 100755 index 0000000..6eb7a19 --- /dev/null +++ b/tests/system_test/system_test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +echo "Preparing system test..." +WORK_DIR=$(mktemp --directory) +THIS_DIR="$(dirname "$0")" +TEST_DIR="${THIS_DIR}/../pytester" +cp "${TEST_DIR}/*" "${WORK_DIR}" +pushd "${WORK_DIR}" &> /dev/null || exit 1 +VENV="${WORK_DIR}/venv" +python -m venv "${VENV}" +. "${VENV}/bin/activate" +"${VENV}/bin/python3" -m pip install --quiet --upgrade pip +"${VENV}/bin/python3" -m pip install --quiet --editable "$(git rev-parse --show-toplevel)" || exit 1 + +echo "Executing system test..." +pytest -p pytest-litter --basetemp=tmp || exit 1 + +popd &> /dev/null || exit 1 + +echo "System test passed!" diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 0000000..ddc52d8 --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,222 @@ +"""Tests for the plugin module.""" +from pathlib import Path +from typing import TYPE_CHECKING, Iterable, Optional +from unittest.mock import Mock, call + +import pytest + +from pytest_litter import snapshots +from pytest_litter.plugin import plugin, utils + +if TYPE_CHECKING: + from _pytest.monkeypatch import MonkeyPatch + + +@pytest.mark.parametrize( + "test_name, paths_added, paths_deleted, expected_msg", + [ + ("test_fake", [Path("new")], [], "The test 'test_fake' added 'new'"), + ( + "test_madeup", + [Path("new"), Path("newer"), Path("newest")], + [], + "The test 'test_madeup' added 'new', 'newer', 'newest'", + ), + ( + "test_fake", + [Path("new")], + [Path("gone")], + "The test 'test_fake' added 'new' and deleted 'gone'", + ), + ( + "test_fake", + [Path("new"), Path("newer")], + [Path("gone"), Path("deleted")], + "The test 'test_fake' added 'new', 'newer' and deleted 'gone', 'deleted'", + ), + ( + "test_fake", + [], + [Path("gone")], + "The test 'test_fake' deleted 'gone'", + ), + ], +) +def test_format_test_snapshot_mismatch_message( + test_name: str, + paths_added: Iterable[Path], + paths_deleted: Iterable[Path], + expected_msg: str, +) -> None: + actual_msg = utils.format_test_snapshot_mismatch_message( + test_name=test_name, + paths_added=paths_added, + paths_deleted=paths_deleted, + ) + assert actual_msg == expected_msg + + +def test_raise_test_error_from_comparison( + monkeypatch: "MonkeyPatch", tmp_path: Path +) -> None: + test_name = "test_fake" + path_a = tmp_path / "a" + path_b = tmp_path / "b" + fake_error_msg = "fake message" + mock_format = Mock( + spec=utils.format_test_snapshot_mismatch_message, + return_value=fake_error_msg, + ) + monkeypatch.setattr( + "pytest_litter.plugin.utils.format_test_snapshot_mismatch_message", + mock_format, + ) + mock_comparison = Mock( + spec=snapshots.SnapshotComparison, + test_name=test_name, + only_a=[Mock(spec=snapshots.PathSnapshot, path=path_a)], + only_b=[Mock(spec=snapshots.PathSnapshot, path=path_b)], + ) + + with pytest.raises(utils.ProblematicTestLitterError, match=fake_error_msg): + utils.raise_test_error_from_comparison( + test_name=test_name, + comparison=mock_comparison, + ) + mock_format.assert_called_once_with( + test_name=test_name, + paths_added=(path_b,), + paths_deleted=(path_a,), + ) + + +@pytest.mark.parametrize("matches", [False, True]) +def test_run_snapshot_comparison( + monkeypatch: "MonkeyPatch", + matches: bool, +) -> None: + test_name = "test_fake" + mock_snapshot_old = Mock(spec=snapshots.TreeSnapshot, root=Path("fake")) + mock_comparator = Mock(spec=snapshots.SnapshotComparator) + mock_comparator.compare.return_value = Mock( + spec=snapshots.SnapshotComparison, matches=matches + ) + mock_config = Mock( + spec=pytest.Config, + stash={ + utils.SNAPSHOT_KEY: mock_snapshot_old, + utils.COMPARATOR_KEY: mock_comparator, + }, + ) + mock_snapshot_cls = Mock(spec=snapshots.TreeSnapshot) + monkeypatch.setattr( + "pytest_litter.plugin.utils.TreeSnapshot", + mock_snapshot_cls, + ) + mock_cb = Mock() + + def fake_cb(tc: str, comparison: snapshots.SnapshotComparison) -> None: + mock_cb(tc, comparison) + + utils.run_snapshot_comparison( + test_name=test_name, + config=mock_config, + mismatch_cb=fake_cb, + ) + + mock_snapshot_cls.assert_called_once_with(root=mock_snapshot_old.root) + assert mock_config.stash[utils.SNAPSHOT_KEY] is mock_snapshot_cls.return_value + mock_comparator.compare.assert_called_once_with( + mock_snapshot_old, + mock_snapshot_cls.return_value, + ) + if matches: + mock_cb.assert_not_called() + else: + mock_cb.assert_called_once_with(test_name, mock_comparator.compare.return_value) + + +@pytest.mark.parametrize("basetemp", [None, Path("tmp")]) +def test_pytest_configure(monkeypatch: "MonkeyPatch", basetemp: Optional[Path]) -> None: + mock_snapshot_cls = Mock(spec=snapshots.TreeSnapshot) + monkeypatch.setattr( + "pytest_litter.plugin.plugin.TreeSnapshot", + mock_snapshot_cls, + ) + mock_comparator_cls = Mock(spec=snapshots.SnapshotComparator) + monkeypatch.setattr( + "pytest_litter.plugin.plugin.SnapshotComparator", + mock_comparator_cls, + ) + mock_config = Mock( + spec=pytest.Config, + rootpath=Path("rootpath"), + stash={}, + getoption=Mock(spec=pytest.Config.getoption, return_value=basetemp), + ) + mock_dir_ignore_spec = Mock(spec=snapshots.DirectoryIgnoreSpec) + monkeypatch.setattr( + "pytest_litter.plugin.plugin.DirectoryIgnoreSpec", + mock_dir_ignore_spec, + ) + mock_regex_ignore_spec = Mock(spec=snapshots.RegexIgnoreSpec) + monkeypatch.setattr( + "pytest_litter.plugin.plugin.RegexIgnoreSpec", + mock_regex_ignore_spec, + ) + expected_ignore_specs = [] + if basetemp is not None: + expected_ignore_specs.append(mock_dir_ignore_spec.return_value) + expected_ignore_specs.append(mock_regex_ignore_spec.return_value) + + plugin.pytest_configure(mock_config) + + mock_config.getoption.assert_called_once_with("basetemp", None) + assert mock_config.stash[utils.SNAPSHOT_KEY] is mock_snapshot_cls.return_value + mock_snapshot_cls.assert_called_once_with(root=mock_config.rootpath) + assert mock_config.stash[utils.COMPARATOR_KEY] is mock_comparator_cls.return_value + mock_comparator_cls.assert_called_once_with( + ignore_specs=expected_ignore_specs, + ) + if basetemp is not None: + mock_dir_ignore_spec.assert_has_calls( + [call(directory=mock_config.rootpath / basetemp)] + ) + else: + mock_dir_ignore_spec.assert_not_called() + mock_regex_ignore_spec.assert_has_calls([call(regex=r".*/__pycache__.*")]) + + +def test_pytest_runtest_call(monkeypatch: "MonkeyPatch") -> None: + mock_raise_test_error = Mock(spec=utils.raise_test_error_from_comparison) + monkeypatch.setattr( + "pytest_litter.plugin.plugin.raise_test_error_from_comparison", + mock_raise_test_error, + ) + mock_run_comparison = Mock(spec=utils.run_snapshot_comparison) + monkeypatch.setattr( + "pytest_litter.plugin.plugin.run_snapshot_comparison", + mock_run_comparison, + ) + mock_item = Mock(spec=pytest.Item) + mock_item.name = "test_fake" + mock_item.config = Mock(spec=pytest.Config) + + # The list comprehension is just to get past the yield statement# + _ = list(plugin.pytest_runtest_call(mock_item)) + + mock_run_comparison.assert_called_once_with( + test_name=mock_item.name, + config=mock_item.config, + mismatch_cb=mock_raise_test_error, + ) + + +@pytest.mark.integration_test +def test_plugin_with_pytester(pytester: pytest.Pytester) -> None: + # pytester uses basetemp internally, so the case without basetemp + # cannot be tested using pytester. + pytester.copy_example("pytest.ini") + pytester.copy_example("pytester_tests.py") + result: pytest.RunResult = pytester.runpytest() + result.assert_outcomes(passed=2, failed=1, xfailed=1) diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py new file mode 100644 index 0000000..7b544aa --- /dev/null +++ b/tests/test_snapshots.py @@ -0,0 +1,185 @@ +"""Tests for the snapshots module.""" +import re +from collections.abc import Hashable, Iterable +from pathlib import Path +from typing import Union + +import pytest + +from pytest_litter import snapshots + + +@pytest.fixture(name="tmp_tree_root") +def fixture_tmp_tree_root(tmp_path: Path) -> Path: + root = tmp_path + subdir_a = root / "a" + subdir_a_1 = subdir_a / "a_1" + subdir_b = root / "b" + subdir_a_1.mkdir(parents=True, exist_ok=True) + subdir_b.mkdir() + a_file = subdir_a / "a_file" + a_file.touch() + b_file = subdir_b / "b_file" + b_file.touch() + a_1_file = subdir_a_1 / "a_1_file" + a_1_file.touch() + return root + + +@pytest.fixture(name="tree_paths") +def fixture_tree_paths(tmp_tree_root: Path) -> list[snapshots.PathSnapshot]: + return [ + snapshots.PathSnapshot(path=tmp_tree_root / "a"), + snapshots.PathSnapshot(path=tmp_tree_root / "a" / "a_file"), + snapshots.PathSnapshot(path=tmp_tree_root / "b"), + snapshots.PathSnapshot(path=tmp_tree_root / "b" / "b_file"), + snapshots.PathSnapshot(path=tmp_tree_root / "a" / "a_1"), + snapshots.PathSnapshot(path=tmp_tree_root / "a" / "a_1" / "a_1_file"), + ] + + +def test_path_snapshot(tmp_tree_root: Path) -> None: + input_path = tmp_tree_root / "a" + snapshot = snapshots.PathSnapshot(path=input_path) + assert str(snapshot) == str(input_path) + assert str(input_path) in repr(snapshot) + assert isinstance(snapshot, Hashable) + assert hash(snapshot) == hash(str(input_path)) + assert snapshot != input_path + assert snapshot.path == input_path + + +def test_tree_snapshot__bad_root(tmp_path: Path) -> None: + input_path = tmp_path / "fake_file" + with pytest.raises(snapshots.UnexpectedLitterError): + _ = snapshots.TreeSnapshot(root=input_path) + + +def test_tree_snapshot( + tmp_tree_root: Path, tree_paths: list[snapshots.PathSnapshot] +) -> None: + snapshot = snapshots.TreeSnapshot(root=tmp_tree_root) + assert snapshot.root == tmp_tree_root + assert snapshot.paths == frozenset(tree_paths) + + +@pytest.mark.parametrize( + "only_a, only_b, expected_match", + [ + ([], [], True), + ((), (), True), + (set(), set(), True), + ([], [snapshots.PathSnapshot(path=Path("a"))], False), + ([snapshots.PathSnapshot(path=Path("a"))], [], False), + ( + [ + snapshots.PathSnapshot(path=Path("a")), + snapshots.PathSnapshot(path=Path("b")), + ], + [], + False, + ), + ], +) +def test_snapshot_comparison( + only_a: Iterable[snapshots.PathSnapshot], + only_b: Iterable[snapshots.PathSnapshot], + expected_match: bool, +) -> None: + comparison = snapshots.SnapshotComparison(only_a=only_a, only_b=only_b) + assert comparison.only_a == frozenset(only_a) + assert comparison.only_b == frozenset(only_b) + assert comparison.matches == expected_match + + +@pytest.mark.parametrize( + "directory, path, expected_match", + [ + (Path("subdir/a"), snapshots.PathSnapshot(path=Path("subdir/a")), True), + (Path("subdir/a"), snapshots.PathSnapshot(path=Path("subdir/b")), False), + (Path("subdir"), snapshots.PathSnapshot(path=Path("subdir/a")), True), + (Path("subdir"), snapshots.PathSnapshot(path=Path("subdir/subsub/b")), True), + ], +) +def test_directory_ignore_spec( + directory: Path, path: snapshots.PathSnapshot, expected_match: bool +) -> None: + ignore_spec = snapshots.DirectoryIgnoreSpec(directory=directory) + assert ignore_spec.matches(path=path) == expected_match + + +@pytest.mark.parametrize( + "regex, path, expected_match", + [ + (r"sub\w+/a", snapshots.PathSnapshot(path=Path("subdir/a")), True), + (r"sub\w+/a", snapshots.PathSnapshot(path=Path("subdir/b")), False), + (re.compile(r"sub\w+/a"), snapshots.PathSnapshot(path=Path("subdir/a")), True), + (re.compile(r"sub\w+/a"), snapshots.PathSnapshot(path=Path("subdir/b")), False), + ], +) +def test_regex_ignore_spec( + regex: Union[str, re.Pattern[str]], + path: snapshots.PathSnapshot, + expected_match: bool, +) -> None: + ignore_spec = snapshots.RegexIgnoreSpec(regex=regex) + assert ignore_spec.matches(path=path) == expected_match + + +def test_snapshot_comparator__same(tmp_tree_root: Path) -> None: + snapshot = snapshots.TreeSnapshot(root=tmp_tree_root) + comparison = snapshots.SnapshotComparator().compare(snapshot, snapshot) + assert comparison.matches + assert not comparison.only_a + assert not comparison.only_b + + +def test_snapshot_comparator__only_b(tmp_tree_root: Path) -> None: + snapshot_a = snapshots.TreeSnapshot(root=tmp_tree_root) + new_file = tmp_tree_root / "new_file" + new_file.touch() + snapshot_b = snapshots.TreeSnapshot(root=tmp_tree_root) + comparison = snapshots.SnapshotComparator().compare(snapshot_a, snapshot_b) + assert not comparison.matches + assert not comparison.only_a + assert comparison.only_b == frozenset((snapshots.PathSnapshot(path=new_file),)) + + +def test_snapshot_comparator__only_a(tmp_tree_root: Path) -> None: + snapshot_b = snapshots.TreeSnapshot(root=tmp_tree_root) + new_file = tmp_tree_root / "new_file" + new_file.touch() + snapshot_a = snapshots.TreeSnapshot(root=tmp_tree_root) + comparison = snapshots.SnapshotComparator().compare(snapshot_a, snapshot_b) + assert not comparison.matches + assert comparison.only_a == frozenset((snapshots.PathSnapshot(path=new_file),)) + assert not comparison.only_b + + +@pytest.mark.parametrize( + "ignore_spec", + [ + snapshots.RegexIgnoreSpec(regex=r".*/new_file\.\w+"), + ], +) +def test_snapshot_comparator__only_b__ignored( + tmp_tree_root: Path, ignore_spec: snapshots.IgnoreSpec +) -> None: + snapshot_a = snapshots.TreeSnapshot(root=tmp_tree_root) + (tmp_tree_root / "new_file.txt").touch() + (tmp_tree_root / "new_file.yml").touch() + snapshot_b = snapshots.TreeSnapshot(root=tmp_tree_root) + comparison = snapshots.SnapshotComparator(ignore_specs=[ignore_spec]).compare( + snapshot_a, snapshot_b + ) + assert comparison.matches + assert not comparison.only_a + assert not comparison.only_b + + +def test_snapshot_comparator__incompatible_snapshots(tmp_tree_root: Path) -> None: + snapshot_a = snapshots.TreeSnapshot(root=tmp_tree_root) + snapshot_b = snapshots.TreeSnapshot(root=tmp_tree_root / "a") + comparator = snapshots.SnapshotComparator() + with pytest.raises(snapshots.UnexpectedLitterError): + _ = comparator.compare(snapshot_a, snapshot_b) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0abc0a9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = py38,py39,py310,py311 +isolated_build = True +minversion = 4.0.0 + +[gh-actions] +python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + +[testenv] +deps = + -rrequirements-test.txt + -rrequirements-lint.txt +commands = + ruff check . + black --check src tests + coverage run -m pytest --debug=pathmap + coverage report + mypy