Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Transfer of fasteners dependency to filelock #4800

Merged
merged 54 commits into from
Jan 12, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
0d02218
Update pyproject.toml
talagayev Nov 22, 2024
833f58b
Update requirements.txt
talagayev Nov 22, 2024
0e45f80
Update action.yaml
talagayev Nov 22, 2024
0505cc5
Update action.yaml
talagayev Nov 22, 2024
ea5a61c
Update environment.yml
talagayev Nov 22, 2024
40251b6
Update azure-pipelines.yml
talagayev Nov 22, 2024
c98a11f
Update XDR.py
talagayev Nov 22, 2024
97927bb
Update XDR.py
talagayev Nov 23, 2024
f8f05f2
Update test_xdr.py
talagayev Nov 23, 2024
3cd7307
Update XDR.py
talagayev Nov 23, 2024
b66e3d2
Update test_xdr.py
talagayev Nov 23, 2024
e71943f
Update XDR.py
talagayev Nov 26, 2024
ce3656a
Update test_xdr.py
talagayev Nov 26, 2024
aa9f83f
Merge branch 'MDAnalysis:develop' into fasteners_to_filelock
talagayev Nov 27, 2024
307027b
Update test_xdr.py
talagayev Dec 8, 2024
77abba0
Update test_xdr.py
talagayev Dec 8, 2024
fc52a18
Update test_xdr.py
talagayev Dec 8, 2024
e1fe6fb
Update test_xdr.py
talagayev Dec 8, 2024
fbcd060
Update test_xdr.py
talagayev Dec 8, 2024
9c2ed87
Update test_xdr.py
talagayev Dec 8, 2024
2d3cf13
Update test_xdr.py
talagayev Dec 8, 2024
78b61e9
Update test_xdr.py
talagayev Dec 8, 2024
30e4561
Update test_xdr.py
talagayev Dec 8, 2024
947966c
Update test_xdr.py
talagayev Dec 8, 2024
b32b716
Merge branch 'MDAnalysis:develop' into fasteners_to_filelock
talagayev Dec 12, 2024
33c0ea4
Update CHANGELOG
talagayev Dec 12, 2024
088b867
Merge branch 'develop' into fasteners_to_filelock
orbeckst Dec 17, 2024
16132f2
Update test_xdr.py
talagayev Dec 17, 2024
b100232
Update test_xdr.py
talagayev Dec 17, 2024
8de3f27
Update test_xdr.py
talagayev Dec 17, 2024
8ca8893
Update test_xdr.py
talagayev Dec 18, 2024
86c3db2
Update test_xdr.py
talagayev Dec 19, 2024
c1436b9
Update test_xdr.py
talagayev Dec 20, 2024
1fd2374
Update test_xdr.py
talagayev Dec 20, 2024
b7150e2
Update test_xdr.py
talagayev Dec 20, 2024
b038a21
Update test_xdr.py
talagayev Dec 21, 2024
db46e40
Update test_xdr.py
talagayev Dec 21, 2024
497f14e
Update test_xdr.py
talagayev Dec 21, 2024
47bfb7c
Merge branch 'develop' into fasteners_to_filelock
yuxuanzhuang Jan 8, 2025
c271f6b
Update test_xdr.py
talagayev Jan 8, 2025
332f8ca
Update test_xdr.py
talagayev Jan 8, 2025
73e1d03
Update test_xdr.py
talagayev Jan 8, 2025
a6bb5fe
Update test_xdr.py
talagayev Jan 8, 2025
a5d0f3a
Update test_xdr.py
talagayev Jan 8, 2025
fe59638
Update test_xdr.py
talagayev Jan 9, 2025
d1df5ac
Update test_xdr.py
talagayev Jan 9, 2025
8bb1c77
Update test_xdr.py
talagayev Jan 9, 2025
42539bd
Update test_xdr.py
talagayev Jan 9, 2025
3c5cb66
check lock file exists for traj
yuxuanzhuang Jan 10, 2025
2fb9b69
use mock to test cannot write offsets
yuxuanzhuang Jan 10, 2025
40d09cc
Merge branch 'develop' into fasteners_to_filelock
talagayev Jan 10, 2025
9b8209a
skip lock file test for windows
yuxuanzhuang Jan 10, 2025
fd537cc
import sys
yuxuanzhuang Jan 10, 2025
7b580c1
test filelock permission error
yuxuanzhuang Jan 11, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/actions/setup-deps/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ inputs:
default: 'codecov'
cython:
default: 'cython'
fasteners:
default: 'fasteners'
filelock:
default: 'filelock'
griddataformats:
default: 'griddataformats'
gsd:
Expand Down Expand Up @@ -110,7 +110,7 @@ runs:
CONDA_MIN_DEPS: |
${{ inputs.codecov }}
${{ inputs.cython }}
${{ inputs.fasteners }}
${{ inputs.filelock }}
${{ inputs.griddataformats }}
${{ inputs.hypothesis }}
${{ inputs.matplotlib }}
Expand Down
2 changes: 1 addition & 1 deletion azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
scikit-learn
tqdm
threadpoolctl
fasteners
filelock
displayName: 'Install dependencies'
# for wheel install testing, we pin to an
# older NumPy, the oldest version we support and that
Expand Down
2 changes: 1 addition & 1 deletion maintainer/conda/environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ dependencies:
- codecov
- cython
- docutils
- fasteners
- filelock
- griddataformats
- gsd
- h5py>=2.10
Expand Down
1 change: 1 addition & 0 deletions package/CHANGELOG
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Enhancements


Changes
* Changed `fasteners` dependency to `filelock` (Issue #4797, PR #4800)
* Codebase is now formatted with black (version `24`) (PR #4886)

Deprecations
Expand Down
10 changes: 6 additions & 4 deletions package/MDAnalysis/coordinates/XDR.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
import numpy as np
from os.path import getctime, getsize, isfile, split, join
import warnings
import fasteners
from filelock import FileLock

from . import base
from ..lib.mdamath import triclinic_box
Expand Down Expand Up @@ -121,6 +121,8 @@ class XDRBaseReader(base.ReaderBase):
Add a InterProcessLock when generating offsets
.. versionchanged:: 2.4.0
Use a direct read into ts attributes
.. versionchanged:: 2.9.0
Changed fasteners.InterProcessLock() to filelock.FileLock
"""
@store_init_arguments
def __init__(self, filename, convert_units=True, sub=None,
Expand Down Expand Up @@ -195,18 +197,18 @@ def _load_offsets(self):

# check if the location of the lock is writable.
try:
with fasteners.InterProcessLock(lock_name) as filelock:
with FileLock(lock_name) as filelock:
pass
except OSError as e:
if isinstance(e, PermissionError) or e.errno == errno.EROFS:
warnings.warn(f"Cannot write lock/offset file in same location as "
f"{self.filename}. Using slow offset calculation.")
self._read_offsets(store=True)
self._read_offsets(store=False)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good!

(I think this also explain why you changed the check for a warning in the tests)

return
else:
raise

with fasteners.InterProcessLock(lock_name) as filelock:
with FileLock(lock_name) as filelock:
if not isfile(fname):
self._read_offsets(store=True)
return
Expand Down
2 changes: 1 addition & 1 deletion package/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dependencies = [
'tqdm>=4.43.0',
'threadpoolctl',
'packaging',
'fasteners',
'filelock',
'mda-xdrlib',
'waterdynamics',
'pathsimanalysis',
Expand Down
2 changes: 1 addition & 1 deletion package/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
biopython>=1.80
codecov
cython
fasteners
filelock
griddataformats
gsd
hypothesis
Expand Down
51 changes: 24 additions & 27 deletions testsuite/MDAnalysisTests/coordinates/test_xdr.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@

import re
import os
import sys
import shutil
import subprocess
import time
from pathlib import Path

import numpy as np
Expand Down Expand Up @@ -60,6 +62,7 @@
from MDAnalysis.coordinates.base import Timestep
from MDAnalysis.coordinates import XDR
from MDAnalysisTests.util import get_userid
from filelock import FileLock


@pytest.mark.parametrize(
Expand Down Expand Up @@ -977,45 +980,39 @@ def test_unsupported_format(self, traj):
reader = self._reader(traj)
reader[idx_frame]

@pytest.mark.skipif(get_userid() == 0, reason="cannot readonly as root")
def test_persistent_offsets_readonly(self, tmpdir):
def test_persistent_offsets_readonly(self, tmpdir, trajectory):
shutil.copy(self.filename, str(tmpdir))

if os.name == "nt":
# Windows platform has a unique way to deny write access
subprocess.call(
"icacls {fname} /deny Users:W".format(fname=tmpdir), shell=True
filename = str(tmpdir.join(os.path.basename(self.filename)))
print('filename', filename)
ref_offset = trajectory._xdr.offsets
# Mock filelock acquire to raise an error
with patch.object(FileLock, "acquire", side_effect=PermissionError): # Simulate failure
with pytest.warns(UserWarning, match="Cannot write lock"):
reader = self._reader(filename)
saved_offsets = reader._xdr.offsets

# Check if offsets are handled properly and match reference offsets
assert_almost_equal(
saved_offsets, # Compare with reference offsets
ref_offset,
err_msg="error loading frame offsets",
)
else:
os.chmod(str(tmpdir), 0o555)

filename = str(tmpdir.join(os.path.basename(self.filename)))
# try to write a offsets file
with pytest.warns(
UserWarning, match="Couldn't save offsets"
) and pytest.warns(UserWarning, match="Cannot write"):
self._reader(filename)
assert_equal(os.path.exists(XDR.offsets_filename(filename)), False)
# check the lock file is not created as well.
assert_equal(
os.path.exists(XDR.offsets_filename(filename, ending=".lock")),
False,
)

# pre-teardown permission fix - leaving permission blocked dir
# is problematic on py3.9 + Windows it seems. See issue
# [4123](https://github.com/MDAnalysis/mdanalysis/issues/4123)
# for more details.
if os.name == "nt":
subprocess.call(f"icacls {tmpdir} /grant Users:W", shell=True)
else:
os.chmod(str(tmpdir), 0o777)

shutil.rmtree(tmpdir)

def test_offset_lock_created(self):
@pytest.mark.skipif(
sys.platform.startswith("win"),
reason="The lock file only exists when it's locked in windows"
)
def test_offset_lock_created(self, traj):
assert os.path.exists(
XDR.offsets_filename(self.filename, ending="lock")
XDR.offsets_filename(traj, ending="lock")
)


Expand Down
Loading