From 0ee5c3f2ce3c95ca91a54de195c941ae9721fd3f Mon Sep 17 00:00:00 2001
From: Yadu Babuji <yadudoc1729@gmail.com>
Date: Mon, 26 Aug 2024 15:25:27 -0500
Subject: [PATCH 1/3] Marking tests that require a staging provider with
 @staging_required

---
 parsl/tests/test_staging/test_staging_ftp.py   | 1 +
 parsl/tests/test_staging/test_staging_https.py | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/parsl/tests/test_staging/test_staging_ftp.py b/parsl/tests/test_staging/test_staging_ftp.py
index 12becdf9c4..a004f5a575 100644
--- a/parsl/tests/test_staging/test_staging_ftp.py
+++ b/parsl/tests/test_staging/test_staging_ftp.py
@@ -15,6 +15,7 @@ def sort_strings(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_ftp():
     """Test staging for an ftp file
 
diff --git a/parsl/tests/test_staging/test_staging_https.py b/parsl/tests/test_staging/test_staging_https.py
index 4a68e66a5c..c977472249 100644
--- a/parsl/tests/test_staging/test_staging_https.py
+++ b/parsl/tests/test_staging/test_staging_https.py
@@ -48,6 +48,7 @@ def sort_strings_additional_executor(inputs=(), outputs=()):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_cleannet(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -68,6 +69,7 @@ def test_staging_https_local(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_kwargs(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -78,6 +80,7 @@ def test_staging_https_kwargs(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_args(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')

From 287d11cbb1aee62afa54e95a5aa5acaea4cd6e4f Mon Sep 17 00:00:00 2001
From: Yadu Babuji <yadudoc1729@gmail.com>
Date: Wed, 30 Oct 2024 13:50:00 -0500
Subject: [PATCH 2/3] Adding a new @shared_fs marker to mark tests that expect
 executors with shared filesystem access

---
 parsl/tests/conftest.py                                   | 4 ++++
 parsl/tests/test_bash_apps/test_basic.py                  | 3 +++
 parsl/tests/test_bash_apps/test_error_codes.py            | 4 ++++
 parsl/tests/test_bash_apps/test_kwarg_storage.py          | 1 +
 parsl/tests/test_bash_apps/test_memoize.py                | 8 ++------
 parsl/tests/test_bash_apps/test_memoize_ignore_args.py    | 3 +++
 .../tests/test_bash_apps/test_memoize_ignore_args_regr.py | 1 +
 parsl/tests/test_bash_apps/test_multiline.py              | 1 +
 parsl/tests/test_bash_apps/test_stdout.py                 | 3 +++
 parsl/tests/test_docs/test_from_slides.py                 | 3 +++
 parsl/tests/test_docs/test_kwargs.py                      | 3 +++
 parsl/tests/test_docs/test_workflow1.py                   | 1 +
 parsl/tests/test_python_apps/test_outputs.py              | 1 +
 parsl/tests/test_regression/test_226.py                   | 1 +
 parsl/tests/test_staging/test_docs_1.py                   | 1 +
 parsl/tests/test_staging/test_output_chain_filenames.py   | 3 +++
 parsl/tests/test_staging/test_staging_stdout.py           | 2 ++
 17 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/parsl/tests/conftest.py b/parsl/tests/conftest.py
index b8af73e4bf..63e5c45201 100644
--- a/parsl/tests/conftest.py
+++ b/parsl/tests/conftest.py
@@ -163,6 +163,10 @@ def pytest_configure(config):
         'markers',
         'executor_supports_std_stream_tuples: Marks tests that require tuple support for stdout/stderr'
     )
+    config.addinivalue_line(
+        'markers',
+        'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
+    )
 
 
 @pytest.fixture(autouse=True, scope='session')
diff --git a/parsl/tests/test_bash_apps/test_basic.py b/parsl/tests/test_bash_apps/test_basic.py
index 56d56d8ed1..0eea6d4d97 100644
--- a/parsl/tests/test_bash_apps/test_basic.py
+++ b/parsl/tests/test_bash_apps/test_basic.py
@@ -24,6 +24,7 @@ def foo(x, y, z=10, stdout=None, label=None):
     return f"echo {x} {y} {z}"
 
 
+@pytest.mark.shared_fs
 def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps"""
 
@@ -38,6 +39,7 @@ def test_command_format_1(tmpd_cwd):
     assert so_content == "1 4 10"
 
 
+@pytest.mark.shared_fs
 def test_auto_log_filename_format(caplog):
     """Testing auto log filename format for BashApps
     """
@@ -66,6 +68,7 @@ def test_auto_log_filename_format(caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.shared_fs
 def test_parallel_for(tmpd_cwd, n=3):
     """Testing a simple parallel for loop"""
     outdir = tmpd_cwd / "outputs/test_parallel"
diff --git a/parsl/tests/test_bash_apps/test_error_codes.py b/parsl/tests/test_bash_apps/test_error_codes.py
index 4a0b835728..bccded91a9 100644
--- a/parsl/tests/test_bash_apps/test_error_codes.py
+++ b/parsl/tests/test_bash_apps/test_error_codes.py
@@ -58,6 +58,7 @@ def bad_format(stderr='std.err', stdout='std.out'):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
+@pytest.mark.shared_fs
 def test_div_0(test_fn=div_0):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -73,6 +74,7 @@ def test_div_0(test_fn=div_0):
     os.remove('std.out')
 
 
+@pytest.mark.shared_fs
 def test_bash_misuse(test_fn=bash_misuse):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -87,6 +89,7 @@ def test_bash_misuse(test_fn=bash_misuse):
     os.remove('std.out')
 
 
+@pytest.mark.shared_fs
 def test_command_not_found(test_fn=command_not_found):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -103,6 +106,7 @@ def test_command_not_found(test_fn=command_not_found):
     return True
 
 
+@pytest.mark.shared_fs
 def test_not_executable(test_fn=not_executable):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
diff --git a/parsl/tests/test_bash_apps/test_kwarg_storage.py b/parsl/tests/test_bash_apps/test_kwarg_storage.py
index 8e0d48c661..e88a4c2967 100644
--- a/parsl/tests/test_bash_apps/test_kwarg_storage.py
+++ b/parsl/tests/test_bash_apps/test_kwarg_storage.py
@@ -8,6 +8,7 @@ def foo(z=2, stdout=None):
     return f"echo {z}"
 
 
+@pytest.mark.shared_fs
 def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps
     """
diff --git a/parsl/tests/test_bash_apps/test_memoize.py b/parsl/tests/test_bash_apps/test_memoize.py
index d53460b50b..387837f4d2 100644
--- a/parsl/tests/test_bash_apps/test_memoize.py
+++ b/parsl/tests/test_bash_apps/test_memoize.py
@@ -9,9 +9,7 @@ def fail_on_presence(outputs=()):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
+@pytest.mark.shared_fs
 def test_bash_memoization(tmpd_cwd, n=2):
     """Testing bash memoization
     """
@@ -29,9 +27,7 @@ def fail_on_presence_kw(outputs=(), foo=None):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
+@pytest.mark.shared_fs
 def test_bash_memoization_keywords(tmpd_cwd, n=2):
     """Testing bash memoization
     """
diff --git a/parsl/tests/test_bash_apps/test_memoize_ignore_args.py b/parsl/tests/test_bash_apps/test_memoize_ignore_args.py
index ee3917e561..16cb919f1a 100644
--- a/parsl/tests/test_bash_apps/test_memoize_ignore_args.py
+++ b/parsl/tests/test_bash_apps/test_memoize_ignore_args.py
@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 import parsl
 from parsl.app.app import bash_app
 
@@ -21,6 +23,7 @@ def no_checkpoint_stdout_app_ignore_args(stdout=None):
     return "echo X"
 
 
+@pytest.mark.shared_fs
 def test_memo_stdout(tmpd_cwd):
     path_x = tmpd_cwd / "test.memo.stdout.x"
 
diff --git a/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py b/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py
index 8f03c055a1..e1a9465fb3 100644
--- a/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py
+++ b/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py
@@ -29,6 +29,7 @@ def no_checkpoint_stdout_app(stdout=None):
     return "echo X"
 
 
+@pytest.mark.shared_fs
 def test_memo_stdout(tmpd_cwd):
     assert const_list_x == const_list_x_arg
 
diff --git a/parsl/tests/test_bash_apps/test_multiline.py b/parsl/tests/test_bash_apps/test_multiline.py
index 59fb5ed7b2..cfdb674e9d 100644
--- a/parsl/tests/test_bash_apps/test_multiline.py
+++ b/parsl/tests/test_bash_apps/test_multiline.py
@@ -14,6 +14,7 @@ def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
     """.format(inputs=inputs, outputs=outputs)
 
 
+@pytest.mark.shared_fs
 def test_multiline(tmpd_cwd):
     so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
     f = multiline(
diff --git a/parsl/tests/test_bash_apps/test_stdout.py b/parsl/tests/test_bash_apps/test_stdout.py
index eba6a7b80d..c8404451f6 100644
--- a/parsl/tests/test_bash_apps/test_stdout.py
+++ b/parsl/tests/test_bash_apps/test_stdout.py
@@ -34,6 +34,7 @@ def echo_to_streams(msg, stderr=None, stdout=None):
 ]
 
 
+@pytest.mark.shared_fs
 @pytest.mark.parametrize('spec', speclist, ids=testids)
 def test_bad_stdout_specs(spec):
     """Testing bad stdout spec cases"""
@@ -91,6 +92,7 @@ def test_bad_stderr_file():
 
 
 @pytest.mark.executor_supports_std_stream_tuples
+@pytest.mark.shared_fs
 def test_stdout_truncate(tmpd_cwd, caplog):
     """Testing truncation of prior content of stdout"""
 
@@ -110,6 +112,7 @@ def test_stdout_truncate(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.shared_fs
 def test_stdout_append(tmpd_cwd, caplog):
     """Testing appending to prior content of stdout (default open() mode)"""
 
diff --git a/parsl/tests/test_docs/test_from_slides.py b/parsl/tests/test_docs/test_from_slides.py
index b07092b4ae..b3242e813e 100644
--- a/parsl/tests/test_docs/test_from_slides.py
+++ b/parsl/tests/test_docs/test_from_slides.py
@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 
@@ -15,6 +17,7 @@ def cat(inputs=[]):
         return f.readlines()
 
 
+@pytest.mark.staging_required
 def test_slides():
     """Testing code snippet from slides """
 
diff --git a/parsl/tests/test_docs/test_kwargs.py b/parsl/tests/test_docs/test_kwargs.py
index 5524c0b819..80907ebe08 100644
--- a/parsl/tests/test_docs/test_kwargs.py
+++ b/parsl/tests/test_docs/test_kwargs.py
@@ -1,6 +1,8 @@
 """Functions used to explain kwargs"""
 from pathlib import Path
 
+import pytest
+
 from parsl import File, python_app
 
 
@@ -19,6 +21,7 @@ def reduce_app(inputs=()):
     assert reduce_future.result() == 6
 
 
+@pytest.mark.shared_fs
 def test_outputs(tmpd_cwd):
     @python_app()
     def write_app(message, outputs=()):
diff --git a/parsl/tests/test_docs/test_workflow1.py b/parsl/tests/test_docs/test_workflow1.py
index 271baab4d8..b0f92b6ab9 100644
--- a/parsl/tests/test_docs/test_workflow1.py
+++ b/parsl/tests/test_docs/test_workflow1.py
@@ -22,6 +22,7 @@ def save(message, outputs=[]):
     return 'echo {m} &> {o}'.format(m=message, o=outputs[0])
 
 
+@pytest.mark.shared_fs
 @pytest.mark.staging_required
 def test_procedural(N=2):
     """Procedural workflow example from docs on
diff --git a/parsl/tests/test_python_apps/test_outputs.py b/parsl/tests/test_python_apps/test_outputs.py
index c4b9dbabe2..b4273cf286 100644
--- a/parsl/tests/test_python_apps/test_outputs.py
+++ b/parsl/tests/test_python_apps/test_outputs.py
@@ -16,6 +16,7 @@ def double(x, outputs=[]):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
+@pytest.mark.shared_fs
 def test_launch_apps(tmpd_cwd, n=2):
     outdir = tmpd_cwd / "outputs"
     outdir.mkdir()
diff --git a/parsl/tests/test_regression/test_226.py b/parsl/tests/test_regression/test_226.py
index 2f560466dc..8babf4cbe0 100644
--- a/parsl/tests/test_regression/test_226.py
+++ b/parsl/tests/test_regression/test_226.py
@@ -53,6 +53,7 @@ def test_get_dataframe():
     assert res.equals(data), 'Unexpected dataframe'
 
 
+@pytest.mark.shared_fs
 def test_bash_default_arg():
     if os.path.exists('std.out'):
         os.remove('std.out')
diff --git a/parsl/tests/test_staging/test_docs_1.py b/parsl/tests/test_staging/test_docs_1.py
index 8f549ae9b3..cc6236076a 100644
--- a/parsl/tests/test_staging/test_docs_1.py
+++ b/parsl/tests/test_staging/test_docs_1.py
@@ -12,6 +12,7 @@ def convert(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.shared_fs
 def test():
     # create an remote Parsl file
     inp = File('ftp://ftp.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')
diff --git a/parsl/tests/test_staging/test_output_chain_filenames.py b/parsl/tests/test_staging/test_output_chain_filenames.py
index 016714497b..442bd6fd5a 100644
--- a/parsl/tests/test_staging/test_output_chain_filenames.py
+++ b/parsl/tests/test_staging/test_output_chain_filenames.py
@@ -1,5 +1,7 @@
 from concurrent.futures import Future
 
+import pytest
+
 from parsl import File
 from parsl.app.app import bash_app
 
@@ -14,6 +16,7 @@ def app2(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
     return f"echo '{inputs[0]}' > {outputs[0]}"
 
 
+@pytest.mark.shared_fs
 def test_behavior(tmpd_cwd):
     expected_path = str(tmpd_cwd / "simple-out.txt")
     app1_future = app1(
diff --git a/parsl/tests/test_staging/test_staging_stdout.py b/parsl/tests/test_staging/test_staging_stdout.py
index aaa45440a7..dc4044ec2e 100644
--- a/parsl/tests/test_staging/test_staging_stdout.py
+++ b/parsl/tests/test_staging/test_staging_stdout.py
@@ -15,6 +15,7 @@ def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
     return "echo hello ; echo goodbye >&2"
 
 
+@pytest.mark.shared_fs
 def test_stdout_staging_file(tmpd_cwd, caplog):
     basename = str(tmpd_cwd) + "/stdout.txt"
     stdout_file = File("file://" + basename)
@@ -30,6 +31,7 @@ def test_stdout_staging_file(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.shared_fs
 def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
     zipfile_name = str(tmpd_cwd) + "/staging.zip"
     stdout_relative_path = "somewhere/test-out.txt"

From d52fbe42bcaa807aa4f24a50189125fe2e202c58 Mon Sep 17 00:00:00 2001
From: Yadu Babuji <yadudoc1729@gmail.com>
Date: Thu, 31 Oct 2024 08:50:03 -0500
Subject: [PATCH 3/3] Updating tests to use staging_required marker instead of
 shared_fs

---
 parsl/tests/test_bash_apps/test_stdout.py       | 1 -
 parsl/tests/test_docs/test_workflow1.py         | 1 -
 parsl/tests/test_staging/test_docs_1.py         | 2 +-
 parsl/tests/test_staging/test_staging_stdout.py | 4 ++--
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/parsl/tests/test_bash_apps/test_stdout.py b/parsl/tests/test_bash_apps/test_stdout.py
index c8404451f6..c935def758 100644
--- a/parsl/tests/test_bash_apps/test_stdout.py
+++ b/parsl/tests/test_bash_apps/test_stdout.py
@@ -34,7 +34,6 @@ def echo_to_streams(msg, stderr=None, stdout=None):
 ]
 
 
-@pytest.mark.shared_fs
 @pytest.mark.parametrize('spec', speclist, ids=testids)
 def test_bad_stdout_specs(spec):
     """Testing bad stdout spec cases"""
diff --git a/parsl/tests/test_docs/test_workflow1.py b/parsl/tests/test_docs/test_workflow1.py
index b0f92b6ab9..271baab4d8 100644
--- a/parsl/tests/test_docs/test_workflow1.py
+++ b/parsl/tests/test_docs/test_workflow1.py
@@ -22,7 +22,6 @@ def save(message, outputs=[]):
     return 'echo {m} &> {o}'.format(m=message, o=outputs[0])
 
 
-@pytest.mark.shared_fs
 @pytest.mark.staging_required
 def test_procedural(N=2):
     """Procedural workflow example from docs on
diff --git a/parsl/tests/test_staging/test_docs_1.py b/parsl/tests/test_staging/test_docs_1.py
index cc6236076a..c4f0e3b007 100644
--- a/parsl/tests/test_staging/test_docs_1.py
+++ b/parsl/tests/test_staging/test_docs_1.py
@@ -12,7 +12,7 @@ def convert(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
-@pytest.mark.shared_fs
+@pytest.mark.staging_required
 def test():
     # create an remote Parsl file
     inp = File('ftp://ftp.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')
diff --git a/parsl/tests/test_staging/test_staging_stdout.py b/parsl/tests/test_staging/test_staging_stdout.py
index dc4044ec2e..0eae6df70f 100644
--- a/parsl/tests/test_staging/test_staging_stdout.py
+++ b/parsl/tests/test_staging/test_staging_stdout.py
@@ -15,7 +15,7 @@ def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
     return "echo hello ; echo goodbye >&2"
 
 
-@pytest.mark.shared_fs
+@pytest.mark.staging_required
 def test_stdout_staging_file(tmpd_cwd, caplog):
     basename = str(tmpd_cwd) + "/stdout.txt"
     stdout_file = File("file://" + basename)
@@ -31,7 +31,7 @@ def test_stdout_staging_file(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
-@pytest.mark.shared_fs
+@pytest.mark.staging_required
 def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
     zipfile_name = str(tmpd_cwd) + "/staging.zip"
     stdout_relative_path = "somewhere/test-out.txt"