diff --git a/flint/imager/wsclean.py b/flint/imager/wsclean.py index cf9f5904..3b3e1c77 100644 --- a/flint/imager/wsclean.py +++ b/flint/imager/wsclean.py @@ -164,6 +164,8 @@ class WSCleanOptions(BaseOptions): """Image a channel range between a lower (inclusive) and upper (exclusive) bound""" no_reorder: bool = False """If True turn off the reordering of the MS at the beginning of wsclean""" + flint_no_log_wsclean_output: bool = False + """If True do not log the wsclean output""" class WSCleanCommand(BaseOptions): @@ -444,9 +446,9 @@ def delete_wsclean_outputs( # which is only created when -join-channels is used def wsclean_cleanup_files( prefix: Union[str, Path], - output_types: Optional[Tuple[str]] = ("dirty", "psf", "model", "residual"), + output_types: Optional[Tuple[str, ...]] = ("dirty", "psf", "model", "residual"), single_channel: bool = False, -) -> Tuple[Path]: +) -> Tuple[Path, ...]: """Clean up (i.e. delete) files from wsclean. Args: @@ -455,17 +457,18 @@ def wsclean_cleanup_files( single_channel (bool, optional): Whether there is the subband part of the wsclean file names to consider. Defaults to False. Returns: - Tuple[Path]: Set of files that were deleted + Tuple[Path, ...]: Set of files that were deleted """ rm_files = [] logger.info(f"Removing wsclean files with {prefix=} {output_types=}") - for output_type in output_types: - rm_files += delete_wsclean_outputs( - prefix=prefix, - output_type=output_type, - no_subbands=single_channel and output_type == "image", - ) + if output_types is not None: + for output_type in output_types: + rm_files += delete_wsclean_outputs( + prefix=str(prefix), + output_type=output_type, + no_subbands=single_channel and output_type == "image", + ) return tuple(rm_files) @@ -516,11 +519,13 @@ class ResolvedCLIResult(NamedTuple): """Mapping results to provide to wsclean""" cmd: Optional[str] = None - """The argument value pair to place on the CLI""" + """The argument value pair to place on the CLI. """ unknown: Optional[Any] = None """Unknown options that could not be converted""" bindpath: Optional[Path] = None """A path to bind to when called within a container""" + ignore: bool = False + """Ignore this CLIResult if True""" def _resolve_wsclean_key_value_to_cli_str(key: str, value: Any) -> ResolvedCLIResult: @@ -556,7 +561,10 @@ def _resolve_wsclean_key_value_to_cli_str(key: str, value: Any) -> ResolvedCLIRe original_key = key key = key.replace("_", "-") - if key == "size": + if original_key.startswith("flint_"): + # No need to do anything more + return ResolvedCLIResult(ignore=True) + elif key == "size": cmd = f"-size {value} {value}" elif isinstance(value, bool): if value: @@ -629,11 +637,17 @@ def create_wsclean_cmd( unknowns: List[Tuple[Any, Any]] = [] logger.info("Creating wsclean command.") - cli_results = map( - _resolve_wsclean_key_value_to_cli_str, - wsclean_options_dict.keys(), - wsclean_options_dict.values(), + cli_results = list( + map( + _resolve_wsclean_key_value_to_cli_str, + wsclean_options_dict.keys(), + wsclean_options_dict.values(), + ) ) + + # Ignore any CLIResult if it has been explicitly instructed to + cli_results = [cli_result for cli_result in cli_results if not cli_result.ignore] + cmds = [cli_result.cmd for cli_result in cli_results if cli_result.cmd] unknowns = [cli_result.unknown for cli_result in cli_results if cli_result.unknown] bind_dir_paths += [ @@ -838,6 +852,7 @@ def run_wsclean_imager( command=wsclean_cmd.cmd, bind_dirs=sclient_bind_dirs, stream_callback_func=_wsclean_output_callback, + ignore_logging_output=wsclean_cmd.options.flint_no_log_wsclean_output, ) if wsclean_cleanup: rm_files = wsclean_cleanup_files( @@ -859,8 +874,12 @@ def run_wsclean_imager( command=wsclean_cmd.cmd, bind_dirs=sclient_bind_dirs, stream_callback_func=_wsclean_output_callback, + ignore_logging_output=wsclean_cmd.options.flint_no_log_wsclean_output, ) + # prefix should be set at this point + assert prefix is not None, f"{prefix=}, which should not happen" + if wsclean_cleanup: logger.info("Will clean up files created by wsclean. ") rm_files = wsclean_cleanup_files(prefix=prefix, single_channel=single_channel) diff --git a/flint/sclient.py b/flint/sclient.py index fa0426c4..b337b965 100644 --- a/flint/sclient.py +++ b/flint/sclient.py @@ -16,6 +16,7 @@ def run_singularity_command( command: str, bind_dirs: Optional[Union[Path, Collection[Path]]] = None, stream_callback_func: Optional[Callable] = None, + ignore_logging_output: bool = False, ) -> None: """Executes a command within the context of a nominated singularity container @@ -25,6 +26,7 @@ def run_singularity_command( command (str): The command to execute bind_dirs (Optional[Union[Path,Collection[Path]]], optional): Specifies a Path, or list of Paths, to bind to in the container. Defaults to None. stream_callback_func (Optional[Callable], optional): Provide a function that is applied to each line of output text when singularity is running and `stream=True`. IF provide it should accept a single (string) parameter. If None, nothing happens. Defaultds to None. + ignore_logging_output (bool, optional): If `True` output from the executed singularity command is not logged. Defaults to False. Raises: FileNotFoundError: Thrown when container image not found @@ -65,7 +67,8 @@ def run_singularity_command( ) for line in output: - logger.info(line.rstrip()) + if not ignore_logging_output: + logger.info(line.rstrip()) if stream_callback_func: stream_callback_func(line) @@ -103,6 +106,7 @@ def wrapper( container: Path, bind_dirs: Optional[Union[Path, Collection[Path]]] = None, stream_callback_func: Optional[Callable] = None, + ignore_logging_output: bool = False, **kwargs, ) -> str: """Function that can be used as a decorator on an input function. This function @@ -113,6 +117,7 @@ def wrapper( container (Path): Path to the container that will be usede to execute the generated command bind_dirs (Optional[Union[Path,Collection[Path]]], optional): Specifies a Path, or list of Paths, to bind to in the container. Defaults to None. stream_callback_func (Optional[Callable], optional): Provide a function that is applied to each line of output text when singularity is running and `stream=True`. IF provide it should accept a single (string) parameter. If None, nothing happens. Defaultds to None. + ignore_logging_output (bool, optional): If `True` output from the executed singularity command is not logged. Defaults to False. Returns: str: The command that was executed @@ -129,6 +134,7 @@ def wrapper( image=container, command=f"{task_str}", bind_dirs=bind_dirs, + ignore_logging_output=ignore_logging_output, stream_callback_func=stream_callback_func, ) diff --git a/tests/test_wsclean.py b/tests/test_wsclean.py index 08ded413..42d5c60f 100644 --- a/tests/test_wsclean.py +++ b/tests/test_wsclean.py @@ -292,6 +292,13 @@ def test_resolve_key_value_to_cli(): assert res.bindpath is None assert res.unknown == ("temp_dir", unknown) + ignore = WSCleanOptions + res = _resolve_wsclean_key_value_to_cli_str("flint_this_should_be_ignored", ignore) + assert res.cmd is None + assert res.bindpath is None + assert res.unknown is None + assert res.ignore + def test_create_wsclean_name(ms_example): """Test the creation of a wsclean name argument"""