diff --git a/alpenhorn/client/acq.py b/alpenhorn/client/acq.py deleted file mode 100644 index e877fbd92..000000000 --- a/alpenhorn/client/acq.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Alpenhorn client interface for operations on `ArchiveAcq`s.""" - -import re -import sys - -import click -import peewee as pw - -from ..db import ArchiveAcq, ArchiveFile, ArchiveFileCopy, StorageGroup, StorageNode - -RE_LOCK_FILE = re.compile(r"^\..*\.lock$") - - -@click.group(context_settings={"help_option_names": ["-h", "--help"]}) -def cli(): - """Commands operating on archival data products. Use to list acquisitions, their contents, and locations of copies.""" - - -@cli.command(name="list") -@click.argument("node_name", required=False) -def acq_list(node_name): - """List known acquisitions. With NODE specified, list acquisitions with files on NODE.""" - config_connect() - - import tabulate - - if node_name: - try: - node = StorageNode.get(name=node_name) - except pw.DoesNotExist: - print("No such storage node:", node_name) - sys.exit(1) - - query = ( - ArchiveFileCopy.select(ArchiveAcq.name, pw.fn.count(ArchiveFileCopy.id)) - .join(ArchiveFile) - .join(ArchiveAcq) - .where(ArchiveFileCopy.node == node) - .group_by(ArchiveAcq.id) - ) - else: - query = ( - ArchiveAcq.select(ArchiveAcq.name, pw.fn.COUNT(ArchiveFile.id)) - .join(ArchiveFile, pw.JOIN.LEFT_OUTER) - .group_by(ArchiveAcq.name) - ) - - data = query.tuples() - - if data: - print(tabulate.tabulate(data, headers=["Name", "Files"])) - else: - print("No matching acquisitions") - - -@cli.command() -@click.argument("acquisition") -@click.argument("node_name", required=False) -def files(acquisition, node_name): - """List files that are in the ACQUISITION. With NODE specified, list acquisitions with files on NODE.""" - config_connect() - - import tabulate - - try: - acq = ArchiveAcq.get(name=acquisition) - except pw.DoesNotExist: - print("No such acquisition:", acquisition) - sys.exit(1) - - if node_name: - try: - node = StorageNode.get(name=node_name) - except pw.DoesNotExist: - print("No such storage node:", node_name) - sys.exit(1) - - query = ( - ArchiveFile.select( - ArchiveFile.name, - ArchiveFileCopy.size_b, - ArchiveFileCopy.has_file, - ArchiveFileCopy.wants_file, - ) - .join(ArchiveFileCopy) - .where( - ArchiveFile.acq == acq, - ArchiveFileCopy.node == node, - ) - ) - headers = ["Name", "Size", "Has", "Wants"] - else: - query = ArchiveFile.select( - ArchiveFile.name, ArchiveFile.size_b, ArchiveFile.md5sum - ).where(ArchiveFile.acq_id == acq.id) - headers = ["Name", "Size", "MD5"] - - data = query.tuples() - - if data: - print(tabulate.tabulate(data, headers=headers)) - else: - print("No registered archive files.") - - -@cli.command() -@click.argument("acquisition") -def where(acquisition): - """List locations of files that are in the ACQUISITION.""" - config_connect() - - import tabulate - - try: - acq = ArchiveAcq.get(name=acquisition) - except pw.DoesNotExist: - print("No such acquisition:", acquisition) - sys.exit(1) - - nodes = ( - StorageNode.select() - .join(ArchiveFileCopy) - .join(ArchiveFile) - .where(ArchiveFile.acq == acq) - .distinct() - ).execute() - if not nodes: - print("No registered archive files.") - return - - for node in nodes: - print("Storage node:", node.name) - query = ( - ArchiveFile.select( - ArchiveFile.name, - ArchiveFileCopy.size_b, - ArchiveFileCopy.has_file, - ArchiveFileCopy.wants_file, - ) - .join(ArchiveFileCopy) - .where( - ArchiveFile.acq == acq, - ArchiveFileCopy.node == node, - ) - ) - headers = ["Name", "Size", "Has", "Wants"] - data = query.tuples() - print(tabulate.tabulate(data, headers=headers)) - print() - - -@cli.command() -@click.argument("acquisition") -@click.argument("source_node") -@click.argument("destination_group") -def syncable(acquisition, source_node, destination_group): - """List all files that are in the ACQUISITION that still need to be moved to DESTINATION_GROUP and are available on SOURCE_NODE.""" - config_connect() - - import tabulate - - try: - acq = ArchiveAcq.get(name=acquisition) - except pw.DoesNotExist: - print("No such acquisition:", acquisition) - - try: - src = StorageNode.get(name=source_node) - except pw.DoesNotExist: - print("No such storage node:", source_node) - sys.exit(1) - - try: - dest = StorageGroup.get(name=destination_group) - except pw.DoesNotExist: - print("No such storage group:", destination_group) - sys.exit(1) - - # First get the nodes at the destination... - nodes_at_dest = StorageNode.select().where(StorageNode.group == dest) - - # Then use this to get a list of all files at the destination... - files_at_dest = ( - ArchiveFile.select() - .join(ArchiveFileCopy) - .where( - ArchiveFile.acq == acq, - ArchiveFileCopy.node << nodes_at_dest, - ArchiveFileCopy.has_file == "Y", - ) - ) - - # Then combine to get all file(copies) that are available at the source but - # not at the destination... - query = ( - ArchiveFile.select( - ArchiveFile.name, - ArchiveFile.size_b, - ) - .where(ArchiveFile.acq == acq) - .join(ArchiveFileCopy) - .where( - ArchiveFileCopy.node == src, - ArchiveFileCopy.has_file == "Y", - ~(ArchiveFile.id << files_at_dest), - ) - ) - - data = query.tuples() - - if data: - print(tabulate.tabulate(data, headers=["Name", "Size"])) - else: - print( - "No files to copy from node '", - source_node, - "' to group '", - destination_group, - "'.", - sep="", - ) diff --git a/alpenhorn/client/acq/__init__.py b/alpenhorn/client/acq/__init__.py new file mode 100644 index 000000000..2b7c18f48 --- /dev/null +++ b/alpenhorn/client/acq/__init__.py @@ -0,0 +1,20 @@ +"""Alpenhorn client interface for operations on `ArchiveAcq`s.""" + +import re +import sys +import click +import peewee as pw + +from .files import files +from .list import list_ +from .show import show + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +def cli(): + """Manage Acquisitions.""" + + +cli.add_command(files, "files") +cli.add_command(list_, "list") +cli.add_command(show, "show") diff --git a/alpenhorn/client/acq/files.py b/alpenhorn/client/acq/files.py new file mode 100644 index 000000000..3c4b360ae --- /dev/null +++ b/alpenhorn/client/acq/files.py @@ -0,0 +1,111 @@ +"""alpenhorn acq files command.""" + +import click +import peewee as pw +from tabulate import tabulate + +from ...common.util import pretty_bytes +from ...db import ArchiveAcq, ArchiveFile, ArchiveFileCopy, StorageGroup, StorageNode +from ..cli import echo +from ..options import client_option, not_both + + +@click.command() +@click.argument("acq", metavar="ACQ") +@client_option("group") +@client_option("node") +@click.option( + "--show-removed", + is_flag=True, + help="If used with --node or --group, also list files that have been removed from the node/group.", +) +def files(acq, group, node, show_removed): + """List files in an Acqusition. + + Lists files in the Acqusition named ACQ.""" + + not_both(node, "node", group, "group") + + try: + acq = ArchiveAcq.get(name=acq) + except pw.DoesNotExist: + raise click.ClickException("No such Acquisition: " + acq) + + data = [] + if node: + headers = ["Name", "Size", "Node State", "Size on Node", "MD5"] + + try: + node = StorageNode.get(name=node) + except pw.DoesNotExist: + raise click.ClickException("No such Storage Node: " + node) + query = ( + ArchiveFileCopy.select() + .join(ArchiveFile) + .where(ArchiveFile.acq == acq, ArchiveFileCopy.node == node) + ) + + for copy in query: + state = copy.state + if show_removed or state != "Removed": + file_size = ( + "-" if copy.file.size_b is None else pretty_bytes(copy.file.size_b) + ) + node_size = pretty_bytes(copy.size_b) if copy.size_b else "-" + data.append( + ( + copy.file.name, + file_size, + state, + node_size, + copy.file.md5sum, + ) + ) + elif group: + headers = ["Name", "Size", "Group State", "MD5"] + + try: + group = StorageGroup.get(name=group) + except pw.DoesNotExist: + raise click.ClickException("No such Storage Group: " + group) + query = ( + ArchiveFileCopy.select() + .join(ArchiveFile) + .switch(ArchiveFileCopy) + .join(StorageNode) + .where(ArchiveFile.acq == acq, StorageNode.group == group) + .group_by(ArchiveFile.id) + ) + + for copy in query: + state = group.filecopy_state(copy.file) + + if show_removed or state != "N": + # Convert state to words + if state == "Y": + state = "Present" + elif state == "M": + state = "Needs Check" + elif state == "N": + state = "Removed" + else: + state = "Corrupt" + + file_size = ( + "-" if copy.file.size_b is None else pretty_bytes(copy.file.size_b) + ) + + data.append((copy.file.name, file_size, state, copy.file.md5sum)) + else: + headers = ["Name", "Size", "MD5"] + + query = ArchiveFile.select().where(ArchiveFile.acq == acq) + + for file in query: + file_size = "-" if file.size_b is None else pretty_bytes(file.size_b) + data.append((file.name, file_size, file.md5sum)) + + if data: + echo(tabulate(data, headers=headers)) + else: + echo("No files.") diff --git a/alpenhorn/client/acq/list.py b/alpenhorn/client/acq/list.py new file mode 100644 index 000000000..23b4bc3d1 --- /dev/null +++ b/alpenhorn/client/acq/list.py @@ -0,0 +1,62 @@ +"""alpenhorn acq list command.""" + +import click +import tabulate +import peewee as pw +from tabulate import tabulate + +from ...db import ArchiveAcq, ArchiveFile, ArchiveFileCopy, StorageGroup, StorageNode +from ..cli import echo +from ..options import client_option, not_both + + +@click.command() +@client_option( + "group", + help="Limit to acquisitions with files existing on Storage Group named " + "GROUP. In this case, only files in GROUP are counted.", +) +@client_option( + "node", + help="Limit to acquisitions with files existing on Storage Node named " + "NODE. In this case, only files on NODE are counted.", +) +def list_(group, node): + """List acquisitions.""" + + not_both(node, "node", group, "group") + + query = ( + ArchiveAcq.select(ArchiveAcq.name, pw.fn.COUNT(ArchiveFile.id)) + .join(ArchiveFile, pw.JOIN.LEFT_OUTER) + .group_by(ArchiveAcq.id) + ) + + if group: + try: + group = StorageGroup.get(name=group) + except pw.DoesNotExist: + raise click.ClickException("No such group: " + group) + + query = ( + query.join(ArchiveFileCopy) + .join(StorageNode) + .where(StorageNode.group == group, ArchiveFileCopy.has_file == "Y") + ) + + elif node: + try: + node = StorageNode.get(name=node) + except pw.DoesNotExist: + raise click.ClickException("No such node: " + node) + + query = query.join(ArchiveFileCopy).where( + ArchiveFileCopy.node == node, ArchiveFileCopy.has_file == "Y" + ) + + data = query.tuples() + + if data: + echo(tabulate(data, headers=["Name", "Files"])) + else: + echo("No acquisitions") diff --git a/alpenhorn/client/acq/show.py b/alpenhorn/client/acq/show.py new file mode 100644 index 000000000..3c422d045 --- /dev/null +++ b/alpenhorn/client/acq/show.py @@ -0,0 +1,116 @@ +"""alpenhorn acq show command.""" + +import click +import peewee as pw +from tabulate import tabulate + +from ...common.util import pretty_bytes +from ...db import ArchiveAcq, ArchiveFile, ArchiveFileCopy, StorageGroup, StorageNode +from ..cli import echo + + +@click.command() +@click.argument("acq") +@click.option("--show-groups", is_flag=True, help="Show Storage Groups containing ACQ.") +@click.option("--show-nodes", is_flag=True, help="Show Storage Nodes containing ACQ.") +def show(acq, show_groups, show_nodes): + """Show details of an Acquisition. + + Shows details of the Acquisition named ACQ. + """ + + try: + acq = ArchiveAcq.get(name=acq) + except pw.DoesNotExist: + raise click.ClickException("No such Acquisition: " + acq) + + # File count and size + totals = ( + ArchiveFile.select( + pw.fn.COUNT(ArchiveFile.id).alias("count"), + pw.fn.Sum(ArchiveFile.size_b).alias("size"), + ) + .where(ArchiveFile.acq == acq) + .group_by(ArchiveFile.acq) + .dicts() + )[0] + + echo("Acquisition: " + acq.name) + echo(" File Count: " + str(totals["count"])) + echo(" Total Size: " + pretty_bytes(totals["size"])) + + if show_nodes: + # If show_groups and show_nodes are True, keys are group names and value are sub-dicts. + # If only show_nodes is Ture, keys are node names. Not set if show_nodes is False. + node_totals = {} + + query = ( + ArchiveFile.select( + StorageNode.name.alias("node"), + StorageNode.group.alias("group"), + pw.fn.COUNT(ArchiveFile.id).alias("count"), + pw.fn.Sum(ArchiveFile.size_b).alias("size"), + ) + .join(ArchiveFileCopy) + .join(StorageNode) + .where(ArchiveFile.acq == acq, ArchiveFileCopy.has_file == "Y") + .group_by(StorageNode.id) + ) + for row in query.dicts(): + pretty_size = pretty_bytes(row["size"]) if row["size"] is not None else "-" + if show_groups: + group_dict = node_totals.setdefault(row["group"], {}) + group_dict[row["node"]] = (row["count"], pretty_size) + else: + node_totals[row["node"]] = (row["count"], pretty_size) + + if show_groups: + group_totals = ( + ArchiveFile.select( + StorageGroup.id.alias("gid"), + StorageGroup.name.alias("name"), + pw.fn.COUNT(ArchiveFile.id).alias("count"), + pw.fn.Sum(ArchiveFile.size_b).alias("size"), + ) + .join(ArchiveFileCopy) + .join(StorageNode) + .join(StorageGroup) + .where(ArchiveFile.acq == acq, ArchiveFileCopy.has_file == "Y") + .group_by(StorageGroup.id) + .order_by(StorageGroup.name) + ) + if show_nodes: + name_header = "Group/Node" + else: + name_header = "Group" + echo(f"\n{name_header} breakdown:\n") + data = [] + for group in group_totals.dicts(): + if not group["count"]: + continue + data.append( + ( + group["name"], + group["count"], + pretty_bytes(group["size"]) if group["size"] is not None else "-", + ) + ) + if show_nodes and group["gid"] in node_totals: + add_blank = False + for node_name, node_data in node_totals[group["gid"]].items(): + if node_data[0]: + add_blank = True + data.append(("-- " + node_name, *node_data)) + if add_blank: + data.append(("", "", "")) + if data: + echo(tabulate(data, headers=[name_header, "Count", "Size"])) + else: + echo("No nodes with data") + elif show_nodes: + echo("\nNode breakdown:\n") + if node_totals: + node_data = [(node, *node_totals[node]) for node in sorted(node_totals)] + echo(tabulate(node_data, headers=["Node", "Count", "Size"])) + else: + echo("No nodes with data") diff --git a/alpenhorn/client/options.py b/alpenhorn/client/options.py index da0bf8337..a3e01134c 100644 --- a/alpenhorn/client/options.py +++ b/alpenhorn/client/options.py @@ -19,7 +19,13 @@ def client_option(option: str, **extra_kwargs): """ # Set args for the click.option decorator - if option == "io_class": + if option == "group": + args = ("--group",) + kwargs = { + "metavar": "GROUP", + "help": "Limit to files in Storage Group named GROUP.", + } + elif option == "io_class": args = ( "io_class", "-i", @@ -49,6 +55,12 @@ def client_option(option: str, **extra_kwargs): "multiple times. Modifies any config specified by --io-config. If VALUE " "is empty (--io-var VAR=), VAR is deleted if present.", } + elif option == "node": + args = ("--node",) + kwargs = { + "metavar": "NODE", + "help": "Limit to files on Storage Node named NODE.", + } elif option == "notes": args = ("--notes",) kwargs = {"metavar": "COMMENT", "help": "Set notes to COMMENT."} @@ -66,6 +78,15 @@ def _decorator(func): return _decorator +def not_both(opt1_set: bool, opt1_name: str, opt2_set: bool, opt2_name: str) -> None: + """Check whether two incompatible options were used. + + If they were, raise click.UsageError.""" + + if opt1_set and opt2_set: + raise click.UsageError(f"cannot use both --{opt1_name} and --{opt2_name}") + + def set_io_config( io_config: str | None, io_var: list | tuple, default: str | dict = {} ) -> dict | None: diff --git a/alpenhorn/db/archive.py b/alpenhorn/db/archive.py index e749837cc..6365843b4 100644 --- a/alpenhorn/db/archive.py +++ b/alpenhorn/db/archive.py @@ -59,6 +59,27 @@ def path(self) -> pathlib.Path: return pathlib.Path(self.node.root, self.file.path) + @property + def state(self) -> str: + """A human-readable description of the copy state.""" + + if self.wants_file == "N": + return "Removed" if self.has_file == "N" else "Pending Removal" + if self.wants_file == "M": + return "Removed" if self.has_file == "N" else "Removable" + + # Otherwise, wants_file == 'Y' + if self.has_file == "Y": + return "Present" + if self.has_file == "M": + return "Needs Check" + if self.has_file == "N": + # i.e. a third-party deleted it + return "Missing" + + # wants_file == 'X' and has_file == 'Y' .. or something completely wrong + return "Corrupt" + class Meta: indexes = ((("file", "node"), True),) # (file, node) is unique diff --git a/pyproject.toml b/pyproject.toml index d363d0eed..54acc4b25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,3 +40,6 @@ find = {} [tool.setuptools.dynamic] readme = {file = ["README.rst"], content-type = "text/x-rst"} + +[tool.pytest.ini_options] +addopts = "--import-mode=importlib" diff --git a/tests/client/acq/test_files.py b/tests/client/acq/test_files.py new file mode 100644 index 000000000..d5dadcac8 --- /dev/null +++ b/tests/client/acq/test_files.py @@ -0,0 +1,303 @@ +"""Test CLI: alpenhorn acq files""" + +import re + +from alpenhorn.db import ( + StorageGroup, + StorageNode, + ArchiveAcq, + ArchiveFile, + ArchiveFileCopy, +) + + +def test_no_acq(clidb, client): + """Test with no acq specified.""" + + client(2, ["acq", "files"]) + + +def test_acq_not_found(clidb, client): + """Test with non-existent acq.""" + + client(1, ["acq", "files", "Acq"]) + + +def test_no_files(clidb, client): + """Test with nothing to list.""" + + ArchiveAcq.create(name="Acq") + + result = client(0, ["acq", "files", "Acq"]) + + # i.e. the header hasn't been printed + assert "Name" not in result.output + + +def test_list(clidb, client): + """Test with no constraints.""" + + acq = ArchiveAcq.create(name="Acq1") + ArchiveFile.create( + name="File1", acq=acq, size_b=123, md5sum="0123456789abcdef0123456789abcdef" + ) + ArchiveFile.create( + name="File2", acq=acq, size_b=45678, md5sum="fedcba9876543210fedcba9876543210" + ) + acq = ArchiveAcq.create(name="Acq2") + ArchiveFile.create( + name="File3", acq=acq, size_b=9, md5sum="fedcba98765432100123456789abcdef" + ) + + result = client(0, ["acq", "files", "Acq1"]) + + # Check table + assert ( + re.search(r"File1\s+123 B\s+0123456789abcdef0123456789abcdef", result.output) + is not None + ) + assert ( + re.search( + r"File2\s+44\.61 kiB\s+fedcba9876543210fedcba9876543210", result.output + ) + is not None + ) + assert "File3" not in result.output + assert result.output.count("File") == 2 + + +def test_list_node(clidb, client): + """Test with node constraint.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="FileYY", acq=acq, size_b=123) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="Y") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + file = ArchiveFile.create(name="FileMY", acq=acq, size_b=789) + ArchiveFileCopy.create( + file=file, node=node1, has_file="M", wants_file="Y", size_b=1234 + ) + file = ArchiveFile.create(name="FileXY", acq=acq, size_b=5678) + ArchiveFileCopy.create( + file=file, node=node1, has_file="X", wants_file="Y", size_b=0 + ) + file = ArchiveFile.create(name="FileNY", acq=acq, size_b=0) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="Y") + + file = ArchiveFile.create(name="FileYM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="M") + file = ArchiveFile.create(name="FileMM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="M", wants_file="M") + file = ArchiveFile.create(name="FileXM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="X", wants_file="M") + file = ArchiveFile.create(name="FileNM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="M") + + file = ArchiveFile.create(name="FileYN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="N") + file = ArchiveFile.create(name="FileMN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="M", wants_file="N") + file = ArchiveFile.create(name="FileXN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="X", wants_file="N") + file = ArchiveFile.create(name="FileNN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + + # Acq2 has one file on Node1 + acq = ArchiveAcq.create(name="Acq2") + file = ArchiveFile.create(name="File3", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N") + + result = client(0, ["acq", "files", "Acq1", "--node=Node1"]) + + assert re.search(r"FileYY\s+123 B\s+Present\s+-", result.output) is not None + assert "File2" not in result.output + assert "File3" not in result.output + assert ( + re.search(r"FileMY\s+789 B\s+Needs Check\s+1\.205 kiB", result.output) + is not None + ) + assert re.search(r"FileXY\s+5.545 kiB\s+Corrupt\s+-", result.output) is not None + assert re.search(r"FileNY\s+0 B\s+Missing", result.output) is not None + assert re.search(r"FileYM\s+-\s+Removable", result.output) is not None + assert re.search(r"FileMM\s+-\s+Removable", result.output) is not None + assert re.search(r"FileXM\s+-\s+Removable", result.output) is not None + assert "FileNM" not in result.output + assert re.search(r"FileYN\s+-\s+Pending Removal", result.output) is not None + assert re.search(r"FileMN\s+-\s+Pending Removal", result.output) is not None + assert re.search(r"FileXN\s+-\s+Pending Removal", result.output) is not None + assert "FileNN" not in result.output + assert result.output.count("File") == 10 + + +def test_list_node_removed(clidb, client): + """Test --node --show-removed.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="FileYY", acq=acq, size_b=123) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="Y") + file = ArchiveFile.create(name="FileNY", acq=acq, size_b=0) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="Y") + + file = ArchiveFile.create(name="FileYM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="X", wants_file="M") + file = ArchiveFile.create(name="FileNM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="M") + + file = ArchiveFile.create(name="FileYN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="X", wants_file="N") + file = ArchiveFile.create(name="FileNN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + + result = client(0, ["acq", "files", "Acq1", "--node=Node1", "--show-removed"]) + + assert re.search(r"FileYY\s+123 B\s+Present\s+-", result.output) is not None + assert re.search(r"FileNY\s+0 B\s+Missing", result.output) is not None + assert re.search(r"FileYM\s+-\s+Removable", result.output) is not None + assert re.search(r"FileNM\s+-\s+Removed", result.output) is not None + assert re.search(r"FileYN\s+-\s+Pending Removal", result.output) is not None + assert re.search(r"FileNN\s+-\s+Removed", result.output) is not None + assert result.output.count("File") == 6 + + +def test_list_group_removed(clidb, client): + """Test --group --show-removed.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="FileYY", acq=acq, size_b=123) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="Y") + file = ArchiveFile.create(name="FileNY", acq=acq, size_b=0) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="Y") + + file = ArchiveFile.create(name="FileYM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="M") + file = ArchiveFile.create(name="FileNM", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="M") + + file = ArchiveFile.create(name="FileYN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="N") + file = ArchiveFile.create(name="FileNN", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + + result = client(0, ["acq", "files", "Acq1", "--group=Group1", "--show-removed"]) + + assert re.search(r"FileYY\s+123 B\s+Present", result.output) is not None + assert re.search(r"FileNY\s+0 B\s+Removed", result.output) is not None + assert re.search(r"FileYM\s+-\s+Present", result.output) is not None + assert re.search(r"FileNM\s+-\s+Removed", result.output) is not None + assert re.search(r"FileYN\s+-\s+Present", result.output) is not None + assert re.search(r"FileNN\s+-\s+Removed", result.output) is not None + assert result.output.count("File") == 6 + + +def test_list_no_node(clidb, client): + """Test with non-existent node.""" + + ArchiveAcq.create(name="Acq1") + + client(1, ["acq", "files", "Acq1", "--node=Missing"]) + + +def test_no_list_node(clidb, client): + """Test no list with node constraint.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + + # On Node2, not Node1 + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + + result = client(0, ["acq", "files", "Acq1", "--node=Node1"]) + + # i.e. the header hasn't been printed + assert "Name" not in result.output + + +def test_list_node_only_removed(clidb, client): + """Test list --node with only removed files.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + + # On Node2, not Node1 + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + + result = client(0, ["acq", "files", "Acq1", "--node=Node1", "--show-removed"]) + + assert "File1" in result.output + + +def test_list_no_group(clidb, client): + """Test with non-existent group.""" + + ArchiveAcq.create(name="Acq1") + + client(1, ["acq", "files", "Acq1", "--group=Missing"]) + + +def test_no_list_group(clidb, client): + """Test no list with group constraint.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + group = StorageGroup.create(name="Group2") + node2 = StorageNode.create(name="Node3", group=group) + + # Everything is in Group2 + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + + result = client(0, ["acq", "files", "Acq1", "--group=Group1"]) + + # i.e. the header hasn't been printed + assert "Name" not in result.output + + +def test_list_group_only_removed(clidb, client): + """Test no list with group constraint.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + group = StorageGroup.create(name="Group2") + node2 = StorageNode.create(name="Node3", group=group) + + # Everything is in Group2 + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + + result = client(0, ["acq", "files", "Acq1", "--group=Group1", "--show-removed"]) + + # i.e. the header hasn't been printed + assert "File1" in result.output + + +def test_list_node_group(clidb, client): + """Test with both node and group.""" + + ArchiveAcq.create(name="Acq1") + + client(2, ["acq", "files", "Acq1", "--node=Node", "--group=Group"]) diff --git a/tests/client/acq/test_list.py b/tests/client/acq/test_list.py new file mode 100644 index 000000000..db2be2aeb --- /dev/null +++ b/tests/client/acq/test_list.py @@ -0,0 +1,185 @@ +"""Test CLI: alpenhorn acq list""" + +import re + +from alpenhorn.db import ( + StorageGroup, + StorageNode, + ArchiveAcq, + ArchiveFile, + ArchiveFileCopy, +) + + +def test_no_list(clidb, client): + """Test with nothing to list.""" + + result = client(0, ["acq", "list"]) + + # i.e. the header hasn't been printed + assert "Name" not in result.output + + +def test_list(clidb, client): + """Test with no constraints.""" + + acq = ArchiveAcq.create(name="Acq1") + ArchiveFile.create(name="File1", acq=acq) + ArchiveFile.create(name="File2", acq=acq) + acq = ArchiveAcq.create(name="Acq2") + ArchiveFile.create(name="File3", acq=acq) + + # Acq 3 has no files, but should still get listed + ArchiveAcq.create(name="Acq3") + + result = client(0, ["acq", "list"]) + + # Check table + assert re.search(r"Acq1\s+2", result.output) is not None + assert re.search(r"Acq2\s+1", result.output) is not None + assert re.search(r"Acq3\s+0", result.output) is not None + + +def test_list_node(clidb, client): + """Test with node constraint.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + + # Acq1 has two files; File1 is on Node1; File2 is on Node2 + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + file = ArchiveFile.create(name="File2", acq=acq) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + + # Acq2 has one file; File3 is on Node2 + acq = ArchiveAcq.create(name="Acq2") + file = ArchiveFile.create(name="File3", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + + # Acq3 has one file; File4 is on Node2 + acq = ArchiveAcq.create(name="Acq3") + file = ArchiveFile.create(name="File4", acq=acq) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + + result = client(0, ["acq", "list", "--node=Node1"]) + + # Only Acq1 should be listed, with only one file + assert re.search(r"Acq1\s+1", result.output) is not None + assert "Acq2" not in result.output + assert "Acq3" not in result.output + + +def test_list_no_node(clidb, client): + """Test with non-existent node.""" + + client(1, ["acq", "list", "--node=Missing"]) + + +def test_no_list_node(clidb, client): + """Test no list with node constraint.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + StorageNode.create(name="Node2", group=group) + + # All files are on Node1 + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + file = ArchiveFile.create(name="File2", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + acq = ArchiveAcq.create(name="Acq2") + file = ArchiveFile.create(name="File3", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + acq = ArchiveAcq.create(name="Acq3") + file = ArchiveFile.create(name="File4", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + + result = client(0, ["acq", "list", "--node=Node2"]) + + # i.e. the header hasn't been printed + assert "Name" not in result.output + + +def test_list_group(clidb, client): + """Test with group constraint.""" + + group1 = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group1) + node2 = StorageNode.create(name="Node2", group=group1) + group2 = StorageGroup.create(name="Group2") + node3 = StorageNode.create(name="Node3", group=group2) + + # Acq1 has two files, both in Group1 + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + file = ArchiveFile.create(name="File2", acq=acq) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + + # Acq2 has two file; File3 is in Group1, File4 is in Group2 + acq = ArchiveAcq.create(name="Acq2") + file = ArchiveFile.create(name="File3", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + file = ArchiveFile.create(name="File4", acq=acq) + ArchiveFileCopy.create(file=file, node=node3, has_file="Y") + + # Acq3 has one file; File5 is in Group 2 + acq = ArchiveAcq.create(name="Acq3") + file = ArchiveFile.create(name="File5", acq=acq) + ArchiveFileCopy.create(file=file, node=node3, has_file="Y") + + result = client(0, ["acq", "list", "--group=Group1"]) + + # Check results. Only files in the group should be counted + assert re.search(r"Acq1\s+2", result.output) is not None + assert re.search(r"Acq2\s+1", result.output) is not None + assert "Acq3" not in result.output + + +def test_list_no_group(clidb, client): + """Test with non-existent group.""" + + client(1, ["acq", "list", "--group=Missing"]) + + +def test_no_list_group(clidb, client): + """Test no list with group constraint.""" + + group1 = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group1) + node2 = StorageNode.create(name="Node2", group=group1) + group2 = StorageGroup.create(name="Group2") + StorageNode.create(name="Node3", group=group2) + + # Everything is in Group1 + acq = ArchiveAcq.create(name="Acq1") + file = ArchiveFile.create(name="File1", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y") + file = ArchiveFile.create(name="File2", acq=acq) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + acq = ArchiveAcq.create(name="Acq2") + file = ArchiveFile.create(name="File3", acq=acq) + ArchiveFileCopy.create(file=file, node=node1, has_file="N") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + file = ArchiveFile.create(name="File4", acq=acq) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + acq = ArchiveAcq.create(name="Acq3") + file = ArchiveFile.create(name="File5", acq=acq) + ArchiveFileCopy.create(file=file, node=node2, has_file="Y") + + result = client(0, ["acq", "list", "--group=Group2"]) + + # i.e. the header hasn't been printed + assert "Name" not in result.output + + +def test_list_node_group(clidb, client): + """Test with both node and group.""" + + client(2, ["acq", "list", "--node=Node", "--group=Group"]) diff --git a/tests/client/acq/test_show.py b/tests/client/acq/test_show.py new file mode 100644 index 000000000..df2e40ff2 --- /dev/null +++ b/tests/client/acq/test_show.py @@ -0,0 +1,205 @@ +"""Test CLI: alpenhorn acq files""" + +import re + +from alpenhorn.db import ( + StorageGroup, + StorageNode, + ArchiveAcq, + ArchiveFile, + ArchiveFileCopy, +) + + +def test_no_acq(clidb, client): + """Test with no acq specified.""" + + client(2, ["acq", "show"]) + + +def test_acq_not_found(clidb, client): + """Test with non-existent acq.""" + + client(1, ["acq", "show", "Acq"]) + + +def test_simple_show(clidb, client): + """Test show with no options.""" + + acq = ArchiveAcq.create(name="Acq") + ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFile.create(name="File2", acq=acq, size_b=456) + + result = client(0, ["acq", "show", "Acq"]) + + assert "Acq" in result.output + assert ": 2" in result.output + assert "579 B" in result.output + + +def test_show_nodes(clidb, client): + """Test show with --show-nodes.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + StorageNode.create(name="Node3", group=group) + + acq = ArchiveAcq.create(name="Acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFileCopy.create(node=node1, file=file, has_file="Y") + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node1, file=file, has_file="Y") + file = ArchiveFile.create(name="File3", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node2, file=file, has_file="Y") + + result = client(0, ["acq", "show", "Acq", "--show-nodes"]) + + assert re.search(r"Node1\s+2\s+579 B", result.output) is not None + assert re.search(r"Node2\s+1\s+456 B", result.output) is not None + assert "Node3" not in result.output + + +def test_show_groups(clidb, client): + """Test show with --show-groups.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + group = StorageGroup.create(name="Group2") + node3 = StorageNode.create(name="Node3", group=group) + group = StorageGroup.create(name="Group3") + StorageNode.create(name="Node4", group=group) + + acq = ArchiveAcq.create(name="Acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFileCopy.create(node=node1, file=file, has_file="Y") + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node1, file=file, has_file="Y") + file = ArchiveFile.create(name="File3", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node2, file=file, has_file="Y") + file = ArchiveFile.create(name="File4", acq=acq, size_b=789) + ArchiveFileCopy.create(node=node3, file=file, has_file="Y") + + result = client(0, ["acq", "show", "Acq", "--show-groups"]) + + assert re.search(r"Group1\s+3\s+1.011 kiB", result.output) is not None + assert re.search(r"Group2\s+1\s+789 B", result.output) is not None + assert "Group3" not in result.output + + +def test_show_groups_nodes(clidb, client): + """Test show with --show-groups and --show-nodes.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + group = StorageGroup.create(name="Group2") + node3 = StorageNode.create(name="Node3", group=group) + StorageNode.create(name="Node4", group=group) + group = StorageGroup.create(name="Group3") + StorageNode.create(name="Node5", group=group) + + acq = ArchiveAcq.create(name="Acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFileCopy.create(node=node1, file=file, has_file="Y") + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node1, file=file, has_file="Y") + file = ArchiveFile.create(name="File3", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node2, file=file, has_file="Y") + file = ArchiveFile.create(name="File4", acq=acq, size_b=789) + ArchiveFileCopy.create(node=node3, file=file, has_file="Y") + + result = client(0, ["acq", "show", "Acq", "--show-groups", "--show-nodes"]) + + assert re.search(r"Group1\s+3\s+1.011 kiB", result.output) is not None + assert re.search(r"-- Node1\s+2\s+579 B", result.output) is not None + assert re.search(r"-- Node2\s+1\s+456 B", result.output) is not None + assert re.search(r"Group2\s+1\s+789 B", result.output) is not None + assert re.search(r"-- Node3\s+1\s+789 B", result.output) is not None + assert "Node4" not in result.output + assert "Group3" not in result.output + assert "Node5" not in result.output + + +def test_show_no_nodes(clidb, client): + """Test --show-nodes, with no nodes to show.""" + + group = StorageGroup.create(name="Group") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + StorageNode.create(name="Node3", group=group) + + acq = ArchiveAcq.create(name="Acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFileCopy.create(node=node1, file=file, has_file="M") + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node1, file=file, has_file="X") + file = ArchiveFile.create(name="File3", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + + result = client(0, ["acq", "show", "Acq", "--show-nodes"]) + + # No table printed + assert result.output.count("Size") == 1 + + +def test_show_no_groups(clidb, client): + """Test show with --show-groups with no groups to show.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + group = StorageGroup.create(name="Group2") + node3 = StorageNode.create(name="Node3", group=group) + group = StorageGroup.create(name="Group3") + StorageNode.create(name="Node4", group=group) + + acq = ArchiveAcq.create(name="Acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFileCopy.create(node=node1, file=file, has_file="X") + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node1, file=file, has_file="M") + file = ArchiveFile.create(name="File3", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File4", acq=acq, size_b=789) + ArchiveFileCopy.create(node=node3, file=file, has_file="N") + + result = client(0, ["acq", "show", "Acq", "--show-groups"]) + + # No table printed + assert result.output.count("Size") == 1 + + +def test_show_groups_nodes(clidb, client): + """Test show with --show-groups and --show-nodes with no groups to show.""" + + group = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group) + node2 = StorageNode.create(name="Node2", group=group) + group = StorageGroup.create(name="Group2") + node3 = StorageNode.create(name="Node3", group=group) + StorageNode.create(name="Node4", group=group) + group = StorageGroup.create(name="Group3") + StorageNode.create(name="Node5", group=group) + + acq = ArchiveAcq.create(name="Acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=123) + ArchiveFileCopy.create(node=node1, file=file, has_file="N") + ArchiveFileCopy.create(node=node2, file=file, has_file="N") + file = ArchiveFile.create(name="File2", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node1, file=file, has_file="X") + file = ArchiveFile.create(name="File3", acq=acq, size_b=456) + ArchiveFileCopy.create(node=node2, file=file, has_file="M") + file = ArchiveFile.create(name="File4", acq=acq, size_b=789) + ArchiveFileCopy.create(node=node3, file=file, has_file="N") + + result = client(0, ["acq", "show", "Acq", "--show-groups", "--show-nodes"]) + + # No table printed + assert result.output.count("Size") == 1 diff --git a/tests/client/test_client_acq.py b/tests/client/test_client_acq.py deleted file mode 100644 index 05840697c..000000000 --- a/tests/client/test_client_acq.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -test_client_acq ----------------------------------- - -Tests for `alpenhorn.client.acq` module. -""" - -import re - -import pytest -from click.testing import CliRunner - -# import alpenhorn.acquisition as ac -# import alpenhorn.archive as ar -# import alpenhorn.client as cli -# import alpenhorn.db as db -# import alpenhorn.storage as st - -# XXX: client is broken -pytest.skip("client is broken", allow_module_level=True) - - -@pytest.fixture -def fixtures(tmpdir): - """Initializes an in-memory Sqlite database with data in tests/fixtures""" - db._connect() - - yield ti.load_fixtures(tmpdir) - - db.database_proxy.close() - - -@pytest.fixture(autouse=True) -def no_cli_init(monkeypatch): - monkeypatch.setattr(cli.acq, "config_connect", lambda: None) - - -def test_help(fixtures): - """Test the acq command help""" - runner = CliRunner() - - help_result = runner.invoke(cli.cli, ["acq", "--help"]) - assert help_result.exit_code == 0 - assert """Commands operating on archival data products.""" in help_result.output - - -def test_list(fixtures): - """Test the 'acq list' command""" - runner = CliRunner() - - help_result = runner.invoke(cli.cli, ["acq", "list", "--help"]) - assert help_result.exit_code == 0 - assert "List known acquisitions." in help_result.output - - # List all registered acquisitions (there is only one, 'x', with files 'red', 'jim', and 'sheila') - result = runner.invoke(cli.cli, ["acq", "list"]) - assert result.exit_code == 0 - assert re.match(r"Name +Files\n" r"-+ -+\n" r"x +3 *\n", result.output, re.DOTALL) - - # Fail when given a non-existent storage node - result = runner.invoke(cli.cli, ["acq", "list", "y"]) - assert result.exit_code == 1 - assert "No such storage node: y" in result.output - - # Check acquisitions present on node 'x' (only 'fred' and 'sheila' from 'x', 'jim' is missing) - result = runner.invoke(cli.cli, ["acq", "list", "x"]) - assert result.exit_code == 0 - assert re.match(r"Name +Files\n" r"-+ -+\n" r"x +2 *\n", result.output, re.DOTALL) - - -def test_files(fixtures): - """Test the 'acq files' command""" - runner = CliRunner() - - # Check help output - help_result = runner.invoke(cli.cli, ["acq", "files", "--help"]) - assert help_result.exit_code == 0 - assert "List files that are in the ACQUISITION." in help_result.output - - # Fail when given a non-existent acquisition - result = runner.invoke(cli.cli, ["acq", "files", "z"]) - assert result.exit_code == 1 - assert "No such acquisition: z" in result.output - - # Check regular case - result = runner.invoke(cli.cli, ["acq", "files", "x"]) - assert result.exit_code == 0 - assert re.match( - r".*Name +Size +MD5 *\n" - r"-+ -+ -+\n" - r"fred *\n" - r"jim +0 +d41d8cd98f00b204e9800998ecf8427e *\n" - r"sheila *\n$", - result.output, - re.DOTALL, - ) - - # Fail when given a non-existent storage node - result = runner.invoke(cli.cli, ["acq", "files", "x", "y"]) - assert result.exit_code == 1 - assert "No such storage node: y" in result.output - - # Check files from 'x' present on node 'x' (only 'fred' and 'sheila', with 'missing' and 'corrupted' has_file status) - result = runner.invoke(cli.cli, ["acq", "files", "x", "x"]) - assert result.exit_code == 0 - assert re.match( - r".*Name +Size +Has +Wants\n" - r"-+ -+ -+ -+\n" - r"fred +512 +N +Y *\n" - r"sheila +512 +X +M *\n$", - result.output, - re.DOTALL, - ) - - -def test_where(fixtures): - """Test the 'acq where' command""" - runner = CliRunner() - - # Check help output - help_result = runner.invoke(cli.cli, ["acq", "where", "--help"]) - assert help_result.exit_code == 0 - assert "List locations of files that are in the ACQUISITION." in help_result.output - - # Fail when given a non-existent acquisition - result = runner.invoke(cli.cli, ["acq", "where", "z"]) - assert result.exit_code == 1 - assert "No such acquisition: z" in result.output - - # Check regular case - result = runner.invoke(cli.cli, ["acq", "where", "x"], catch_exceptions=False) - assert result.exit_code == 0 - assert re.match( - r"Storage node: x\n" - r".*Name +Size +Has +Wants\n" - r"-+ -+ -+ -+\n" - r"fred +512 +N +Y *\n" - r"sheila +512 +X +M *\n$", - result.output, - re.DOTALL, - ) - - # Now pretend node 'z' also has a copy of 'fred' - z_node = st.StorageNode.get(name="z") - fred_file = ar.ArchiveFile.get(name="fred") - fred2_copy = ar.ArchiveFileCopy.create( - file=fred_file, node=z_node, has_file="Y", wants_file="Y", size_b=123 - ) - result = runner.invoke(cli.cli, ["acq", "where", "x"], catch_exceptions=False) - assert result.exit_code == 0 - assert re.match( - r"Storage node: x\n" - r".*Name +Size +Has +Wants\n" - r"-+ -+ -+ -+\n" - r"fred +512 +N +Y *\n" - r"sheila +512 +X +M *\n\n" - r"Storage node: z\n" - r".*Name +Size +Has +Wants\n" - r"-+ -+ -+ -+\n" - r"fred +123 +Y +Y *\n$", - result.output, - re.DOTALL, - ) - - # Check when an acquisition does not have any files - zoo_acq = ac.ArchiveAcq.create(name="zoo", type=ac.AcqType.get(name="zab")) - result = runner.invoke(cli.cli, ["acq", "where", "zoo"], catch_exceptions=False) - assert result.exit_code == 0 - assert result.output == "No registered archive files.\n" - - # Check when there are no copies of an acquisition's files - ac.ArchiveFile.create(name="boo", acq=zoo_acq, type=ac.FileType.get(name="spqr")) - result = runner.invoke(cli.cli, ["acq", "where", "zoo"], catch_exceptions=False) - assert result.exit_code == 0 - assert result.output == "No registered archive files.\n" - - -def test_syncable(fixtures): - """Test the 'acq syncable' command""" - runner = CliRunner() - - # Check help output - help_result = runner.invoke(cli.cli, ["acq", "syncable", "--help"]) - assert help_result.exit_code == 0 - assert "List all files that are in the ACQUISITION" in help_result.output - - # Fail when given a non-existent acquisition - result = runner.invoke(cli.cli, ["acq", "syncable", "z", "x", "bar"]) - assert result.exit_code == 1 - assert "No such acquisition: z" in result.output - - # Fail when given a non-existent source node - result = runner.invoke(cli.cli, ["acq", "syncable", "x", "doesnotexist", "bar"]) - assert result.exit_code == 1 - assert "No such storage node: doesnotexist" in result.output - - # Fail when given a non-existent target group node - result = runner.invoke(cli.cli, ["acq", "syncable", "x", "x", "doesnotexist"]) - assert result.exit_code == 1 - assert "No such storage group: doesnotexist" in result.output - - # Check that initially there are no files in 'x' acq to copy from 'x' to 'bar' - result = runner.invoke(cli.cli, ["acq", "syncable", "x", "x", "bar"]) - assert result.exit_code == 0 - assert "No files to copy from node 'x' to group 'bar'" in result.output - - # Now pretend node 'x' has a copy of 'fred', 1 GB in size - fred_file = ar.ArchiveFile.get(name="fred") - fred_file.size_b = 1073741824.0 - fred_copy = fred_file.copies[0] - fred_copy.has_file = "Y" - fred_copy.save() - fred_file.save() - - # ...and a copy of 'sheila', 0 bytes in size - sheila_copy = ( - ar.ArchiveFileCopy.select() - .join(ac.ArchiveFile) - .where(ac.ArchiveFile.name == "sheila") - .get() - ) - sheila_copy.has_file = "Y" - sheila_copy.file.size_b = 0 - sheila_copy.save() - sheila_copy.file.save() - - # And so 'fred' and 'sheila' should be syncable from 'x' to 'bar' - result = runner.invoke(cli.cli, ["acq", "syncable", "x", "x", "bar"]) - assert result.exit_code == 0 - assert re.match( - r".*Name +Size\n" r"-+ -+\n" r"fred +1073741824 *\n" r"sheila +0 *\n$", - result.output, - re.DOTALL, - ) - - # Now pretend node 'z' also has a copy of 'fred' - z_node = st.StorageNode.get(name="z") - fred2_copy = ar.ArchiveFileCopy.create( - file=fred_file, node=z_node, has_file="Y", wants_file="Y", size_b=123 - ) - - # And so only 'sheila' should be syncable from 'x' to 'bar' - result = runner.invoke(cli.cli, ["acq", "syncable", "x", "x", "bar"]) - assert result.exit_code == 0 - assert re.match( - r".*Name +Size\n" r"-+ -+\n" r"sheila +0 *\n$", result.output, re.DOTALL - )