diff --git a/alpenhorn/cli/group/__init__.py b/alpenhorn/cli/group/__init__.py index a09a87843..4c629cc91 100644 --- a/alpenhorn/cli/group/__init__.py +++ b/alpenhorn/cli/group/__init__.py @@ -5,6 +5,7 @@ from ...db import StorageGroup, StorageNode +from .autosync import autosync from .create import create from .list import list_ from .modify import modify @@ -17,6 +18,7 @@ def cli(): """Manage Storage Groups.""" +cli.add_command(autosync, "auto-sync") cli.add_command(create, "create") cli.add_command(list_, "list") cli.add_command(modify, "modify") diff --git a/alpenhorn/cli/group/autosync.py b/alpenhorn/cli/group/autosync.py new file mode 100644 index 000000000..ee457a229 --- /dev/null +++ b/alpenhorn/cli/group/autosync.py @@ -0,0 +1,76 @@ +"""alpenhorn group auto-sync command""" + +import click +import peewee as pw + +from ...db import StorageGroup, StorageNode, StorageTransferAction, database_proxy +from ..cli import echo + + +@click.command() +@click.argument("group_name", metavar="GROUP") +@click.argument("node_name", metavar="NODE") +@click.option( + "--remove", + is_flag=True, + help="Remove (instead of add) NODE as an auto-sync source.", +) +@click.pass_context +def autosync(ctx, group_name, node_name, remove): + """Manage auto-sync sources for this group. + + This allows you to add (the default) or remove (using --remove) + the StorageNode named NODE as a an auto-sync souce for the Storage + Group named GROUP. + + If NODE is added as an auto-sync source for GROUP, then, whenever + a file is added to NODE, it will be automatically synced into the + Group GROUP, so long as the file isn't already present in the Group. + """ + + with database_proxy.atomic(): + try: + group = StorageGroup.get(name=group_name) + except pw.DoesNotExist: + raise click.ClickException(f"no such group: {group_name}") + + try: + node = StorageNode.get(name=node_name) + except pw.DoesNotExist: + raise click.ClickException(f"no such node: {node_name}") + + # Sanity check: can't auto-sync within a group + if group == node.group and not remove: + raise click.ClickException( + "can't enable auto-sync: " + f'Node "{node_name}" is in group "{group_name}"' + ) + + # What's the current state? + try: + action = StorageTransferAction.get(node_from=node, group_to=group) + if action.autosync is not remove: + echo("No change") + ctx.exit() + except pw.DoesNotExist: + # No need to create a record to set autosync to zero + if remove: + echo("No change") + ctx.exit() + action = None + + # Upsert the change + if action: + StorageTransferAction.update(autosync=not remove).where( + StorageTransferAction.id == action.id + ).execute() + else: + StorageTransferAction.create( + node_from=node, group_to=group, autosync=not remove + ) + + echo( + 'Auto-sync from "' + + node.name + + ('" started' if not remove else '" stopped.') + ) diff --git a/alpenhorn/cli/group/show.py b/alpenhorn/cli/group/show.py index ebdd88e62..0e2307fca 100644 --- a/alpenhorn/cli/group/show.py +++ b/alpenhorn/cli/group/show.py @@ -5,21 +5,32 @@ import peewee as pw from tabulate import tabulate -from ...db import StorageGroup, StorageNode +from ...db import StorageGroup, StorageNode, StorageTransferAction from ..cli import echo from ..node.stats import get_stats @click.command() @click.argument("group_name", metavar="GROUP") +@click.option( + "--actions", + is_flag=True, + help="Show post-transfer auto-actions affecting this group.", +) +@click.option("all_", "--all", "-a", is_flag=True, help="Show all additional data.") @click.option("--node-details", is_flag=True, help="Show details of listed nodes.") @click.option("--node-stats", is_flag=True, help="Show usage stats of listed nodes.") -def show(group_name, node_details, node_stats): +def show(group_name, actions, all_, node_details, node_stats): """Show details of a storage group. Shows details of the storage group named GROUP. """ + if all_: + node_details = True + node_stats = True + actions = True + try: group = StorageGroup.get(name=group_name) except pw.DoesNotExist: @@ -108,3 +119,36 @@ def show(group_name, node_details, node_stats): echo(" " + node.name) else: echo(" none") + + # List transfer actions, if requested + if actions: + echo("\nAuto-actions:\n") + + # It's possible for there to be entries in the table with nothing + # activated. So filter those out + query = ( + StorageTransferAction.select() + .join(StorageNode) + .where( + StorageTransferAction.group_to == group, + (StorageTransferAction.autosync == 1) + | (StorageTransferAction.autoclean == 1), + ) + .order_by(StorageTransferAction.node_from_id) + ) + + data = [] + for action in query.execute(): + if action.autoclean: + data.append( + (action.node_from.name, "Auto-clean", "File added to this group") + ) + if action.autosync: + data.append( + (action.node_from.name, "Auto-sync", "File added to that node") + ) + + if data: + echo(tabulate(data, headers=["Node", "Action", "Trigger"])) + else: + echo(" none") diff --git a/alpenhorn/cli/node/__init__.py b/alpenhorn/cli/node/__init__.py index d6d06e0c3..071805db9 100644 --- a/alpenhorn/cli/node/__init__.py +++ b/alpenhorn/cli/node/__init__.py @@ -13,6 +13,7 @@ from ...db import ArchiveAcq, ArchiveFile, ArchiveFileCopy, StorageGroup, StorageNode from .activate import activate +from .autoclean import autoclean from .clean import clean from .create import create from .deactivate import deactivate @@ -31,6 +32,7 @@ def cli(): cli.add_command(activate, "activate") +cli.add_command(autoclean, "auto-clean") cli.add_command(clean, "clean") cli.add_command(create, "create") cli.add_command(deactivate, "deactivate") diff --git a/alpenhorn/cli/node/autoclean.py b/alpenhorn/cli/node/autoclean.py new file mode 100644 index 000000000..16ed09fd5 --- /dev/null +++ b/alpenhorn/cli/node/autoclean.py @@ -0,0 +1,76 @@ +"""alpenhorn node auto-clean command""" + +import click +import peewee as pw + +from ...db import StorageGroup, StorageNode, StorageTransferAction, database_proxy +from ..cli import echo + + +@click.command() +@click.argument("node_name", metavar="NODE") +@click.argument("group_name", metavar="GROUP") +@click.option( + "--remove", + is_flag=True, + help="Remove (instead of add) GROUP as an auto-clean trigger.", +) +@click.pass_context +def autoclean(ctx, group_name, node_name, remove): + """Manage auto-clean triggers for this node. + + This allows you to add (the default) or remove (using --remove) + the StorageGroup named GROUP as a an auto-clean trigger for the + Storage Node named NODE. + + If GROUP is added as an auto-clean trigger for NODE, then, whenever + a file is added to GROUP, it will be automatically released for + deletion on NODE. + """ + + with database_proxy.atomic(): + try: + node = StorageNode.get(name=node_name) + except pw.DoesNotExist: + raise click.ClickException(f"no such node: {node_name}") + + try: + group = StorageGroup.get(name=group_name) + except pw.DoesNotExist: + raise click.ClickException(f"no such group: {group_name}") + + # Sanity check: can't auto-clean within a group + if group == node.group and not remove: + raise click.ClickException( + "can't enable auto-clean: " + f'Node "{node_name}" is in group "{group_name}"' + ) + + # What's the current state? + try: + action = StorageTransferAction.get(node_from=node, group_to=group) + if action.autoclean is not remove: + echo("No change") + ctx.exit() + except pw.DoesNotExist: + # No need to create a record to set autoclean to zero + if remove: + echo("No change") + ctx.exit() + action = None + + # Upsert the change + if action: + StorageTransferAction.update(autoclean=not remove).where( + StorageTransferAction.id == action.id + ).execute() + else: + StorageTransferAction.create( + node_from=node, group_to=group, autoclean=not remove + ) + + echo( + 'Auto-clean trigger: Group "' + + group.name + + ('" added' if not remove else '" removed.') + ) diff --git a/alpenhorn/cli/node/show.py b/alpenhorn/cli/node/show.py index d71c00619..1686af323 100644 --- a/alpenhorn/cli/node/show.py +++ b/alpenhorn/cli/node/show.py @@ -3,22 +3,33 @@ import json import click import peewee as pw +from tabulate import tabulate from ...common.util import pretty_bytes -from ...db import StorageGroup, StorageNode +from ...db import StorageGroup, StorageNode, StorageTransferAction from ..cli import echo from .stats import get_stats @click.command() @click.argument("name", metavar="NAME") +@click.option( + "--actions", + is_flag=True, + help="Show post-transfer auto-actions affecting this group.", +) +@click.option("all_", "--all", "-a", is_flag=True, help="Show all additional data.") @click.option("--stats", is_flag=True, help="Show usage stats of the node.") -def show(name, stats): +def show(name, actions, all_, stats): """Show details of a Storage Node. Shows details of the Storage Node named NODE. """ + if all_: + actions = True + stats = True + try: node = StorageNode.get(name=name) except pw.DoesNotExist: @@ -96,3 +107,36 @@ def show(name, stats): echo(" Total Files: " + str(stats["count"])) echo(" Total Size: " + stats["size"]) echo(" Usage: " + stats["percent"].lstrip() + "%") + + # List transfer actions, if requested + if actions: + echo("\nAuto-actions:\n") + + # It's possible for there to be entries in the table with nothing + # activated. So filter those out + query = ( + StorageTransferAction.select() + .join(StorageGroup) + .where( + StorageTransferAction.node_from == node, + (StorageTransferAction.autosync == 1) + | (StorageTransferAction.autoclean == 1), + ) + .order_by(StorageTransferAction.group_to_id) + ) + + data = [] + for action in query.execute(): + if action.autoclean: + data.append( + (action.group_to.name, "Auto-clean", "File added to that group") + ) + if action.autosync: + data.append( + (action.group_to.name, "Auto-sync", "File added to this node") + ) + + if data: + echo(tabulate(data, headers=["Group", "Action", "Trigger"])) + else: + echo(" none") diff --git a/tests/cli/group/test_autosync.py b/tests/cli/group/test_autosync.py new file mode 100644 index 000000000..3b768aa80 --- /dev/null +++ b/tests/cli/group/test_autosync.py @@ -0,0 +1,106 @@ +"""Test CLI: alpenhorn group auto-sync""" + +import pytest +from alpenhorn.db import StorageGroup, StorageNode, StorageTransferAction + + +def test_no_group(clidb, cli): + """Test auto-sync with a bad group name.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(1, ["group", "auto-sync", "MISSING", "Node"]) + + +def test_no_node(clidb, cli): + """Test auto-sync with a bad node name.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(1, ["group", "auto-sync", "Group", "Node"]) + + +def test_node_in_group(clidb, cli): + """Can't start auto-sync with NODE in GROUP.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(1, ["group", "auto-sync", "Group", "Node"]) + + +def test_stop_node_in_group(clidb, cli): + """But we can stop auto-sync with NODE in GROUP.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(0, ["group", "auto-sync", "Group", "Node", "--remove"]) + + +def test_start_noop(clidb, cli): + """Test auto-sync already on.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autosync=1) + + cli(0, ["group", "auto-sync", "Group", "Node"]) + + +def test_stop_noop(clidb, cli): + """Test stopping auto-sync already explicity off.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autosync=0) + + cli(0, ["group", "auto-sync", "Group", "Node", "--remove"]) + + assert not StorageTransferAction.get(node_from=node, group_to=group).autosync + + +def test_start_from_stop(clidb, cli): + """Test starting auto-sync already explicitly off.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autosync=0) + + cli(0, ["group", "auto-sync", "Group", "Node"]) + + assert StorageTransferAction.get(node_from=node, group_to=group).autosync + + +def test_stop_from_start(clidb, cli): + """Test starting auto-sync already explicitly off.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autosync=1) + + cli(0, ["group", "auto-sync", "Group", "Node", "--remove"]) + + assert not StorageTransferAction.get(node_from=node, group_to=group).autosync + + +def test_start_create(clidb, cli): + """Test starting auto-sync through record creation.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + cli(0, ["group", "auto-sync", "Group", "Node"]) + + assert StorageTransferAction.get(node_from=node, group_to=group).autosync diff --git a/tests/cli/group/test_show.py b/tests/cli/group/test_show.py index ccf60645b..65bd60b65 100644 --- a/tests/cli/group/test_show.py +++ b/tests/cli/group/test_show.py @@ -4,6 +4,7 @@ from alpenhorn.db import ( StorageGroup, StorageNode, + StorageTransferAction, ArchiveAcq, ArchiveFile, ArchiveFileCopy, @@ -174,3 +175,94 @@ def test_show_node_details_stats(clidb, cli, assert_row_present): assert_row_present( result.output, "Node2", "over_there", "No", "NodeClass", 2, "5.665 kiB", "0.00" ) + + +def test_show_actions(clidb, cli, assert_row_present): + """Test show --actions.""" + + group = StorageGroup.create(name="Group1") + + group2 = StorageGroup.create(name="Group2") + node = StorageNode.create(name="Node1", group=group2) + StorageTransferAction.create( + node_from=node, group_to=group, autosync=1, autoclean=1 + ) + + node = StorageNode.create(name="Node2", group=group2) + StorageTransferAction.create( + node_from=node, group_to=group, autosync=0, autoclean=1 + ) + + node = StorageNode.create(name="Node3", group=group2) + StorageTransferAction.create( + node_from=node, group_to=group, autosync=1, autoclean=0 + ) + + node = StorageNode.create(name="Node4", group=group2) + StorageTransferAction.create( + node_from=node, group_to=group, autosync=0, autoclean=0 + ) + + node = StorageNode.create(name="Node5", group=group2) + StorageTransferAction.create( + node_from=node, group_to=group2, autosync=1, autoclean=1 + ) + + result = cli(0, ["group", "show", "Group1", "--actions"]) + + # Nodes 1 and 2 are autocleaned + assert_row_present(result.output, "Node1", "Auto-clean", "File added to this group") + assert_row_present(result.output, "Node2", "Auto-clean", "File added to this group") + + # Nodes 1 and 3 are autosynced + assert_row_present(result.output, "Node1", "Auto-sync", "File added to that node") + assert_row_present(result.output, "Node3", "Auto-sync", "File added to that node") + + assert "Node4" not in result.output + assert "Node5" not in result.output + + +def test_show_all(clidb, cli, assert_row_present): + """Test show --all.""" + + # Make a StorageGroup with some nodes in it. + group = StorageGroup.create(name="SGroup", io_class="IOClass") + node1 = StorageNode.create(name="Node1", group=group, active=True, host="over_here") + node2 = StorageNode.create( + name="Node2", + group=group, + active=False, + host="over_there", + io_class="NodeClass", + max_total_gb=1, + ) + + StorageTransferAction.create( + node_from=node1, group_to=group, autosync=1, autoclean=1 + ) + + # And some files + acq = ArchiveAcq.create(name="acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=1234) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="Y") + ArchiveFileCopy.create(file=file, node=node2, has_file="X", wants_file="Y") + + file = ArchiveFile.create(name="File2", acq=acq, size_b=2345) + ArchiveFileCopy.create(file=file, node=node1, has_file="N", wants_file="Y") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + + file = ArchiveFile.create(name="File3", acq=acq, size_b=3456) + ArchiveFileCopy.create(file=file, node=node1, has_file="Y", wants_file="Y") + ArchiveFileCopy.create(file=file, node=node2, has_file="Y", wants_file="Y") + + result = cli(0, ["group", "show", "SGroup", "--all"]) + + assert_row_present( + result.output, "Node1", "over_here", "Yes", "Default", 2, "4.580 kiB", "-" + ) + assert_row_present( + result.output, "Node2", "over_there", "No", "NodeClass", 2, "5.665 kiB", "0.00" + ) + + assert_row_present(result.output, "Node1", "Auto-clean", "File added to this group") + assert_row_present(result.output, "Node1", "Auto-sync", "File added to that node") diff --git a/tests/cli/node/test_autoclean.py b/tests/cli/node/test_autoclean.py new file mode 100644 index 000000000..fb88537b7 --- /dev/null +++ b/tests/cli/node/test_autoclean.py @@ -0,0 +1,106 @@ +"""Test CLI: alpenhorn node auto-clean""" + +import pytest +from alpenhorn.db import StorageGroup, StorageNode, StorageTransferAction + + +def test_no_node(clidb, cli): + """Test auto-clean with a bad node name.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(1, ["node", "auto-clean", "MISSING", "Group"]) + + +def test_no_group(clidb, cli): + """Test auto-clean with a bad group name.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(1, ["node", "auto-clean", "Node", "MISSING"]) + + +def test_node_in_group(clidb, cli): + """Can't add auto-clean with NODE in GROUP.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(1, ["node", "auto-clean", "Node", "Group"]) + + +def test_remove_node_in_group(clidb, cli): + """But we can remove auto-clean with NODE in GROUP.""" + + group = StorageGroup.create(name="Group") + StorageNode.create(name="Node", group=group) + + cli(0, ["node", "auto-clean", "Node", "Group", "--remove"]) + + +def test_add_noop(clidb, cli): + """Test auto-clean already on.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autoclean=1) + + cli(0, ["node", "auto-clean", "Node", "Group"]) + + +def test_remove_noop(clidb, cli): + """Test removing auto-clean already explicity off.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autoclean=0) + + cli(0, ["node", "auto-clean", "Node", "Group", "--remove"]) + + assert not StorageTransferAction.get(node_from=node, group_to=group).autoclean + + +def test_add_from_remove(clidb, cli): + """Test adding auto-clean already explicitly off.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autoclean=0) + + cli(0, ["node", "auto-clean", "Node", "Group"]) + + assert StorageTransferAction.get(node_from=node, group_to=group).autoclean + + +def test_remove_from_add(clidb, cli): + """Test adding auto-clean already explicitly off.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + StorageTransferAction.create(node_from=node, group_to=group, autoclean=1) + + cli(0, ["node", "auto-clean", "Node", "Group", "--remove"]) + + assert not StorageTransferAction.get(node_from=node, group_to=group).autoclean + + +def test_add_create(clidb, cli): + """Test adding auto-clean through record creation.""" + + group = StorageGroup.create(name="NodeGroup") + node = StorageNode.create(name="Node", group=group) + group = StorageGroup.create(name="Group") + + cli(0, ["node", "auto-clean", "Node", "Group"]) + + assert StorageTransferAction.get(node_from=node, group_to=group).autoclean diff --git a/tests/cli/node/test_show.py b/tests/cli/node/test_show.py index 11a0581cb..2234abf31 100644 --- a/tests/cli/node/test_show.py +++ b/tests/cli/node/test_show.py @@ -4,6 +4,7 @@ from alpenhorn.db import ( StorageGroup, StorageNode, + StorageTransferAction, ArchiveAcq, ArchiveFile, ArchiveFileCopy, @@ -135,3 +136,92 @@ def test_show_node_stats(clidb, cli): # 4.58 out of 8 == 57.25 percent assert "57.25%" in result.output + + +def test_show_actions(clidb, cli, assert_row_present): + """Test show --actions.""" + + group1 = StorageGroup.create(name="Group1") + node1 = StorageNode.create(name="Node1", group=group1) + + group2 = StorageGroup.create(name="Group2") + node2 = StorageNode.create(name="Node2", group=group2) + + StorageTransferAction.create( + node_from=node1, group_to=group2, autosync=1, autoclean=1 + ) + + group = StorageGroup.create(name="Group3") + StorageTransferAction.create( + node_from=node1, group_to=group, autosync=0, autoclean=1 + ) + + group = StorageGroup.create(name="Group4") + StorageTransferAction.create( + node_from=node1, group_to=group, autosync=1, autoclean=0 + ) + + group = StorageGroup.create(name="Group5") + StorageTransferAction.create( + node_from=node1, group_to=group, autosync=0, autoclean=0 + ) + + group = StorageGroup.create(name="Group6") + StorageTransferAction.create( + node_from=node2, group_to=group, autosync=1, autoclean=1 + ) + + result = cli(0, ["node", "show", "Node1", "--actions"]) + + # Groups 2 and 3 are autocleaned + assert_row_present( + result.output, "Group2", "Auto-clean", "File added to that group" + ) + assert_row_present( + result.output, "Group3", "Auto-clean", "File added to that group" + ) + + # Groups 2 and 4 are autosynced + assert_row_present(result.output, "Group2", "Auto-sync", "File added to this node") + assert_row_present(result.output, "Group4", "Auto-sync", "File added to this node") + + assert "Node5" not in result.output + assert "Node6" not in result.output + + +def test_show_all(clidb, cli, assert_row_present): + """Test show --all.""" + + group = StorageGroup.create(name="Group") + node = StorageNode.create( + name="Node", + group=group, + active=True, + max_total_gb=2**-17, # 2**(30-17) == 2**13 == 8 kiB + ) + StorageTransferAction.create( + node_from=node, group_to=group, autosync=1, autoclean=1 + ) + + acq = ArchiveAcq.create(name="acq") + file = ArchiveFile.create(name="File1", acq=acq, size_b=1234) + ArchiveFileCopy.create(file=file, node=node, has_file="Y", wants_file="Y") + + file = ArchiveFile.create(name="File2", acq=acq, size_b=2345) + ArchiveFileCopy.create(file=file, node=node, has_file="X", wants_file="Y") + + file = ArchiveFile.create(name="File3", acq=acq, size_b=3456) + ArchiveFileCopy.create(file=file, node=node, has_file="Y", wants_file="Y") + + result = cli(0, ["node", "show", "Node", "--all"]) + + assert "Total Files: 2" in result.output + + # 1234 + 3456 = 4690 bytes = 4.580078 kiB + assert "4.580 kiB" in result.output + + # 4.58 out of 8 == 57.25 percent + assert "57.25%" in result.output + + assert_row_present(result.output, "Group", "Auto-clean", "File added to that group") + assert_row_present(result.output, "Group", "Auto-sync", "File added to this node")