From 0a396cc60dfddd4009f5d386ae2817a4bfb044a2 Mon Sep 17 00:00:00 2001 From: lpalovsky Date: Mon, 16 Dec 2024 12:21:54 +0100 Subject: [PATCH] Stop hana test module --- lib/hacluster.pm | 74 +++++++ lib/saputils.pm | 45 +++++ lib/sles4sap.pm | 1 + lib/sles4sap/database_hana.pm | 183 ++++++++++++++++++ lib/sles4sap/sap_host_agent.pm | 102 ++++++++++ .../hanasr_redirected.yml | 9 +- t/11_hacluster.t | 39 ++++ t/18_saputils.t | 16 ++ t/31_sap_host_agent.t | 57 ++++++ t/32_database_hana.pm | 72 +++++++ .../SLES4SAP_HA_data_structures.md | 14 ++ .../hana_sr_primary_failover.pm | 130 +++++++++++++ .../hanasr_schedule_tests.pm | 83 ++++++++ .../redirection_tests/redirection_check.pm | 13 +- .../deploy_hanasr.pm | 7 + 15 files changed, 838 insertions(+), 7 deletions(-) create mode 100644 lib/sles4sap/database_hana.pm create mode 100644 lib/sles4sap/sap_host_agent.pm create mode 100644 t/31_sap_host_agent.t create mode 100644 t/32_database_hana.pm create mode 100644 tests/sles4sap/redirection_tests/SLES4SAP_HA_data_structures.md create mode 100644 tests/sles4sap/redirection_tests/hana_sr_primary_failover.pm create mode 100644 tests/sles4sap/redirection_tests/hanasr_schedule_tests.pm diff --git a/lib/hacluster.pm b/lib/hacluster.pm index 743ad8b1bf56..e475158a36f5 100644 --- a/lib/hacluster.pm +++ b/lib/hacluster.pm @@ -83,6 +83,9 @@ our @EXPORT = qw( crm_wait_for_maintenance crm_check_resource_location generate_lun_list + show_cluster_parameter + set_cluster_parameter + show_hana_resource_name ); =head1 SYNOPSIS @@ -1460,4 +1463,75 @@ sub generate_lun_list { $index += $num_luns; } } + + +=head2 set_cluster_parameter + + set_cluster_parameter(); + +Manage HA cluster parameter using crm shell. + +=over + +=item * B: Resource containing parameter + +=item * B: Parameter name + +=item * B: Target parameter value + +=back + +=cut + +sub set_cluster_parameter { + my (%args) = @_; + for my $arg ('resource', 'parameter', 'value') { + croak("Mandatory argument '$arg' missing.") unless $arg; + } + my $cmd = join(' ', 'crm', 'resource', 'param', $args{resource}, 'set', $args{parameter}, $args{value}); + assert_script_run($cmd); +} + +=head2 show_cluster_parameter + + show_cluster_parameter(); + +Show cluster parameter value using CRM shell. + +=over + +=item * B: Resource containing parameter + +=item * B: Parameter name + +=back + +=cut + +sub show_cluster_parameter { + my (%args) = @_; + for my $arg ('resource', 'parameter') { + croak("Mandatory argument '$arg' missing.") unless $arg; + } + my $cmd = join(' ', 'crm', 'resource', 'param', $args{resource}, 'show', $args{parameter}); + return script_output($cmd); +} + +=head2 show_hana_resource_name + + show_hana_resource_name(); + +Show SAP hana resource name. + +=cut + +sub show_hana_resource_name { + my $cmd = join(' ', 'crm', 'configure', 'show', 'related:ocf:suse:SAPHana', + '| awk \'$1 == "primitive" && $3 == "ocf:suse:SAPHana" {print $2}\''); + my $cmd_output = script_output($cmd); + # additional check if returned HANA resource exists + assert_script_run("crm resource status $cmd_output"); + return $cmd_output; +} + 1; diff --git a/lib/saputils.pm b/lib/saputils.pm index db4691135113..8a11bd7f52b7 100644 --- a/lib/saputils.pm +++ b/lib/saputils.pm @@ -20,6 +20,8 @@ our @EXPORT = qw( calculate_hana_topology check_hana_topology check_crm_output + get_primary_node + get_failover_node ); =head1 SYNOPSIS @@ -176,4 +178,47 @@ sub check_crm_output { return (($resource_starting != 1) && ($failed_actions != 1) ? 1 : 0); } +=head2 get_primary_node + get_primary_node(); + + Returns hostname of current primary node obtained from C command output + +=over + +=item B - return value of calculate_hana_topology + +=back +=cut + +sub get_primary_node { + my (%args) = @_; + croak("Argument missing") unless $args{input}; + my $topology = $args{input}; + for my $db (keys %$topology) { + return $db if $topology->{$db}{sync_state} eq 'PRIM'; + } +} + +=head2 get_failover_node + get_failover_node(); + + Returns hostname of current failover (replica) node obtained from C command output. + Returns node hostname even if it's in 'SFAIL' state. + +=over + +=item B - return value of calculate_hana_topology + +=back +=cut + +sub get_failover_node { + my (%args) = @_; + croak("Argument missing") unless $args{input}; + my $topology = $args{input}; + for my $db (keys %$topology) { + return $db if grep /$topology->{$db}{sync_state}/, ('SOK', 'SFAIL'); + } +} + 1; diff --git a/lib/sles4sap.pm b/lib/sles4sap.pm index 462e7b024b13..981701e9f6ae 100644 --- a/lib/sles4sap.pm +++ b/lib/sles4sap.pm @@ -71,6 +71,7 @@ our @EXPORT = qw( get_instance_profile_path load_ase_env upload_ase_logs + saphostctrl_list_databases ); =head1 SYNOPSIS diff --git a/lib/sles4sap/database_hana.pm b/lib/sles4sap/database_hana.pm new file mode 100644 index 000000000000..b5a9b094bfdf --- /dev/null +++ b/lib/sles4sap/database_hana.pm @@ -0,0 +1,183 @@ +# SUSE's openQA tests +# +# Copyright 2017-2024 SUSE LLC +# SPDX-License-Identifier: FSFAP +# +# Summary: Functions for SAP tests +# Maintainer: QE-SAP + +package sles4sap::database_hana; +use strict; +use warnings; +use testapi; +use Carp qw(croak); +use Exporter qw(import); +use saputils qw(check_crm_output get_primary_node get_failover_node calculate_hana_topology); +use sles4sap::sapcontrol; + +our @EXPORT = qw( + hdb_stop + wait_for_failed_resources + wait_for_takeover + register_replica + get_node_roles +); + + +=head1 SYNOPSIS + +Package contains functions for interacting with hana database and related actions. + +=cut + +=head2 sudo + + sudo($activate); + +Return string 'sudo ' (space included) if there is any 'true' equivalent passed as an argument, otherwise return empty string. +This is to be used to prepend 'sudo' to any command under a condition. +Example: script_run(sudo($args{as_root}) . 'whoami'); + +=over + +=item * B<$activate>: Any value which is an equivalent to 'true' makes functkion return 'sudo'. Default: false + +=back + +=cut + +sub sudo { + return ('sudo ') if @_; +} + +=head2 hdb_stop + + hdb_stop(instance_id=>'00', [switch_user=>'sidadm']); + +Stop hana database using C command. Function expects to be executed as sidadm, however you can use B +to execute command using C as a different user. The user needs to have correct permissions for performing +requested action. +Function waits till all DB processes are stopped. + +=over + +=item * B: Database instance ID. Mandatory. + +=item * B: Execute command as specified user with help of C. Default: undef + +=item * B: HDB command to trigger. Default: stop + +=back + +=cut + +sub hdb_stop { + my (%args) = @_; + my $stop_timeout = 600; + $args{command} //= 'stop'; + croak("Command '$args{command}' is not supported.") unless grep(/$args{command}/, ('kill', 'stop')); + + $args{command} = 'kill -x' if( $args{command} eq 'kill'); + my $sudo_su = $args{switch_user} ? "sudo su - $args{switch_user} -c" : ''; + my $cmd = join(' ', $sudo_su, '"', 'HDB', $args{command}, '"'); + record_info('HDB stop', "Executing '$cmd' on " . script_output('hostname')); + assert_script_run($cmd, timeout => $stop_timeout); + sapcontrol_process_check(instance_id => $args{instance_id}, expected_state => 'stopped', wait_for_state => 'yes', timeout => $stop_timeout); + record_info('DB stopped'); +} + +=head2 wait_for_failed_resources + + wait_for_failed_resources(); + +Wait until 'crm_mon' starts showing failed resources. This can be used as first indicator of a started failover. + +=cut + +sub wait_for_failed_resources { + my $timeout = 300; + my $start_time = time; + while (check_crm_output(input => script_output('crm_mon -R -r -n -N -1', quiet => 1))) { + sleep 30; + die("Cluster did not register any failed resource within $timeout sec") if (time - $timeout > $start_time); + } + record_info('CRM info', "Cluster registered failed resources\n" . script_output('crm_mon -R -r -n -N -1', quiet => 1)); +} + +=head2 wait_for_takeover + + wait_for_takeover(target_node=>'expeliarmus'); + +Waits until B performs takeover and reaches 'PRIM' state. + +=over + +=item * B: Node hostname which is expected to take over. + +=back + +=cut + +sub wait_for_takeover { + my (%args) = @_; + my $timeout = 300; + my $start_time = time; + my $topology; + my $takeover_ok; + until ($takeover_ok) { + die("Node '$args{target_node}' did not take over within $timeout sec") if (time - $timeout > $start_time); + $topology = calculate_hana_topology(input => script_output('SAPHanaSR-showAttr --format=script')); + $takeover_ok = 1 if (get_primary_node(input => $topology) eq $args{target_node}); + sleep 30; + } +} + +=head2 register_replica + + register_replica(target_hostname=>'Dumbledore', instance_id=>'00'); + +Executes + +=over + +=item * B: Hostname of the node that should be registered as replica + +=item * B: Instance ID + +=item * B: Execute command as specified user with help of C. Default: undef + + +=back + +=cut + +sub register_replica { + my (%args) = @_; + croak('Argument "$replica_hostname" missing') unless $args{target_hostname}; + my $topology = calculate_hana_topology(input => script_output('SAPHanaSR-showAttr --format=script')); + my $primary_hostname = get_primary_node(input => $topology); + croak("Primary node '$primary_hostname' not found in 'SAPHanaSR-showAttr' output") unless $primary_hostname; + croak("Replica node '$args{target_hostname}' not found in 'SAPHanaSR-showAttr' output") unless + $topology->{$args{target_hostname}}; + + my $cmd = join(' ', + 'hdbnsutil', + '-sr_register', + "--remoteHost=$primary_hostname", + "--remoteInstance=$args{instance_id}", + "--replicationMode=$topology->{$args{target_hostname}}{srmode}", + "--operationMode=$topology->{$args{target_hostname}}{op_mode}", + "--name=$topology->{$args{target_hostname}}{site}", + '--online'); + $cmd = join(' ', 'sudo', 'su', '-', $args{switch_user}, '-c', '"', $cmd, '"') if $args{switch_user}; + assert_script_run($cmd); +} + + + +sub get_node_roles { + my $topology = calculate_hana_topology(input => script_output('SAPHanaSR-showAttr --format=script')); + my $primary_db = get_primary_node(input => $topology); + my $replica_db = get_failover_node(input => $topology); + return(($primary_db, $replica_db)); +} \ No newline at end of file diff --git a/lib/sles4sap/sap_host_agent.pm b/lib/sles4sap/sap_host_agent.pm new file mode 100644 index 000000000000..15eb9e3814ba --- /dev/null +++ b/lib/sles4sap/sap_host_agent.pm @@ -0,0 +1,102 @@ +# SUSE's openQA tests +# +# Copyright 2017-2024 SUSE LLC +# SPDX-License-Identifier: FSFAP +# +# Summary: Functions for SAP tests +# Maintainer: QE-SAP + +package sles4sap::sap_host_agent; +use strict; +use warnings; +use testapi; +use Carp qw(croak); +use Exporter qw(import); + +our @EXPORT = qw( + saphostctrl_list_databases + parse_instance_name +); + +my $saphostctrl = '/usr/sap/hostctrl/exe/saphostctrl'; + +=head1 SYNOPSIS + +Package with functions related to interaction with SAP Host Agent (Command B). Those can be used for collecting +data about instances and performing various operations. Keep in mind that command needs to be executed using either +B or B. + +=cut + +=head2 saphostctrl_list_databases + + saphostctrl_list_databases([as_root=>1]); + +Uses C to get list of all databases residing on host. Returns parsed output as B. +Data for each DB is contained in a B +Example: +$VAR1 = { + 'Hostname' => 'qesdhdb01l029', + 'Release' => '2.00.075.00.1702888292', + 'Vendor' => 'HDB', # HDB = SAP hana database (not SID) + 'Type' => 'hdb', # type of hana DB - hdb, mdc (multitenant), systemdb + 'Instance name' => 'PRD00' + }; +$VAR2 = { + 'Hostname' => 'qesdhdb01l029', + 'Type' => 'hdb', + 'Release' => '2.00.075.00.1702888292', + 'Instance name' => 'QAS01', + 'Vendor' => 'HDB' + }; + +=over + +=item * B: Execute command using sudo. Default: false + +=back +=cut + +sub saphostctrl_list_databases { + my (%args) = @_; + die("Missing saphostctrl binary: $saphostctrl") + if script_run("[ -f $saphostctrl ]"); + + # command returns data for each DB in new line = one array entry for each DB + my $sudo = $args{as_root} ? 'sudo' : ''; + my $cmd = join(' ', $sudo, $saphostctrl, '-function', 'ListDatabases', '| grep Instance'); + my @output = split("\n", script_output($cmd, timeout => 180)); + + # create hash with data from each array entry + @output = map { { split(/,\s|:\s/, $_) } } @output; + + my @result; + # change hash keys to lower case and replace spaces with underscore + for my $entry (@output) { + push(@result, {map { lc($_ =~ s/\s/_/gr) => $entry->{$_} } keys %$entry}); + } + return (\@result); +} + +=head2 parse_instance_name + + parse_instance_name($instance_name); + +Splits instance name into B and B. Example: DBH01 -> sid=DBH, id=01 + +=over + +=item * B<$instance_name>: Instance name + +=back + +=cut + +sub parse_instance_name { + my ($instance_name) = @_; + croak("Invalid instance name: $instance_name\nInstance name is a combination of SID and instance ID.") if + length($instance_name) != 5 or grep(/\s|[a-z]|\W/, $instance_name); + my @result = $instance_name =~ /(.{3})(.{2})/s; + return (\@result); +} + diff --git a/schedule/sles4sap/sap_deployment_automation_framework/hanasr_redirected.yml b/schedule/sles4sap/sap_deployment_automation_framework/hanasr_redirected.yml index 3e63f42af9eb..dc2ad7e0733a 100644 --- a/schedule/sles4sap/sap_deployment_automation_framework/hanasr_redirected.yml +++ b/schedule/sles4sap/sap_deployment_automation_framework/hanasr_redirected.yml @@ -10,6 +10,9 @@ schedule: - boot/boot_to_desktop - sles4sap/sap_deployment_automation_framework/connect_to_deployer - sles4sap/sap_deployment_automation_framework/prepare_ssh_config - - sles4sap/redirection_tests/redirection_check - - sles4sap/redirection_tests/hana_cluster_check - - sles4sap/sap_deployment_automation_framework/cleanup + #- sles4sap/redirection_tests/redirection_check + #- sles4sap/redirection_tests/hana_cluster_check + #- sles4sap/redirection_tests/hanasr_schedule_tests + - sles4sap/redirection_tests/hana_sr_primary_failover + - sles4sap/redirection_tests/hana_sr_primary_failover + #- sles4sap/sap_deployment_automation_framework/cleanup diff --git a/t/11_hacluster.t b/t/11_hacluster.t index d3d0f3d27b04..5ff72e1127f9 100644 --- a/t/11_hacluster.t +++ b/t/11_hacluster.t @@ -242,4 +242,43 @@ subtest '[crm_check_resource_location]' => sub { $hostname, "Return correct hostname: $hostname"; }; +subtest '[set_cluster_parameter]' => sub { + my $hacluster = Test::MockModule->new('hacluster', no_auto => 1); + my @calls; + $hacluster->redefine(assert_script_run => sub { @calls = @_; return; }); + + set_cluster_parameter(resource=>'Hogwarts', parameter=>'RoomOfRequirement', value=>'open'); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /crm/, @calls), 'Execute "crm" command.'); + ok((grep /resource param Hogwarts/, @calls), 'Call "resource" option'); + ok((grep /set/, @calls), 'Specify "set" action'); + ok((grep /RoomOfRequirement open/, @calls), 'Specify parameter name'); +}; + + +subtest '[show_cluster_parameter]' => sub { + my $hacluster = Test::MockModule->new('hacluster', no_auto => 1); + my @calls; + $hacluster->redefine(script_output => sub { @calls = @_; return 'false'; }); + + show_cluster_parameter(resource=>'Hogwarts', parameter=>'RoomOfRequirement'); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /crm/, @calls), 'Execute "crm" command.'); + ok((grep /resource param Hogwarts/, @calls), 'Call "resource" option'); + ok((grep /show/, @calls), 'Specify "show" action'); + ok((grep /RoomOfRequirement/, @calls), 'Specify parameter name'); +}; + +subtest '[show_hana_resource_name]' => sub { + my $hacluster = Test::MockModule->new('hacluster', no_auto => 1); + my @calls; + $hacluster->redefine(script_output => sub { @calls = @_; return 'Dumbledore'; }); + $hacluster->redefine(assert_script_run => sub { return 0; }); + + my $returned_value = show_hana_resource_name(); + note("\n --> " . join("\n --> ", @calls)); + is $returned_value, 'Dumbledore', 'Check for correct value returned'; + +}; + done_testing; diff --git a/t/18_saputils.t b/t/18_saputils.t index 0534f438f048..98b05b8b96ec 100644 --- a/t/18_saputils.t +++ b/t/18_saputils.t @@ -256,4 +256,20 @@ subtest '[check_crm_output] starting and failed' => sub { ok $ret eq 0, "Ret:$ret has to be 0"; }; +subtest '[get_primary_node] starting and failed' => sub { + my $mock_input = { + hana_node_01 => {sync_state => 'PRIM'}, + hana_node_02 => {sync_state => 'SOK'} + }; + is get_primary_node(input => $mock_input), 'hana_node_01', 'Return correct primary node name'; +}; + +subtest '[get_failover_node] starting and failed' => sub { + my $mock_input = { + hana_node_01 => {sync_state => 'PRIM'}, + hana_node_02 => {sync_state => 'SOK'} + }; + is get_failover_node(input => $mock_input), 'hana_node_02', 'Return correct primary node name'; +}; + done_testing; diff --git a/t/31_sap_host_agent.t b/t/31_sap_host_agent.t new file mode 100644 index 000000000000..8e808ca6481c --- /dev/null +++ b/t/31_sap_host_agent.t @@ -0,0 +1,57 @@ +use strict; +use warnings; +use Test::More; +use Test::Exception; +use Test::Warnings; +use Test::MockModule; +use testapi; +use sles4sap::sap_host_agent; + +subtest '[saphostctrl_list_databases] Verify command compilation' => sub { + my $saphostctrl_output = 'Instance name: PRD00, Hostname: qesdhdb01l029, Vendor: HDB, Type: hdb, Release: 42'; + my $mock = Test::MockModule->new('sles4sap::sap_host_agent', no_auto => 1); + my @calls; + $mock->redefine(script_output => sub { push @calls, $_[0]; return $saphostctrl_output; }); + $mock->redefine(script_run => sub { return 0; }); + + saphostctrl_list_databases(); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /saphostctrl/, @calls), 'Execute "saphostctrl" binary'); + ok((grep /-function ListDatabases/, @calls), 'Execute "ListDatabases" fucntion'); + ok((grep /\| grep Instance/, @calls), 'Show only "Instances" entries'); + + saphostctrl_list_databases(as_root => 1); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /sudo/, @calls), 'Execute as root'); +}; + +subtest '[saphostctrl_list_databases] Verify output' => sub { + my $saphostctrl_output = 'Instance name: PRD00, Hostname: qesdhdb01l029, Vendor: HDB, Type: hdb, Release: 42'; + + my $mock = Test::MockModule->new('sles4sap::sap_host_agent', no_auto => 1); + $mock->redefine(script_output => sub { return $saphostctrl_output; }); + $mock->redefine(script_run => sub { return 0; }); + + my @output = @{saphostctrl_list_databases()}; + is $output[0]->{instance_name}, 'PRD00', 'Check "instance_name" value'; + is $output[0]->{hostname}, 'qesdhdb01l029', 'Check "hostname" value'; + is $output[0]->{vendor}, 'HDB', 'Check "vendor" value'; + is $output[0]->{type}, 'hdb', 'Check "type" value'; + is $output[0]->{release}, '42', 'Check "release" value'; +}; + +subtest '[parse_instance_name] ' => sub { + my ($sid, $id) = @{parse_instance_name('POO08')}; + is $sid, 'POO', "Return correct SID: $sid"; + is $id, '08', "Return correct ID: $id"; +}; + +subtest '[parse_instance_name] Exceptions' => sub { + dies_ok { parse_instance_name('POO0') } 'Instance name with less than 5 characters'; + dies_ok { parse_instance_name('POO0ASDF') } 'Instance name with more than 5 characters'; + dies_ok { parse_instance_name('POO0 ') } 'Instance name contains spaces'; + dies_ok { parse_instance_name('Poo0a') } 'Instance name contains lowercase characters'; + dies_ok { parse_instance_name('POO0.') } 'Instance name contains any non-word characters'; +}; + +done_testing; diff --git a/t/32_database_hana.pm b/t/32_database_hana.pm new file mode 100644 index 000000000000..e93ce3240910 --- /dev/null +++ b/t/32_database_hana.pm @@ -0,0 +1,72 @@ +use strict; +use warnings; +use Test::More; +use Test::Exception; +use Test::Warnings; +use Test::MockModule; +use Test::Mock::Time; +use testapi; +use sles4sap::database_hana; + +subtest '[hdb_stop] HDB command compilation' => sub { + my $db_hana = Test::MockModule->new('sles4sap::database_hana', no_auto => 1); + my @calls; + $db_hana->redefine(assert_script_run => sub { @calls = $_[0]; return 0; }); + $db_hana->redefine(script_output => sub { return 'Dumbledore'; }); + $db_hana->redefine(sapcontrol_process_check => sub { return 0; }); + $db_hana->redefine(record_info => sub { note(join(' ', 'RECORD_INFO -->', @_)); }); + + hdb_stop(instance_id => '00', switch_user => 'Albus'); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /HDB/, @calls), 'Execute HDB command'); + ok((grep /stop/, @calls), 'Use "stop" function'); + ok((grep /sudo su \- Albus/, @calls), 'Run as another user'); + + hdb_stop(instance_id => '00', switch_user => 'Albus', command=>'kill'); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /kill \-x/, @calls), 'Use "kill" function'); +}; + +subtest '[hdb_stop] Sapcontrol arguments' => sub { + my $db_hana = Test::MockModule->new('sles4sap::database_hana', no_auto => 1); + my @sapcontrol_args; + $db_hana->redefine(assert_script_run => sub { return 0; }); + $db_hana->redefine(script_output => sub { return 'Dumbledore'; }); + $db_hana->redefine(sapcontrol_process_check => sub { @sapcontrol_args = @_; return 0; }); + $db_hana->redefine(record_info => sub { note(join(' ', 'RECORD_INFO -->', @_)); }); + + hdb_stop(instance_id => 'Albus'); + note("\n --> " . join("\n --> ", @sapcontrol_args)); + ok((grep /instance_id/, @sapcontrol_args), 'Madatory arg "instance_id"'); + ok((grep /expected_state/, @sapcontrol_args), 'Define expected state'); + ok((grep /wait_for_state/, @sapcontrol_args), 'Wait until processes are in correct state'); +}; + +subtest '[register_replica] Command compilation' => sub { + my $db_hana = Test::MockModule->new('sles4sap::database_hana', no_auto => 1); + my $topology = { + Hogwarts=>{ + srmode=>'FunnyGuy', + site=>'Dumbledore', + op_mode=>'VeryOP' + }, + Durmstrang=>{ + srmode=>'DeathEater', + site=>'Karkaroff', + op_mode=>'StatsUnknown' + } + }; + my @calls; + $db_hana->redefine(assert_script_run => sub { @calls = @_; return 0; }); + $db_hana->redefine(script_output => sub { return 'Revelio'; }); + $db_hana->redefine(calculate_hana_topology => sub { return $topology; }); + $db_hana->redefine(get_primary_node => sub { return 'Durmstrang'; }); + + register_replica(instance_id => '00', target_hostname => 'Hogwarts'); + note("\n --> " . join("\n --> ", @calls)); + ok((grep /hdbnsutil/, @calls), 'Main "hdbnsutil" command'); + ok((grep /-sr_register/, @calls), '"-sr_rergister" option'); + +}; + +done_testing; diff --git a/tests/sles4sap/redirection_tests/SLES4SAP_HA_data_structures.md b/tests/sles4sap/redirection_tests/SLES4SAP_HA_data_structures.md new file mode 100644 index 000000000000..71e36081d9ef --- /dev/null +++ b/tests/sles4sap/redirection_tests/SLES4SAP_HA_data_structures.md @@ -0,0 +1,14 @@ +# SLES4SAP/HA related data structure + +# Structure example + +``` +$run_args->{ha-sap_data} = { + + hana => { + + }, + +}; + +``` \ No newline at end of file diff --git a/tests/sles4sap/redirection_tests/hana_sr_primary_failover.pm b/tests/sles4sap/redirection_tests/hana_sr_primary_failover.pm new file mode 100644 index 000000000000..452f8e0fdb14 --- /dev/null +++ b/tests/sles4sap/redirection_tests/hana_sr_primary_failover.pm @@ -0,0 +1,130 @@ +# SUSE's openQA tests +# +# Copyright SUSE LLC +# SPDX-License-Identifier: FSFAP +# Maintainer: QE-SAP +# Summary: +# - Test collects data about current cluster setup +# - Failover is performed on primary database and +# - + +# HANA_PRIMARY_FAILOVER_SCENARIOS: stop,kill +# HANA_FAILOVER_RESTORE_ORIGIN: 'yes' +# HANA_FAILOVER_DEFAULT_PRIMARY: '' + + +if (get_var('SDAF_DEPLOYER_RESOURCE_GROUP')) { + use parent 'sles4sap::sap_deployment_automation_framework::basetest'; +} +else { + use parent 'consoletest'; +} + +use warnings; +use strict; +use testapi; +use serial_terminal qw(select_serial_terminal); +use sles4sap::console_redirection; +use saputils qw(calculate_hana_topology get_primary_node get_failover_node); +use hacluster qw(wait_for_idle_cluster + wait_until_resources_started + show_hana_resource_name + set_cluster_parameter + show_cluster_parameter); +use sles4sap::sap_host_agent qw(saphostctrl_list_databases parse_instance_name); +use sles4sap::database_hana; +use sles4sap::sapcontrol qw(sapcontrol_process_check sap_show_status_info); +use Carp qw(croak); +use Data::Dumper; + +sub run { + my ($self, $run_args) = @_; + # perform first scenario from the list + my @failover_scenarios = split(',', get_var('HANA_PRIMARY_FAILOVER_SCENARIOS', 'stop,kill')); + record_info('SC', join(' ', @failover_scenarios)); + record_info('params', join("\n", get_var('HANA_FAILOVER_RESTORE_ORIGIN'), get_var('HANA_FAILOVER_DEFAULT_PRIMARY'), get_var('HANA_PRIMARY_FAILOVER_SCENARIOS'))); + my $failover_type = $failover_scenarios[0]; + croak "Failover type $failover_type not supported" unless grep /$failover_type/, ('stop', 'kill'); + record_info('Test INFO', "Performing primary DB failover scenario: $failover_type"); + + # Connect to one of the DB nodes and collect topology data + my %databases = %{$run_args->{redirection_data}{db_hana}}; + my $target_node = (keys %databases)[0]; + my %target_node_data = %{$databases{$target_node}}; + + ## CONNECT SERIAL ## + # Connect to any DB node in order to get topology data + connect_target_to_serial( + destination_ip => $target_node_data{ip_address}, ssh_user => $target_node_data{ssh_user}, switch_root => 1); + my ($primary_db, $replica_db) = get_node_roles(); + + # Parameter + set_var('HANA_FAILOVER_DEFAULT_PRIMARY', $primary_db) unless get_var('HANA_FAILOVER_DEFAULT_PRIMARY'); + record_info('Primary DB', "Primary DB node found: $primary_db"); + + # Switch console redirection to primary node if not already + unless ($primary_db eq $target_node) { + record_info('Console switch', "Reconnecting console to primary DB node: $primary_db"); + disconnect_target_from_serial(); + connect_target_to_serial( + destination_ip => $databases{$primary_db}{ip_address}, + ssh_user => $databases{$primary_db}{ssh_user}, + switch_root => '1'); + } + + # Retrieve database information: DB SID and instance ID + my @db_data = @{(saphostctrl_list_databases())}; + record_info('DB data', Dumper(@db_data)); + die('Multiple databases on one host not supported') if @db_data > 1; + my ($db_sid, $db_id) = @{parse_instance_name($db_data[0]->{'instance_name'})}; + + # Perform failover on primary + sap_show_status_info(cluster=>1, netweaver=>1, instance_id=>$db_id); + hdb_stop(instance_id => $db_id, switch_user => lc($db_sid) . 'adm', command=>$failover_type); + + # Wait for takeover + record_info('Takeover', "Waiting for node '$replica_db' to become primary"); + wait_for_failed_resources(); + wait_for_takeover(target_node => $replica_db); + + # Register and start replication + my $automatic_register = show_cluster_parameter(resource=>show_hana_resource_name(), parameter=>'AUTOMATED_REGISTER');; + if ($automatic_register eq 'true') { + record_info('REG: Auto', "Parameter: AUTOMATED_REGISTER=true\nNo action to be done"); + } + else { + record_info('REG: Manual', "Parameter: AUTOMATED_REGISTER=false\nRegistration will be done manually"); + # Failed Primary node will be registered for replication after takeover + register_replica(target_hostname=>$primary_db, instance_id=>$db_id, switch_user => lc($db_sid) . 'adm'); + } + + # cleanup resources + assert_script_run('crm resource cleanup'); + + # Wait for database processes to start + record_info('DB wait', "Waiting for database node '$primary_db' to start"); + sapcontrol_process_check( + instance_id => $db_id, expected_state => 'started', wait_for_state => 'yes', timeout => 600); + record_info('DB started', "All database node '$primary_db' processes are 'GREEN'"); + + # Wait for cluster co come up + record_info('Cluster wait', 'Waiting for cluster to come up'); + wait_until_resources_started(); + wait_for_idle_cluster(); + record_info('Cluster OK', 'Cluster resources up and running'); + sap_show_status_info(cluster=>1, netweaver=>1, instance_id=>$db_id); + + # reload current topology data + ($primary_db, $replica_db) = get_node_roles(); + + unless (get_var('HANA_FAILOVER_RESTORE_ORIGIN') && $primary_db eq get_var('HANA_FAILOVER_DEFAULT_PRIMARY') or !get_var('HANA_FAILOVER_RESTORE_ORIGIN')) { + # remove first scenario + set_var('HANA_PRIMARY_FAILOVER_SCENARIOS', 'stop'); + record_info('SC adj', ''); + } + record_info('params', join("\n", get_var('HANA_FAILOVER_RESTORE_ORIGIN'), get_var('HANA_FAILOVER_DEFAULT_PRIMARY'), get_var('HANA_PRIMARY_FAILOVER_SCENARIOS'))); + + disconnect_target_from_serial(); +} + +1; diff --git a/tests/sles4sap/redirection_tests/hanasr_schedule_tests.pm b/tests/sles4sap/redirection_tests/hanasr_schedule_tests.pm new file mode 100644 index 000000000000..ad0e2aeb4b3f --- /dev/null +++ b/tests/sles4sap/redirection_tests/hanasr_schedule_tests.pm @@ -0,0 +1,83 @@ +# SUSE's openQA tests +# +# Copyright SUSE LLC +# SPDX-License-Identifier: FSFAP +# Maintainer: QE-SAP +# Summary: + +if (get_var('SDAF_DEPLOYER_RESOURCE_GROUP')) { + use parent 'sles4sap::sap_deployment_automation_framework::basetest'; +} +else { + use parent 'consoletest'; +} +use strict; +use warnings FATAL => 'all'; +use testapi; +use main_common 'loadtest'; +use sles4sap::console_redirection; +use saputils qw(calculate_hana_topology get_primary_node get_failover_node); + +sub test_flags { + return {fatal => 1, publiccloud_multi_module => 1}; +} + +sub run { + my ($self, $run_args) = @_; + my %databases = %{$run_args->{redirection_data}{db_hana}}; + # Connect to any database cluster node to get topology data + my $target_node = (keys %databases)[0]; + my %target_node_data = %{$databases{$target_node}}; + connect_target_to_serial( + destination_ip => $target_node_data{ip_address}, ssh_user => $target_node_data{ssh_user}, switch_root => '1'); + + my $topology = calculate_hana_topology(input => script_output('SAPHanaSR-showAttr --format=script')); + # No need for open SSH session anymore + disconnect_target_from_serial(); + + my $primary_db = get_primary_node(input => $topology); + my $primary_site = $topology->{$primary_db}{site}; + my $replica_db = get_failover_node(input => $topology); + my $replica_site = $topology->{$replica_db}{site}; + + # stop + # primary -> replica + # replica -> primary + # kill + # primary -> replica + # replica -> primary + + ### Schedule HDB stop tests + record_info('Load test', "Scheduling Primary DB failover using 'HDB stop' method.\n + Primary site $primary_site/$primary_db will be stopped.\nSecondary site $replica_site/$replica_db will take over."); + my $failover_action = 'stop'; + + loadtest( + 'sles4sap/redirection_tests/hana_sr_primary_failover', + name => "Stop_DB-$primary_site-$primary_db", run_args => $run_args, failover_action=>'stop', @_); + + # record_info('Load test', "Scheduling Primary DB failover using 'HDB stop' method.\n + # Primary site $replica_site/$replica_db will be stopped.\nSecondary site $primary_site/$primary_db will take over."); + # loadtest( + # 'sles4sap/redirection_tests/hana_sr_primary_failover', + # name => "Stop_DB-$replica_site-$replica_db", run_args => $run_args, @_); + # + # ### Schedule HDB stop tests + # record_info('Load test', "Scheduling Primary DB failover using 'HDB kill -x' method.\n + # Primary site $primary_site/$primary_db processes will be killed.\nSecondary site $replica_site/$replica_db will take over."); + # loadtest( + # 'sles4sap/redirection_tests/hana_sr_primary_failover', + # name => "Kill_DB_process-$primary_site-$primary_db", run_args => $run_args, @_); + # + # record_info('Load test', "Scheduling Primary DB failover using 'HDB kill -x' method.\n + # Primary site $replica_site/$replica_db processes will be killed.\nSecondary site $primary_site/$primary_db will take over."); + # loadtest( + # 'sles4sap/redirection_tests/hana_sr_primary_failover', + # name => "Kill_DB_process-$replica_site-$replica_db", run_args => $run_args, @_); + # + # ### Schedule clenup + # record_info('Load test', 'Scheduling SDAF cleanup'); + # loadtest('sles4sap/sap_deployment_automation_framework/cleanup', name => 'SDAF_cleanup', run_args => $run_args, @_); +} + +1; \ No newline at end of file diff --git a/tests/sles4sap/redirection_tests/redirection_check.pm b/tests/sles4sap/redirection_tests/redirection_check.pm index a56a026066cc..1a96aabfdc60 100644 --- a/tests/sles4sap/redirection_tests/redirection_check.pm +++ b/tests/sles4sap/redirection_tests/redirection_check.pm @@ -7,7 +7,12 @@ # It loops over all hosts defined in `$run_args->{redirection_data}` and attempts few common OpenQA api calls. # For more information read 'README.md' -use parent 'sles4sap::sap_deployment_automation_framework::basetest'; +if (get_var('SDAF_DEPLOYER_RESOURCE_GROUP')) { + use parent 'sles4sap::sap_deployment_automation_framework::basetest'; +} +else { + use parent 'consoletest'; +} use warnings; use strict; @@ -29,8 +34,8 @@ sub run { # Check if hostnames matches with what is expected # Check API calls: script_output, assert_script_run - my $hostname_real = script_output('hostname'); - assert_script_run("echo \$(hostname) > /tmp/hostname_$hostname_real"); + my $hostname_real = script_output('hostname', quiet=>1); + assert_script_run("echo \$(hostname) > /tmp/hostname_$hostname_real", quiet=>); die "Expected hostname '$hostname' does not match hostname returned '$hostname_real'" unless $hostname_real eq $hostname; record_info('API check', "script_output: PASS\nassert_script_run: PASS\nhostname match: PASS"); @@ -39,7 +44,7 @@ sub run { # Check API calls: save_tmp_file, upload_logs upload_logs("/tmp/hostname_$hostname_real"); save_tmp_file('hostname.txt', $hostname); - assert_script_run('curl -s ' . autoinst_url . "/files/hostname.txt| grep $hostname"); + assert_script_run('curl -s ' . autoinst_url . "/files/hostname.txt| grep $hostname", quiet=>); record_info('API check', "upload_logs: PASS\nsave_tmp_file: PASS\nOpenQA connection: PASS"); disconnect_target_from_serial(); diff --git a/tests/sles4sap/sap_deployment_automation_framework/deploy_hanasr.pm b/tests/sles4sap/sap_deployment_automation_framework/deploy_hanasr.pm index 1f01ffdff354..3770285cc464 100644 --- a/tests/sles4sap/sap_deployment_automation_framework/deploy_hanasr.pm +++ b/tests/sles4sap/sap_deployment_automation_framework/deploy_hanasr.pm @@ -32,6 +32,7 @@ use sles4sap::console_redirection qw(connect_target_to_serial disconnect_target_from_serial ); +use hacluster qw(set_cluster_parameter show_hana_resource_name); use serial_terminal qw(select_serial_terminal); use testapi; @@ -82,6 +83,12 @@ sub run { # Display deployment information ansible_hanasr_show_status(sdaf_config_root_dir => $sdaf_config_root_dir); + # Set AUTOMATED_REGISTER value according to parameter HANA_AUTOMATED_REGISTER + # By default this HA cluster is set after SDAF deployment to 'true'. + my $automated_register = get_var('HANA_AUTOMATED_REGISTER') ? 'true' : 'false'; + set_cluster_parameter( + resource=>show_hana_resource_name(), parameter=>'AUTOMATED_REGISTER', value=>$automated_register); + disconnect_target_from_serial(); serial_console_diag_banner('Module sdaf_deploy_hanasr.pm : stop'); }