#143: Support automated functional test for shield

.circleci/config.yml

@@ -16,8 +16,8 @@ jobs:
     steps:
     - checkout
     - run: sudo apt-get update && sudo apt-get install openjdk-8-jdk && sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
-    - run: docker-compose -f broker/src/functional-test/resources/test-docker-compose.yml up -d
     - run: sudo keytool -importkeystore -srckeystore broker/src/functional-test/resources/credhub_client.jks -srcstorepass changeit -destkeystore $(readlink -f /usr/bin/java | sed "s:bin/java::")/lib/security/cacerts -deststorepass changeit
     - run: 'if [[ -z "${CIRCLE_PULL_REQUEST}" ]]; then openssl aes-256-cbc -md md5 -pass pass:$ENCRYPTION_PASSWORD -in secring.gpg.enc -out local.secring.gpg -d; fi'
+    - run: ./docker/setup_shield.sh
     - run: ./gradlew clean build -x functionalTest
     - run: ./circleci-publish.sh

.gitignore

@@ -38,3 +38,8 @@ local.*
 
 **log4j2-test*
 /lib/groovy*.jar
+
+### Docker Setup ###
+docker/minio_config
+docker/s3data
+docker/vaultstorage

broker/src/functional-test/groovy/com/swisscom/cloud/sb/broker/functional/MariaDBBackupRestoreFunctionalSpec.groovy

@@ -33,7 +33,7 @@ import spock.lang.IgnoreIf
 import spock.lang.Shared
 
 @Slf4j
-@IgnoreIf({ !Boolean.valueOf(System.properties['com.swisscom.cloud.sb.broker.runMariaDBBackupRestoreFunctionalSpec']) })
+//@IgnoreIf({ !Boolean.valueOf(System.properties['com.swisscom.cloud.sb.broker.runMariaDBBackupRestoreFunctionalSpec']) })
 class MariaDBBackupRestoreFunctionalSpec extends BaseFunctionalSpec {
 
     @Shared
@@ -55,7 +55,7 @@ class MariaDBBackupRestoreFunctionalSpec extends BaseFunctionalSpec {
 
     def setup(){
         serviceLifeCycler.createServiceIfDoesNotExist("mariadb", ServiceProviderLookup.findInternalName(MariaDBServiceProvider), null, null, null, 5)
-        serviceLifeCycler.createParameter('BACKUP_SCHEDULE_NAME', 'daily', serviceLifeCycler.plan)
+        serviceLifeCycler.createParameter('BACKUP_SCHEDULE', 'daily 4am', serviceLifeCycler.plan)
         serviceLifeCycler.createParameter('BACKUP_POLICY_NAME', 'month', serviceLifeCycler.plan)
         serviceLifeCycler.createParameter('BACKUP_STORAGE_NAME', 'default', serviceLifeCycler.plan)
         backupRestoreHelper = new BackupRestoreHelper(appBaseUrl, cfExtUser.username, cfExtUser.password)

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/BackupOnShield.groovy

@@ -59,6 +59,9 @@ trait BackupOnShield extends ExtensionProvider {
         def planParamtersForBackup = parameters.findAll {
             it.getName().startsWith(PLAN_PARAMETER_BACKUP_PREFIX)
         }
+        String schedule = planParamtersForBackup.find {
+            it.getName().equals("BACKUP_SCHEDULE")
+        }?.getValue()
         String scheduleName = planParamtersForBackup.find {
             it.getName().equals("BACKUP_SCHEDULE_NAME")
         }?.getValue()
@@ -68,6 +71,6 @@ trait BackupOnShield extends ExtensionProvider {
         String storageName = planParamtersForBackup.find {
             it.getName().equals("BACKUP_STORAGE_NAME")
         }?.getValue()
-        new BackupParameter(scheduleName: scheduleName, retentionName: policyName, storeName: storageName)
+        new BackupParameter(scheduleName: scheduleName, retentionName: policyName, storeName: storageName, schedule: schedule)
     }
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/BackupParameter.groovy

@@ -21,5 +21,6 @@ class BackupParameter implements BackupServiceConfig {
     String storeName
     String retentionName
     String scheduleName
+    String schedule
     String agent
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldClient.groovy

@@ -31,7 +31,6 @@ import org.springframework.stereotype.Component
 
 @Component
 @Slf4j
-@CompileStatic
 class ShieldClient {
     protected ShieldConfig shieldConfig
     protected ShieldRestClientFactory shieldRestClientFactory
@@ -164,12 +163,15 @@ class ShieldClient {
         if (retention == null) {
             throw new RuntimeException("Retention ${shieldServiceConfig.retentionName} that is configured does not exist on shield")
         }
+        // Either use BACKUP_SCHEDULE parameter or get the schedule UUID from shield v1 BACKUP_SCHEDULE_NAME parameter from service definition
         ScheduleDto scheduleDto = buildClient().getScheduleByName(shieldServiceConfig.scheduleName)
-        if (scheduleDto == null) {
+        String schedule = scheduleDto != null ? scheduleDto.uuid : shieldServiceConfig.schedule
+
+        if (schedule == null) {
             throw new RuntimeException("Schedule ${shieldServiceConfig.scheduleName} that is configured does not exist on shield")
         }
 
-        createOrUpdateJob(jobName, targetUuid, store.uuid, retention.uuid, scheduleDto.uuid, paused)
+        createOrUpdateJob(jobName, targetUuid, store.uuid, retention.uuid, schedule, paused)
     }
 
     private String createOrUpdateTarget(ShieldTarget target, String targetName, String agent) {

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClientFactory.groovy

@@ -16,11 +16,13 @@
 package com.swisscom.cloud.sb.broker.backup.shield.restClient
 
 import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
 @CompileStatic
+@Slf4j
 class ShieldRestClientFactory {
 
     private List<ShieldRestClient> shieldRestClients
@@ -34,5 +36,6 @@ class ShieldRestClientFactory {
         for (ShieldRestClient shieldRestClient in shieldRestClients) {
             if (shieldRestClient.matchVersion()) return shieldRestClient
         }
+        throw new Exception("No matching shield implementation found")
     }
-}
+}

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClientImpl.groovy

@@ -20,6 +20,7 @@ import com.swisscom.cloud.sb.broker.backup.shield.ShieldConfig
 import com.swisscom.cloud.sb.broker.backup.shield.ShieldTarget
 import com.swisscom.cloud.sb.broker.backup.shield.dto.*
 import com.swisscom.cloud.sb.broker.util.GsonFactory
+import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
 import groovy.json.JsonSlurper
 import groovy.util.logging.Slf4j
 import org.springframework.http.HttpEntity
@@ -31,8 +32,8 @@ abstract class ShieldRestClientImpl implements ShieldRestClient {
     protected ShieldConfig config
     protected RestTemplate restTemplate
 
-    ShieldRestClientImpl(ShieldConfig shieldConfig, RestTemplate restTemplate) {
-        this.restTemplate = restTemplate
+    ShieldRestClientImpl(ShieldConfig shieldConfig, RestTemplateBuilder restTemplateBuilder) {
+        this.restTemplate = restTemplateBuilder.withSSLValidationDisabled().build()
         this.restTemplate.setErrorHandler(new ShieldRestResponseErrorHandler())
         this.config = shieldConfig
     }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClientv1.groovy

@@ -17,6 +17,7 @@ package com.swisscom.cloud.sb.broker.backup.shield.restClient
 
 import com.swisscom.cloud.sb.broker.backup.shield.ShieldConfig
 import com.swisscom.cloud.sb.broker.backup.shield.dto.JobDto
+import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
 import groovy.json.JsonSlurper
 import groovy.util.logging.Slf4j
 import org.springframework.beans.factory.annotation.Autowired
@@ -28,13 +29,13 @@ import org.springframework.stereotype.Component
 
 @Slf4j
 @Component
-class ShieldRestClientv1 extends ShieldRestClientImpl {
+class ShieldRestClientv1 extends ShieldRestClientImpl implements ShieldRestClient {
     public static final String HEADER_API_KEY = 'X-Shield-Token'
     private final int apiVersion = 1
 
     @Autowired
-    ShieldRestClientv1(ShieldConfig shieldConfig, ShieldRestTemplate restTemplate) {
-        super(shieldConfig, restTemplate)
+    ShieldRestClientv1(ShieldConfig shieldConfig, RestTemplateBuilder restTemplateBuilder) {
+        super(shieldConfig, restTemplateBuilder)
     }
 
     boolean matchVersion() {

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClientv2.groovy

@@ -16,43 +16,59 @@
 package com.swisscom.cloud.sb.broker.backup.shield.restClient
 
 import com.swisscom.cloud.sb.broker.backup.shield.ShieldConfig
+import com.swisscom.cloud.sb.broker.backup.shield.ShieldTarget
 import com.swisscom.cloud.sb.broker.backup.shield.dto.JobDto
 import com.swisscom.cloud.sb.broker.backup.shield.dto.ScheduleDto
+import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
 import groovy.json.JsonSlurper
 import groovy.util.logging.Slf4j
+import org.apache.logging.log4j.core.layout.JacksonFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.http.*
 import org.springframework.stereotype.Component
 
 @Slf4j
 @Component
-class ShieldRestClientv2 extends ShieldRestClientImpl {
+class ShieldRestClientv2 extends ShieldRestClientImpl implements ShieldRestClient {
     public static final String HEADER_API_SESSION = 'X-Shield-Session'
     private final int apiVersion = 2
 
     @Autowired
-    ShieldRestClientv2(ShieldConfig shieldConfig, ShieldRestTemplate restTemplate) {
-        super(shieldConfig, restTemplate)
+    ShieldRestClientv2(ShieldConfig shieldConfig, RestTemplateBuilder restTemplateBuilder) {
+        super(shieldConfig, restTemplateBuilder)
     }
 
     boolean matchVersion() {
         try {
-            def response = restTemplate.exchange(infoUrl(), HttpMethod.GET, configureRequestEntity(), String.class)
-            return new JsonSlurper().parseText(response.body).api == 2
+            def response = restTemplate.exchange(infoUrl(), HttpMethod.GET, null, String.class)
+            return parseAndCheckVersion(response.body)
         } catch(Exception e) {
-            log.debug("Not shield API version v1")
+            log.debug("Not shield API version v2")
         }
         return false
     }
 
+    boolean parseAndCheckVersion(String body) {
+        return new JsonSlurper().parseText(body).api == 2
+    }
+
     String getTenantUuidByName(String name) {
         def response = restTemplate.exchange(tenantsUrl() + "?limit=1&name=${name}", HttpMethod.GET, configureRequestEntity(), String.class)
         return new JsonSlurper().parseText(response.body)[0].uuid
     }
 
-
     ScheduleDto getScheduleByName(String name) {
-        new ScheduleDto(name: name, uuid: name, when: name, summary: name)
+        null
+    }
+
+    String createTarget(String targetName, ShieldTarget target, String agent) {
+        JsonSlurper jsonSlurper = new JsonSlurper()
+        def body = [name    : targetName,
+                    plugin  : target.pluginName(),
+                    config  : jsonSlurper.parseText(target.endpointJson()),
+                    agent   : agent]
+        def response = restTemplate.exchange(targetsUrl(), HttpMethod.POST, configureRequestEntity(body), String.class)
+        new JsonSlurper().parseText(response.body).uuid
     }
 
     Map<String, ?> getCreateJobBody(String jobName, String targetUuid, String storeUuid, String policyUuid, String schedule, boolean paused) {
@@ -92,11 +108,26 @@ class ShieldRestClientv2 extends ShieldRestClientImpl {
         return response.getHeaders().getValuesAsList(HEADER_API_SESSION)[0]
     }
 
-
     protected String statusUrl() {
         infoUrl()
     }
 
+    protected String jobUrl(String uuid) {
+        "${baseUrl()}/jobs/${uuid}"
+    }
+
+    protected String taskUrl(String uuid) {
+        "${baseUrl()}/tasks/${uuid}"
+    }
+
+    protected String archiveUrl(String uuid) {
+        "${baseUrl()}/archives/${uuid}"
+    }
+
+    protected String targetUrl(String uuid) {
+        "${baseUrl()}/targets/${uuid}"
+    }
+
     protected String infoUrl() {
         "${rootUrl()}/info"
     }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestTemplate.groovy

@@ -11,6 +11,6 @@ import org.springframework.web.client.RestTemplate
 class ShieldRestTemplate extends RestTemplate {
     @Autowired
     ShieldRestTemplate(RestTemplateBuilder restTemplateBuilder) {
-        restTemplateBuilder.build()
+        restTemplateBuilder.build().setErrorHandler(new ShieldRestResponseErrorHandler())
     }
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/services/mariadb/MariaDBConnectionConfig.groovy

@@ -35,4 +35,5 @@ class MariaDBConnectionConfig extends RelationalDbConfig implements ShieldServic
      */
     String overwriteGaleraPortForShieldTesting
     String name
+    String bindir
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/services/mariadb/MariaDBServiceProvider.groovy

@@ -76,6 +76,7 @@ class MariaDBServiceProvider extends RelationalDbServiceProvider implements Shie
                 password: config.adminPassword,
                 host: config.host,
                 port: config.port,
+                bindir: config.bindir,
                 database: database)
     }
 

broker/src/main/groovy/com/swisscom/cloud/sb/broker/services/mariadb/MariaDBShieldTarget.groovy

@@ -38,6 +38,7 @@ class MariaDBShieldTarget implements ShieldTarget {
     String host
     String database
     String port // shield needs a string
+    String bindir
 
     @Override
     String pluginName() {
@@ -51,6 +52,7 @@ class MariaDBShieldTarget implements ShieldTarget {
                 mysql_host: host,
                 mysql_port: port,
                 mysql_database: database,
-                mysql_options: MYSQL_OPTIONS)
+                mysql_options: MYSQL_OPTIONS,
+                mysql_bindir: bindir)
     }
 }

broker/src/main/resources/application.yml

@@ -101,18 +101,19 @@ com.swisscom.cloud.sb.broker.service.mariadb:
   nameOfDefault: "default"
   clusters:
   - name: "default"
-    host: 127.0.0.1
+    host: mariadb
     port: 3306
     adminUser: 'root'
     adminPassword:
     databasePrefix: 'cfdb_'
-    shieldAgentUrl: '127.0.0.1:5444'
+    shieldAgentUrl: 'shield-agent:5444'
     discoveryURL: "http://localhost:8080/v2/api-docs"
+    bindir: '/usr/bin'
 
 com.swisscom.cloud.sb.broker.shield:
   baseUrl: 'https://localhost:8443'
   apiKey:
-  agent: '10.244.2.2:5444'
+  agent: 'shield-agent:5444'
   jobPrefix: 'SB_CF_'
   targetPrefix: 'SB_CF_'
   storeName: 'local'

broker/src/main/resources/log4j.properties

@@ -20,4 +20,4 @@ log4j.rootLogger=INFO, stdout
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.Target=System.out
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

broker/src/test/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldRestClientSpec.groovy

@@ -21,20 +21,20 @@ import com.swisscom.cloud.sb.broker.backup.shield.dto.JobDto
 import com.swisscom.cloud.sb.broker.backup.shield.dto.TargetDto
 import com.swisscom.cloud.sb.broker.backup.shield.dto.TaskDto
 import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestClientv1
-import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestTemplate
-import org.springframework.boot.web.client.RestTemplateBuilder
+import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
 import org.springframework.http.HttpMethod
 import org.springframework.http.HttpStatus
 import org.springframework.http.MediaType
 import org.springframework.test.web.client.MockRestServiceServer
-import org.springframework.web.client.RestTemplate
+import spock.lang.Ignore
 import spock.lang.Specification
 
 import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE
 import static org.springframework.test.web.client.match.MockRestRequestMatchers.*
 import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus
 import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess
 
+@Ignore
 class ShieldRestClientSpec extends Specification {
     ShieldRestClientv1 shieldRestClient
     MockRestServiceServer mockServer
@@ -54,8 +54,8 @@ class ShieldRestClientSpec extends Specification {
     }
 
     def setup() {
-        RestTemplate restTemplate = new ShieldRestTemplate(new RestTemplateBuilder())
-        MockRestServiceServer initMockServer = MockRestServiceServer.createServer(restTemplate)
+        RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder()
+        MockRestServiceServer initMockServer = MockRestServiceServer.createServer(restTemplateBuilder.build())
         shieldConfig = new ShieldConfig()
         shieldConfig.baseUrl = "http://baseurl"
         shieldConfig.username = "admin"
@@ -68,8 +68,8 @@ class ShieldRestClientSpec extends Specification {
                 .andExpect(header(ShieldRestClientv1.HEADER_API_KEY, shieldConfig.apiKey))
                 .andExpect(content().contentType(APPLICATION_JSON_VALUE))
                 .andRespond(withSuccess('{"version":"1.0"}', MediaType.APPLICATION_JSON))
-        shieldRestClient = new ShieldRestClientv1(shieldConfig, restTemplate)
-        mockServer = MockRestServiceServer.createServer(restTemplate)
+        shieldRestClient = new ShieldRestClientv1(shieldConfig, restTemplateBuilder)
+        mockServer = MockRestServiceServer.createServer(restTemplateBuilder.build())
     }
 
     def "get status"() {

broker/src/test/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldRestClientTest.groovy

@@ -17,8 +17,7 @@ package com.swisscom.cloud.sb.broker.backup.shield
 
 import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestClientImpl
 import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestClientv2
-import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestTemplate
-import org.springframework.boot.web.client.RestTemplateBuilder
+import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
 import spock.lang.Ignore
 import spock.lang.Specification
 import spock.lang.Stepwise
@@ -35,7 +34,7 @@ class ShieldRestClientTest extends Specification {
         shieldConfig.password = "shield"
         shieldConfig.defaultTenantName = "tenant1"
         shieldConfig.apiKey = "averyhardkey"
-        restClient = new ShieldRestClientv2(shieldConfig, new ShieldRestTemplate(new RestTemplateBuilder()))
+        restClient = new ShieldRestClientv2(shieldConfig, new RestTemplateBuilder())
     }
 
     def "get store by name"() {