Support shield v8

.circleci/config.yml

@@ -16,8 +16,8 @@ jobs:
     steps:
     - checkout
     - run: sudo apt-get update && sudo apt-get install openjdk-8-jdk && sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
-    - run: docker-compose -f broker/src/functional-test/resources/test-docker-compose.yml up -d
     - run: sudo keytool -importkeystore -srckeystore broker/src/functional-test/resources/credhub_client.jks -srcstorepass changeit -destkeystore $(readlink -f /usr/bin/java | sed "s:bin/java::")/lib/security/cacerts -deststorepass changeit
     - run: 'if [[ -z "${CIRCLE_PULL_REQUEST}" ]]; then openssl aes-256-cbc -md md5 -pass pass:$ENCRYPTION_PASSWORD -in secring.gpg.enc -out local.secring.gpg -d; fi'
+    - run: pushd docker && ./setup_docker.sh ; popd
     - run: ./gradlew clean build -x functionalTest
     - run: ./circleci-publish.sh

.gitignore

@@ -38,3 +38,8 @@ local.*
 
 **log4j2-test*
 /lib/groovy*.jar
+
+### Docker Setup ###
+docker/minio_config
+docker/s3data
+docker/vaultstorage

broker/src/functional-test/groovy/com/swisscom/cloud/sb/broker/functional/MariaDBBackupRestoreFunctionalSpec.groovy

@@ -33,7 +33,7 @@ import spock.lang.IgnoreIf
 import spock.lang.Shared
 
 @Slf4j
-@IgnoreIf({ !Boolean.valueOf(System.properties['com.swisscom.cloud.sb.broker.runMariaDBBackupRestoreFunctionalSpec']) })
+//@IgnoreIf({ !Boolean.valueOf(System.properties['com.swisscom.cloud.sb.broker.runMariaDBBackupRestoreFunctionalSpec']) })
 class MariaDBBackupRestoreFunctionalSpec extends BaseFunctionalSpec {
 
     @Shared
@@ -55,10 +55,10 @@ class MariaDBBackupRestoreFunctionalSpec extends BaseFunctionalSpec {
 
     def setup(){
         serviceLifeCycler.createServiceIfDoesNotExist("mariadb", ServiceProviderLookup.findInternalName(MariaDBServiceProvider), null, null, null, 5)
-        serviceLifeCycler.createParameter('BACKUP_SCHEDULE_NAME', 'daily', serviceLifeCycler.plan)
+        serviceLifeCycler.createParameter('BACKUP_SCHEDULE', 'daily 4am', serviceLifeCycler.plan)
         serviceLifeCycler.createParameter('BACKUP_POLICY_NAME', 'month', serviceLifeCycler.plan)
         serviceLifeCycler.createParameter('BACKUP_STORAGE_NAME', 'default', serviceLifeCycler.plan)
-        backupRestoreHelper = new BackupRestoreHelper(appBaseUrl, cfExtUser, cfExtPassword)
+        backupRestoreHelper = new BackupRestoreHelper(appBaseUrl, cfExtUser.username, cfExtUser.password)
     }
 
     @Autowired

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/BackupOnShield.groovy

@@ -59,6 +59,9 @@ trait BackupOnShield extends ExtensionProvider {
         def planParamtersForBackup = parameters.findAll {
             it.getName().startsWith(PLAN_PARAMETER_BACKUP_PREFIX)
         }
+        String schedule = planParamtersForBackup.find {
+            it.getName().equals("BACKUP_SCHEDULE")
+        }?.getValue()
         String scheduleName = planParamtersForBackup.find {
             it.getName().equals("BACKUP_SCHEDULE_NAME")
         }?.getValue()
@@ -68,6 +71,6 @@ trait BackupOnShield extends ExtensionProvider {
         String storageName = planParamtersForBackup.find {
             it.getName().equals("BACKUP_STORAGE_NAME")
         }?.getValue()
-        new BackupParameter(scheduleName: scheduleName, retentionName: policyName, storeName: storageName)
+        new BackupParameter(scheduleName: scheduleName, retentionName: policyName, storeName: storageName, schedule: schedule)
     }
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/BackupParameter.groovy

@@ -21,5 +21,6 @@ class BackupParameter implements BackupServiceConfig {
     String storeName
     String retentionName
     String scheduleName
+    String schedule
     String agent
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldClient.groovy

@@ -18,6 +18,8 @@ package com.swisscom.cloud.sb.broker.backup.shield
 import com.swisscom.cloud.sb.broker.backup.BackupPersistenceService
 import com.swisscom.cloud.sb.broker.async.job.JobStatus
 import com.swisscom.cloud.sb.broker.backup.shield.dto.*
+import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestClient
+import com.swisscom.cloud.sb.broker.backup.shield.restClient.ShieldRestClientFactory
 import com.swisscom.cloud.sb.broker.model.Backup
 import com.swisscom.cloud.sb.broker.model.ServiceDetail
 import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
@@ -29,7 +31,6 @@ import org.springframework.stereotype.Component
 
 @Component
 @Slf4j
-@CompileStatic
 class ShieldClient {
     protected ShieldConfig shieldConfig
     protected ShieldRestClientFactory shieldRestClientFactory
@@ -162,12 +163,15 @@ class ShieldClient {
         if (retention == null) {
             throw new RuntimeException("Retention ${shieldServiceConfig.retentionName} that is configured does not exist on shield")
         }
-        ScheduleDto schedule = buildClient().getScheduleByName(shieldServiceConfig.scheduleName)
+        // Either use BACKUP_SCHEDULE parameter or get the schedule UUID from shield v1 BACKUP_SCHEDULE_NAME parameter from service definition
+        ScheduleDto scheduleDto = buildClient().getScheduleByName(shieldServiceConfig.scheduleName)
+        String schedule = scheduleDto != null ? scheduleDto.uuid : shieldServiceConfig.schedule
+
         if (schedule == null) {
             throw new RuntimeException("Schedule ${shieldServiceConfig.scheduleName} that is configured does not exist on shield")
         }
 
-        createOrUpdateJob(jobName, targetUuid, store.uuid, retention.uuid, schedule.uuid, paused)
+        createOrUpdateJob(jobName, targetUuid, store.uuid, retention.uuid, schedule, paused)
     }
 
     private String createOrUpdateTarget(ShieldTarget target, String targetName, String agent) {
@@ -188,6 +192,6 @@ class ShieldClient {
     }
 
     private ShieldRestClient buildClient() {
-        shieldRestClientFactory.build(restTemplateBuilder.withSSLValidationDisabled().build(), shieldConfig.baseUrl, shieldConfig.apiKey)
+        shieldRestClientFactory.build()
     }
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldConfig.groovy

@@ -16,9 +16,11 @@
 package com.swisscom.cloud.sb.broker.backup.shield
 
 import com.swisscom.cloud.sb.broker.config.Config
+import groovy.transform.CompileStatic
 import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.context.annotation.Configuration
 
+@CompileStatic
 @Configuration
 @ConfigurationProperties(prefix = "com.swisscom.cloud.sb.broker.shield")
 class ShieldConfig implements Config {
@@ -27,4 +29,7 @@ class ShieldConfig implements Config {
     String jobPrefix
     String targetPrefix
     int maxRetryBackup
+    String defaultTenantName
+    String username
+    String password
 }

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClient.groovy

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2018 Swisscom (Switzerland) Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed
+ * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package com.swisscom.cloud.sb.broker.backup.shield.restClient
+
+import com.swisscom.cloud.sb.broker.backup.shield.ShieldTarget
+import com.swisscom.cloud.sb.broker.backup.shield.dto.*
+
+interface ShieldRestClient {
+    Object getStatus()
+
+    boolean matchVersion()
+
+    StoreDto getStoreByName(String name)
+
+    RetentionDto getRetentionByName(String name)
+
+    ScheduleDto getScheduleByName(String name)
+
+    TargetDto getTargetByName(String name)
+
+    String createTarget(String targetName, ShieldTarget target, String agent)
+
+    String updateTarget(TargetDto existingTarget, ShieldTarget target, String agent)
+
+    void deleteTarget(String uuid)
+
+    JobDto getJobByName(String name)
+
+    JobDto getJobByUuid(String uuid)
+
+    String createJob(String jobName,
+                     String targetUuid,
+                     String storeUuid,
+                     String retentionUuid,
+                     String scheduleUuid,
+                     boolean paused)
+
+    String updateJob(JobDto existingJob,
+                     String targetUuid,
+                     String storeUuid,
+                     String retentionUuid,
+                     String scheduleUuid,
+                     boolean paused)
+
+    String runJob(String uuid)
+
+    void deleteJob(String uuid)
+
+    TaskDto getTaskByUuid(String uuid)
+
+    void deleteTaskByUuid(String uuid)
+
+    ArchiveDto getArchiveByUuid(String uuid)
+
+    String restoreArchive(String uuid)
+
+    void deleteArchive(String uuid)
+}

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldRestClientFactory.groovy → broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClientFactory.groovy

@@ -13,17 +13,29 @@
  * specific language governing permissions and limitations under the License.
  */
 
-package com.swisscom.cloud.sb.broker.backup.shield
+package com.swisscom.cloud.sb.broker.backup.shield.restClient
 
 import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
-import org.springframework.web.client.RestTemplate
 
 @Component
 @CompileStatic
+@Slf4j
 class ShieldRestClientFactory {
-    ShieldRestClient build(RestTemplate restTemplate, String baseUrl, String apiKey) {
-        restTemplate.setErrorHandler(new ShieldRestResponseErrorHandler())
-        new ShieldRestClient(restTemplate, baseUrl, apiKey)
+
+    private List<ShieldRestClient> shieldRestClients
+
+    @Autowired
+    ShieldRestClientFactory(List<ShieldRestClient> shieldRestClients) {
+        this.shieldRestClients = shieldRestClients
+    }
+
+    ShieldRestClient build() {
+        for (ShieldRestClient shieldRestClient in shieldRestClients) {
+            if (shieldRestClient.matchVersion()) return shieldRestClient
+        }
+        throw new Exception("No matching shield implementation found")
     }
-}
+}

broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/ShieldRestClient.groovy → broker/src/main/groovy/com/swisscom/cloud/sb/broker/backup/shield/restClient/ShieldRestClientImpl.groovy

@@ -13,27 +13,29 @@
  * specific language governing permissions and limitations under the License.
  */
 
-package com.swisscom.cloud.sb.broker.backup.shield
+package com.swisscom.cloud.sb.broker.backup.shield.restClient
 
+
+import com.swisscom.cloud.sb.broker.backup.shield.ShieldConfig
+import com.swisscom.cloud.sb.broker.backup.shield.ShieldTarget
 import com.swisscom.cloud.sb.broker.backup.shield.dto.*
 import com.swisscom.cloud.sb.broker.util.GsonFactory
+import com.swisscom.cloud.sb.broker.util.RestTemplateBuilder
 import groovy.json.JsonSlurper
 import groovy.util.logging.Slf4j
-import org.springframework.http.*
+import org.springframework.http.HttpEntity
+import org.springframework.http.HttpMethod
 import org.springframework.web.client.RestTemplate
 
 @Slf4j
-class ShieldRestClient {
-    public static final String HEADER_API_KEY = 'X-Shield-Token'
-
-    private RestTemplate restTemplate
-    private String baseUrl
-    private String apiKey
+abstract class ShieldRestClientImpl implements ShieldRestClient {
+    protected ShieldConfig config
+    protected RestTemplate restTemplate
 
-    ShieldRestClient(RestTemplate restTemplate, String baseUrl, String apiKey) {
-        this.restTemplate = restTemplate
-        this.baseUrl = baseUrl
-        this.apiKey = apiKey
+    ShieldRestClientImpl(ShieldConfig shieldConfig, RestTemplateBuilder restTemplateBuilder) {
+        this.restTemplate = restTemplateBuilder.withSSLValidationDisabled().build()
+        this.restTemplate.setErrorHandler(new ShieldRestResponseErrorHandler())
+        this.config = shieldConfig
     }
 
     Object getStatus() {
@@ -142,35 +144,26 @@ class ShieldRestClient {
                      String retentionUuid,
                      String scheduleUuid,
                      boolean paused = true) {
-        def body = [name     : jobName,
-                    target   : targetUuid,
-                    store    : storeUuid,
-                    retention: retentionUuid,
-                    schedule : scheduleUuid,
-                    paused   : paused]
-
+        def body = getCreateJobBody(jobName, targetUuid, storeUuid, retentionUuid, scheduleUuid, paused)
         def response = restTemplate.exchange(jobsUrl(), HttpMethod.POST, configureRequestEntity(body), String.class)
         new JsonSlurper().parseText(response.body).uuid
     }
 
+    abstract Map<String, ?> getCreateJobBody(String jobName, String targetUuid, String storeUuid, String retentionUuid, String scheduleUuid, boolean paused)
+
     String updateJob(JobDto existingJob,
                      String targetUuid,
                      String storeUuid,
                      String retentionUuid,
                      String scheduleUuid,
                      boolean paused = true) {
-        def body = [name     : existingJob.name,
-                    summary  : existingJob.summary,
-                    target   : targetUuid,
-                    store    : storeUuid,
-                    retention: retentionUuid,
-                    schedule : scheduleUuid,
-                    paused   : paused]
-
+        def body = getUpdateJobBody(existingJob, targetUuid, storeUuid, retentionUuid, scheduleUuid, paused)
         restTemplate.exchange(jobUrl(existingJob.uuid), HttpMethod.PUT, configureRequestEntity(body), (Class) null)
         existingJob.uuid
     }
 
+    abstract Map<String, ?> getUpdateJobBody(JobDto existingJob, String targetUuid, String storeUuid, String retentionUuid, String scheduleUuid, boolean paused = true)
+
     String runJob(String uuid) {
         def response = restTemplate.exchange(jobUrl(uuid) + "/run", HttpMethod.POST, configureRequestEntity(), String.class)
         new JsonSlurper().parseText(response.body).task_uuid
@@ -208,20 +201,14 @@ class ShieldRestClient {
         restTemplate.exchange(archiveUrl(uuid), HttpMethod.DELETE, configureRequestEntity(), String.class)
     }
 
-    private <T> List<T> getResources(String endpoint, final Class<T[]> clazz) {
+    protected <T> List<T> getResources(String endpoint, final Class<T[]> clazz) {
         def response = restTemplate.exchange(endpoint, HttpMethod.GET, configureRequestEntity(), String.class)
         final T[] jsonToObject = GsonFactory.withISO8601Datetime().fromJson(response.body.toString(), clazz)
         return Arrays.asList(jsonToObject)
     }
 
 
-    private <T> HttpEntity<T> configureRequestEntity(T t) {
-        HttpHeaders headers = new HttpHeaders()
-        headers.setContentType(MediaType.valueOf(MediaType.APPLICATION_JSON_VALUE))
-        headers.add(HEADER_API_KEY, apiKey)
-        HttpEntity<T> entity = t ? new HttpEntity<T>(t, headers) : new HttpEntity<T>(headers)
-        return entity
-    }
+    abstract protected <T> HttpEntity<T> configureRequestEntity(T t)
 
     protected String storesUrl() {
         "${baseUrl()}/stores"
@@ -263,7 +250,18 @@ class ShieldRestClient {
         "${baseUrl()}/status"
     }
 
-    private String baseUrl() {
-        "${baseUrl}/v1"
+    protected String infoUrl() {
+        "${baseUrl()}/info"
     }
+
+    protected String tenantsUrl() {
+        "${baseUrl()}/tenants"
+    }
+
+    protected String loginUrl() {
+        "${baseUrl()}/auth/login"
+    }
+
+    abstract protected String baseUrl()
+
 }