parser for batch fail status json

gooddata · Feb 11, 2016 · f831092 · f831092
1 parent 1547e16
commit f831092
Show file tree

Hide file tree

Showing 8 changed files with 277 additions and 30 deletions.
diff --git a/src/main/java/com/gooddata/dataset/BatchFailStatus.java b/src/main/java/com/gooddata/dataset/BatchFailStatus.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved.
+ */
+package com.gooddata.dataset;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.gooddata.util.GDDateTimeDeserializer;
+import com.gooddata.util.GDDateTimeSerializer;
+
+import org.joda.time.DateTime;
+
+import java.util.List;
+
+/**
+ * Batch fail status of dataset load.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BatchFailStatus {
+
+    private final List<FailStatus> failStatuses;
+    private final List<String> messages;
+    private final String status;
+    private final DateTime date;
+
+    @JsonCreator
+    private BatchFailStatus(@JsonProperty("uploads") List<FailStatus> failStatuses, @JsonProperty("messages") List<String> messages,
+            @JsonProperty("status") String status, @JsonProperty("date") @JsonDeserialize(using = GDDateTimeDeserializer.class) DateTime date) {
+        this.failStatuses = failStatuses;
+        this.messages = messages;
+        this.status = status;
+        this.date = date;
+    }
+
+    public List<FailStatus> getFailStatuses() {
+        return failStatuses;
+    }
+
+    public List<String> getMessages() {
+        return messages;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+
+    @JsonSerialize(using = GDDateTimeSerializer.class)
+    public DateTime getDate() {
+        return date;
+    }
+}
diff --git a/src/main/java/com/gooddata/dataset/DatasetService.java b/src/main/java/com/gooddata/dataset/DatasetService.java
@@ -35,6 +35,7 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.springframework.util.StringUtils.isEmpty;
 
 /**
@@ -101,7 +102,7 @@ public FutureResult<Void> loadDataset(final Project project, final DatasetManife
             final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8));
             dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream);
 
-            return pullLoad(project, dirPath, manifest.getDataSet());
+            return pullLoad(project, dirPath, manifest.getDataSet(), false);
         } catch (IOException e) {
             throw new DatasetException("Unable to serialize manifest", manifest.getDataSet(), e);
         } catch (DataStoreException | GoodDataRestException | RestClientException e) {
@@ -156,7 +157,7 @@ public FutureResult<Void> loadDatasets(final Project project, final Collection<D
             final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8));
             dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream);
 
-            return pullLoad(project, dirPath, datasetsNames);
+            return pullLoad(project, dirPath, datasetsNames, true);
         } catch (IOException e) {
             throw new DatasetException("Unable to serialize manifest", datasetsNames, e);
         } catch (DataStoreException | GoodDataRestException | RestClientException e) {
@@ -179,11 +180,11 @@ private void validateUploadManifests(final Collection<DatasetManifest> datasets)
         }
     }
 
-    private FutureResult<Void> pullLoad(Project project, final Path dirPath, final String dataset) {
-        return pullLoad(project, dirPath, asList(dataset));
+    private FutureResult<Void> pullLoad(Project project, final Path dirPath, final String dataset, boolean isBatchUpload) {
+        return pullLoad(project, dirPath, singletonList(dataset), isBatchUpload);
     }
 
-    private FutureResult<Void> pullLoad(Project project, final Path dirPath, final Collection<String> datasets) {
+    private FutureResult<Void> pullLoad(Project project, final Path dirPath, final Collection<String> datasets, final boolean isBatchUpload) {
         final PullTask pullTask = restTemplate
                 .postForObject(Pull.URI, new Pull(dirPath.toString()), PullTask.class, project.getId());
         return new PollResult<>(this, new SimplePollHandler<Void>(pullTask.getUri(), Void.class) {
@@ -192,7 +193,7 @@ public boolean isFinished(ClientHttpResponse response) throws IOException {
                 final PullTaskStatus status = extractData(response, PullTaskStatus.class);
                 final boolean finished = status.isFinished();
                 if (finished && !status.isSuccess()) {
-                    final String message = getErrorMessage(status, dirPath);
+                    final String message = getErrorMessage(status, dirPath, isBatchUpload);
                     throw new DatasetException(message, datasets);
                 }
                 return finished;
@@ -215,37 +216,58 @@ protected void onFinish() {
 
     }
 
-    private String getErrorMessage(final PullTaskStatus status, final Path dirPath) {
+    private String getErrorMessage(final PullTaskStatus status, final Path dirPath, boolean isBatchUpload) {
         String message = "status: " + status.getStatus();
         try {
-            final FailStatus failStatus = download(dirPath.resolve(STATUS_FILE_NAME), FailStatus.class);
-            if (failStatus != null) {
-                final List<FailPart> errorParts = failStatus.getErrorParts();
-                if (!errorParts.isEmpty()) {
-                    final List<String> errors = new ArrayList<>();
-                    for (FailPart part: errorParts) {
-                        if (part.getLogName() != null) {
-                            try {
-                                final String[] msg = download(dirPath.resolve(part.getLogName()), String[].class);
-                                errors.addAll(asList(msg));
-                            } catch (IOException | DataStoreException e) {
-                                if (part.getError() != null) {
-                                    errors.add(part.getError().getFormattedMessage());
-                                }
-                            }
-                        }
-                    }
-                    message = errors.toString();
-                } else if (failStatus.getError() != null) {
-                    message = failStatus.getError().getFormattedMessage();
-                }
+            Path statusFile = dirPath.resolve(STATUS_FILE_NAME);
+            if (isBatchUpload) {
+                final BatchFailStatus batchFailStatus = download(statusFile, BatchFailStatus.class);
+                message = getBatchFailStatusErrorMsg(dirPath, message, batchFailStatus);
+            } else {
+                final FailStatus failStatus = download(statusFile, FailStatus.class);
+                message = getFailStatusErrorMsg(dirPath, message, failStatus);
             }
         } catch (IOException | DataStoreException ignored) {
             // todo log?
         }
         return message;
     }
 
+    private String getBatchFailStatusErrorMsg(final Path dirPath, final String defaultMessage, final BatchFailStatus batchFailStatus) {
+        if (batchFailStatus == null || batchFailStatus.getFailStatuses() == null) {
+            return defaultMessage;
+        }
+        final List<String> messages = new ArrayList<>();
+        for (FailStatus failStatus : batchFailStatus.getFailStatuses()) {
+            messages.add(getFailStatusErrorMsg(dirPath, defaultMessage, failStatus));
+        }
+        return messages.isEmpty() ? defaultMessage : messages.toString();
+    }
+
+    private String getFailStatusErrorMsg(final Path dirPath, final String defaultMessage, final FailStatus failStatus) {
+        if (failStatus == null) {
+            return defaultMessage;
+        }
+        final List<FailPart> errorParts = failStatus.getErrorParts();
+        if (errorParts.isEmpty()){
+            return (failStatus.getError() != null) ? failStatus.getError().getFormattedMessage() : defaultMessage;
+        }
+        final List<String> errors = new ArrayList<>();
+        for (FailPart part : errorParts) {
+            if (part.getLogName() != null) {
+                try {
+                    final String[] msg = download(dirPath.resolve(part.getLogName()), String[].class);
+                    errors.addAll(asList(msg));
+                } catch (IOException | DataStoreException e) {
+                    if (part.getError() != null) {
+                        errors.add(part.getError().getFormattedMessage());
+                    }
+                }
+            }
+        }
+        return errors.toString();
+    }
+
     private <T> T download(final Path path, final Class<T> type) throws IOException {
         try (final InputStream input = dataStoreService.download(path.toString())) {
             return mapper.readValue(input, type);
@@ -324,7 +346,6 @@ public void handlePollException(final GoodDataRestException e) {
      * @param project project to be updated
      * @param maqlDml update script to be executed in the project
      * @return poll result
-     *
      * @see com.gooddata.model.ModelService#updateProjectModel
      */
     public FutureResult<Void> updateProjectData(final Project project, final String maqlDml) {
@@ -362,5 +383,4 @@ public void handlePollException(final GoodDataRestException e) {
             }
         });
     }
-
 }
diff --git a/src/test/java/com/gooddata/dataset/BatchFailStatusTest.java b/src/test/java/com/gooddata/dataset/BatchFailStatusTest.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved.
+ */
+package com.gooddata.dataset;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.hamcrest.Matchers;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class BatchFailStatusTest {
+    @Test
+    public void testParser() throws Exception {
+        final InputStream stream = getClass().getResourceAsStream("/dataset/batchFailStatus1.json");
+        final BatchFailStatus value = new ObjectMapper().readValue(stream, BatchFailStatus.class);
+        assertThat(value, is(notNullValue()));
+        assertThat(value.getStatus(),is("ERROR"));
+        assertThat(value.getMessages(),is(Matchers.<String>empty()));
+        assertThat(value.getDate(),is(new DateTime(2016,2,1,10,12,9, DateTimeZone.UTC)));
+        assertThat(value.getFailStatuses(),hasSize(2));
+    }
+
+    @Test
+    public void testParser2() throws IOException {
+        final InputStream stream = getClass().getResourceAsStream("/dataset/batchFailStatus2.json");
+        final BatchFailStatus value = new ObjectMapper().readValue(stream, BatchFailStatus.class);
+        assertThat(value,is(notNullValue()));
+        assertThat(value.getStatus(),is("ERROR"));
+        assertThat(value.getMessages(),containsInAnyOrder("test1","test2"));
+        assertThat(value.getDate(),is(new DateTime(2016,1,1,10,11,15, DateTimeZone.UTC)));
+        assertThat(value.getFailStatuses(),is(Matchers.<FailStatus>empty()));
+    }
+}
diff --git a/src/test/java/com/gooddata/dataset/DatasetServiceAT.java b/src/test/java/com/gooddata/dataset/DatasetServiceAT.java
@@ -1,6 +1,12 @@
 package com.gooddata.dataset;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.testng.Assert.fail;
+
 import com.gooddata.AbstractGoodDataAT;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 import org.testng.annotations.Test;
 
 /**
@@ -34,5 +40,32 @@ public void updateData() {
         datasetService.updateProjectData(project, "DELETE FROM {attr.person.name} WHERE {label.person.name} = \"not exists\";");
     }
 
+    @Test(groups = "dataset", dependsOnGroups = {"md", "datastore"})
+    public void loadDatasetFail(){
+        final DatasetService datasetService = gd.getDatasetService();
+        final DatasetManifest manifest = datasetService.getDatasetManifest(project, "dataset.person");
+        try {
+            datasetService.loadDataset(project, manifest, getClass().getResourceAsStream("/corruptedPerson.csv")).get();
+            fail();
+        } catch (DatasetException ex){
+            assertThat(ex.getMessage(),is(equalTo("Load datasets [dataset.person] failed: Number of columns doesn't corespond on line 3 in dataset.person.csv")));
+        }
+    }
+
+    @Test(groups = "dataset", dependsOnMethods = {"loadDataset"})
+    public void loadDatasetBatchFail() throws Exception {
+        final DatasetService datasetService = gd.getDatasetService();
 
+        final DatasetManifest personManifest = datasetService.getDatasetManifest(project, "dataset.person");
+        personManifest.setSource(getClass().getResourceAsStream("/corruptedPerson.csv"));
+        final DatasetManifest cityManifest = datasetService.getDatasetManifest(project, "dataset.city");
+        cityManifest.setSource(getClass().getResourceAsStream("/city.csv"));
+
+        try {
+            datasetService.loadDatasets(project, personManifest, cityManifest).get();
+            fail();
+        } catch (DatasetException ex){
+            assertThat(ex.getMessage(),is(equalTo("Load datasets [dataset.person, dataset.city] failed: [Number of columns doesn't corespond on line 3 in dataset.person.csv, Number of columns doesn't corespond on line 3 in dataset.person.csv]")));
+        }
+    }
 }
diff --git a/src/test/java/com/gooddata/dataset/DatasetServiceIT.java b/src/test/java/com/gooddata/dataset/DatasetServiceIT.java
@@ -280,4 +280,54 @@ public void shouldFailUpdateProjectData() throws IOException {
 
         gd.getDatasetService().updateProjectData(project, DML_MAQL).get();
     }
+
+    @Test
+    public void shouldReadBatchErrorMessages() throws Exception {
+        onRequest()
+                .havingPathEqualTo("/gdc/md/PROJECT/etl/task/ID")
+                .respond()
+                .withStatus(200)
+                .withBody(readFromResource("/dataset/pullTaskStatusError.json"));
+        onRequest()
+                .havingPath(containsString("upload_status.json"))
+                .havingMethodEqualTo("GET")
+                .respond()
+                .withStatus(200)
+                .withBody(readFromResource("/dataset/batchFailStatus1.json"));
+
+        final DatasetManifest manifest = MAPPER.readValue(readFromResource("/dataset/datasetManifest.json"), DatasetManifest.class);
+        final InputStream source = new ByteArrayInputStream(new byte[]{});
+        manifest.setSource(source);
+        try {
+            gd.getDatasetService().loadDatasets(project, manifest).get();
+            fail("Exception should be thrown");
+        } catch (DatasetException e) {
+            assertThat(e.getMessage(), is("Load datasets [dataset.person] failed: [Manifest consist of columns that are not in single CSV file dataset.stats.csv: f_competitors.nm_name., Manifest consist of columns that are not in single CSV file dataset.stats.csv: f_competitors.nm_name.]"));
+        }
+    }
+
+    @Test
+    public void shouldReadBatchErrorMessagesNoFailStatuses() throws Exception {
+        onRequest()
+                .havingPathEqualTo("/gdc/md/PROJECT/etl/task/ID")
+                .respond()
+                .withStatus(200)
+                .withBody(readFromResource("/dataset/pullTaskStatusError.json"));
+        onRequest()
+                .havingPath(containsString("upload_status.json"))
+                .havingMethodEqualTo("GET")
+                .respond()
+                .withStatus(200)
+                .withBody(readFromResource("/dataset/batchFailStatus2.json"));
+
+        final DatasetManifest manifest = MAPPER.readValue(readFromResource("/dataset/datasetManifest.json"), DatasetManifest.class);
+        final InputStream source = new ByteArrayInputStream(new byte[]{});
+        manifest.setSource(source);
+        try {
+            gd.getDatasetService().loadDatasets(project, manifest).get();
+            fail("Exception should be thrown");
+        } catch (DatasetException e) {
+            assertThat(e.getMessage(), is("Load datasets [dataset.person] failed: status: ERROR"));
+        }
+    }
 }
diff --git a/src/test/resources/corruptedPerson.csv b/src/test/resources/corruptedPerson.csv
@@ -0,0 +1,5 @@
+f_person.f_shoesize,f_person.nm_name,d_person_role.nm_role,f_person.f_age,d_person_department.nm_department
+37,Jane,manager,35,HR
+42,developer,25,DevOps
+35,Sandy,recruiter,27
+40,Jonathan,developer,50,DevOps
diff --git a/src/test/resources/dataset/batchFailStatus1.json b/src/test/resources/dataset/batchFailStatus1.json
@@ -0,0 +1,33 @@
+{
+   "uploads" : [
+      {
+         "date" : "2016-02-01 10:12:09",
+         "status" : "ERROR",
+         "error" : {
+            "parameters" : [
+               "dataset.stats.csv",
+               "f_competitors.nm_name"
+            ],
+            "component" : "GDC::SliToDli",
+            "errorClass" : "GDC::Exception::User",
+            "message" : "Manifest consist of columns that are not in single CSV file %s: %s."
+         }
+      },
+      {
+         "date" : "2016-02-01 10:12:09",
+         "status" : "ERROR",
+         "error" : {
+            "parameters" : [
+               "dataset.stats.csv",
+               "f_competitors.nm_name"
+            ],
+            "component" : "GDC::SliToDli",
+            "errorClass" : "GDC::Exception::User",
+            "message" : "Manifest consist of columns that are not in single CSV file %s: %s."
+         }
+      }
+   ],
+   "messages" : [],
+   "status" : "ERROR",
+   "date" : "2016-02-01 10:12:09"
+}
diff --git a/src/test/resources/dataset/batchFailStatus2.json b/src/test/resources/dataset/batchFailStatus2.json
@@ -0,0 +1,9 @@
+{
+  "uploads": [],
+  "messages": [
+    "test1",
+    "test2"
+  ],
+  "status": "ERROR",
+  "date": "2016-01-01 10:11:15"
+}