From f831092fd39e1261c07fbd9619615846c1b36279 Mon Sep 17 00:00:00 2001 From: Ondrej Benkovsky Date: Mon, 1 Feb 2016 14:55:33 +0100 Subject: [PATCH] parser for batch fail status json --- .../com/gooddata/dataset/BatchFailStatus.java | 54 +++++++++++++ .../com/gooddata/dataset/DatasetService.java | 80 ++++++++++++------- .../gooddata/dataset/BatchFailStatusTest.java | 43 ++++++++++ .../gooddata/dataset/DatasetServiceAT.java | 33 ++++++++ .../gooddata/dataset/DatasetServiceIT.java | 50 ++++++++++++ src/test/resources/corruptedPerson.csv | 5 ++ .../resources/dataset/batchFailStatus1.json | 33 ++++++++ .../resources/dataset/batchFailStatus2.json | 9 +++ 8 files changed, 277 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/gooddata/dataset/BatchFailStatus.java create mode 100644 src/test/java/com/gooddata/dataset/BatchFailStatusTest.java create mode 100644 src/test/resources/corruptedPerson.csv create mode 100755 src/test/resources/dataset/batchFailStatus1.json create mode 100644 src/test/resources/dataset/batchFailStatus2.json diff --git a/src/main/java/com/gooddata/dataset/BatchFailStatus.java b/src/main/java/com/gooddata/dataset/BatchFailStatus.java new file mode 100644 index 000000000..924c9cad6 --- /dev/null +++ b/src/main/java/com/gooddata/dataset/BatchFailStatus.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved. + */ +package com.gooddata.dataset; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.gooddata.util.GDDateTimeDeserializer; +import com.gooddata.util.GDDateTimeSerializer; + +import org.joda.time.DateTime; + +import java.util.List; + +/** + * Batch fail status of dataset load. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class BatchFailStatus { + + private final List failStatuses; + private final List messages; + private final String status; + private final DateTime date; + + @JsonCreator + private BatchFailStatus(@JsonProperty("uploads") List failStatuses, @JsonProperty("messages") List messages, + @JsonProperty("status") String status, @JsonProperty("date") @JsonDeserialize(using = GDDateTimeDeserializer.class) DateTime date) { + this.failStatuses = failStatuses; + this.messages = messages; + this.status = status; + this.date = date; + } + + public List getFailStatuses() { + return failStatuses; + } + + public List getMessages() { + return messages; + } + + public String getStatus() { + return status; + } + + @JsonSerialize(using = GDDateTimeSerializer.class) + public DateTime getDate() { + return date; + } +} diff --git a/src/main/java/com/gooddata/dataset/DatasetService.java b/src/main/java/com/gooddata/dataset/DatasetService.java index 2a5dc57e4..9332208b4 100644 --- a/src/main/java/com/gooddata/dataset/DatasetService.java +++ b/src/main/java/com/gooddata/dataset/DatasetService.java @@ -35,6 +35,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.springframework.util.StringUtils.isEmpty; /** @@ -101,7 +102,7 @@ public FutureResult loadDataset(final Project project, final DatasetManife final ByteArrayInputStream inputStream = new ByteArrayInputStream(manifestJson.getBytes(UTF_8)); dataStoreService.upload(dirPath.resolve(MANIFEST_FILE_NAME).toString(), inputStream); - return pullLoad(project, dirPath, manifest.getDataSet()); + return pullLoad(project, dirPath, manifest.getDataSet(), false); } catch (IOException e) { throw new DatasetException("Unable to serialize manifest", manifest.getDataSet(), e); } catch (DataStoreException | GoodDataRestException | RestClientException e) { @@ -156,7 +157,7 @@ public FutureResult loadDatasets(final Project project, final Collection datasets) } } - private FutureResult pullLoad(Project project, final Path dirPath, final String dataset) { - return pullLoad(project, dirPath, asList(dataset)); + private FutureResult pullLoad(Project project, final Path dirPath, final String dataset, boolean isBatchUpload) { + return pullLoad(project, dirPath, singletonList(dataset), isBatchUpload); } - private FutureResult pullLoad(Project project, final Path dirPath, final Collection datasets) { + private FutureResult pullLoad(Project project, final Path dirPath, final Collection datasets, final boolean isBatchUpload) { final PullTask pullTask = restTemplate .postForObject(Pull.URI, new Pull(dirPath.toString()), PullTask.class, project.getId()); return new PollResult<>(this, new SimplePollHandler(pullTask.getUri(), Void.class) { @@ -192,7 +193,7 @@ public boolean isFinished(ClientHttpResponse response) throws IOException { final PullTaskStatus status = extractData(response, PullTaskStatus.class); final boolean finished = status.isFinished(); if (finished && !status.isSuccess()) { - final String message = getErrorMessage(status, dirPath); + final String message = getErrorMessage(status, dirPath, isBatchUpload); throw new DatasetException(message, datasets); } return finished; @@ -215,30 +216,16 @@ protected void onFinish() { } - private String getErrorMessage(final PullTaskStatus status, final Path dirPath) { + private String getErrorMessage(final PullTaskStatus status, final Path dirPath, boolean isBatchUpload) { String message = "status: " + status.getStatus(); try { - final FailStatus failStatus = download(dirPath.resolve(STATUS_FILE_NAME), FailStatus.class); - if (failStatus != null) { - final List errorParts = failStatus.getErrorParts(); - if (!errorParts.isEmpty()) { - final List errors = new ArrayList<>(); - for (FailPart part: errorParts) { - if (part.getLogName() != null) { - try { - final String[] msg = download(dirPath.resolve(part.getLogName()), String[].class); - errors.addAll(asList(msg)); - } catch (IOException | DataStoreException e) { - if (part.getError() != null) { - errors.add(part.getError().getFormattedMessage()); - } - } - } - } - message = errors.toString(); - } else if (failStatus.getError() != null) { - message = failStatus.getError().getFormattedMessage(); - } + Path statusFile = dirPath.resolve(STATUS_FILE_NAME); + if (isBatchUpload) { + final BatchFailStatus batchFailStatus = download(statusFile, BatchFailStatus.class); + message = getBatchFailStatusErrorMsg(dirPath, message, batchFailStatus); + } else { + final FailStatus failStatus = download(statusFile, FailStatus.class); + message = getFailStatusErrorMsg(dirPath, message, failStatus); } } catch (IOException | DataStoreException ignored) { // todo log? @@ -246,6 +233,41 @@ private String getErrorMessage(final PullTaskStatus status, final Path dirPath) return message; } + private String getBatchFailStatusErrorMsg(final Path dirPath, final String defaultMessage, final BatchFailStatus batchFailStatus) { + if (batchFailStatus == null || batchFailStatus.getFailStatuses() == null) { + return defaultMessage; + } + final List messages = new ArrayList<>(); + for (FailStatus failStatus : batchFailStatus.getFailStatuses()) { + messages.add(getFailStatusErrorMsg(dirPath, defaultMessage, failStatus)); + } + return messages.isEmpty() ? defaultMessage : messages.toString(); + } + + private String getFailStatusErrorMsg(final Path dirPath, final String defaultMessage, final FailStatus failStatus) { + if (failStatus == null) { + return defaultMessage; + } + final List errorParts = failStatus.getErrorParts(); + if (errorParts.isEmpty()){ + return (failStatus.getError() != null) ? failStatus.getError().getFormattedMessage() : defaultMessage; + } + final List errors = new ArrayList<>(); + for (FailPart part : errorParts) { + if (part.getLogName() != null) { + try { + final String[] msg = download(dirPath.resolve(part.getLogName()), String[].class); + errors.addAll(asList(msg)); + } catch (IOException | DataStoreException e) { + if (part.getError() != null) { + errors.add(part.getError().getFormattedMessage()); + } + } + } + } + return errors.toString(); + } + private T download(final Path path, final Class type) throws IOException { try (final InputStream input = dataStoreService.download(path.toString())) { return mapper.readValue(input, type); @@ -324,7 +346,6 @@ public void handlePollException(final GoodDataRestException e) { * @param project project to be updated * @param maqlDml update script to be executed in the project * @return poll result - * * @see com.gooddata.model.ModelService#updateProjectModel */ public FutureResult updateProjectData(final Project project, final String maqlDml) { @@ -362,5 +383,4 @@ public void handlePollException(final GoodDataRestException e) { } }); } - } diff --git a/src/test/java/com/gooddata/dataset/BatchFailStatusTest.java b/src/test/java/com/gooddata/dataset/BatchFailStatusTest.java new file mode 100644 index 000000000..705397fa0 --- /dev/null +++ b/src/test/java/com/gooddata/dataset/BatchFailStatusTest.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved. + */ +package com.gooddata.dataset; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.hamcrest.Matchers; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.InputStream; + +public class BatchFailStatusTest { + @Test + public void testParser() throws Exception { + final InputStream stream = getClass().getResourceAsStream("/dataset/batchFailStatus1.json"); + final BatchFailStatus value = new ObjectMapper().readValue(stream, BatchFailStatus.class); + assertThat(value, is(notNullValue())); + assertThat(value.getStatus(),is("ERROR")); + assertThat(value.getMessages(),is(Matchers.empty())); + assertThat(value.getDate(),is(new DateTime(2016,2,1,10,12,9, DateTimeZone.UTC))); + assertThat(value.getFailStatuses(),hasSize(2)); + } + + @Test + public void testParser2() throws IOException { + final InputStream stream = getClass().getResourceAsStream("/dataset/batchFailStatus2.json"); + final BatchFailStatus value = new ObjectMapper().readValue(stream, BatchFailStatus.class); + assertThat(value,is(notNullValue())); + assertThat(value.getStatus(),is("ERROR")); + assertThat(value.getMessages(),containsInAnyOrder("test1","test2")); + assertThat(value.getDate(),is(new DateTime(2016,1,1,10,11,15, DateTimeZone.UTC))); + assertThat(value.getFailStatuses(),is(Matchers.empty())); + } +} \ No newline at end of file diff --git a/src/test/java/com/gooddata/dataset/DatasetServiceAT.java b/src/test/java/com/gooddata/dataset/DatasetServiceAT.java index 6afb34cdb..42dde60c4 100644 --- a/src/test/java/com/gooddata/dataset/DatasetServiceAT.java +++ b/src/test/java/com/gooddata/dataset/DatasetServiceAT.java @@ -1,6 +1,12 @@ package com.gooddata.dataset; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.testng.Assert.fail; + import com.gooddata.AbstractGoodDataAT; +import static org.hamcrest.MatcherAssert.assertThat; + import org.testng.annotations.Test; /** @@ -34,5 +40,32 @@ public void updateData() { datasetService.updateProjectData(project, "DELETE FROM {attr.person.name} WHERE {label.person.name} = \"not exists\";"); } + @Test(groups = "dataset", dependsOnGroups = {"md", "datastore"}) + public void loadDatasetFail(){ + final DatasetService datasetService = gd.getDatasetService(); + final DatasetManifest manifest = datasetService.getDatasetManifest(project, "dataset.person"); + try { + datasetService.loadDataset(project, manifest, getClass().getResourceAsStream("/corruptedPerson.csv")).get(); + fail(); + } catch (DatasetException ex){ + assertThat(ex.getMessage(),is(equalTo("Load datasets [dataset.person] failed: Number of columns doesn't corespond on line 3 in dataset.person.csv"))); + } + } + + @Test(groups = "dataset", dependsOnMethods = {"loadDataset"}) + public void loadDatasetBatchFail() throws Exception { + final DatasetService datasetService = gd.getDatasetService(); + final DatasetManifest personManifest = datasetService.getDatasetManifest(project, "dataset.person"); + personManifest.setSource(getClass().getResourceAsStream("/corruptedPerson.csv")); + final DatasetManifest cityManifest = datasetService.getDatasetManifest(project, "dataset.city"); + cityManifest.setSource(getClass().getResourceAsStream("/city.csv")); + + try { + datasetService.loadDatasets(project, personManifest, cityManifest).get(); + fail(); + } catch (DatasetException ex){ + assertThat(ex.getMessage(),is(equalTo("Load datasets [dataset.person, dataset.city] failed: [Number of columns doesn't corespond on line 3 in dataset.person.csv, Number of columns doesn't corespond on line 3 in dataset.person.csv]"))); + } + } } diff --git a/src/test/java/com/gooddata/dataset/DatasetServiceIT.java b/src/test/java/com/gooddata/dataset/DatasetServiceIT.java index 05f5dc9a0..1e43fac44 100644 --- a/src/test/java/com/gooddata/dataset/DatasetServiceIT.java +++ b/src/test/java/com/gooddata/dataset/DatasetServiceIT.java @@ -280,4 +280,54 @@ public void shouldFailUpdateProjectData() throws IOException { gd.getDatasetService().updateProjectData(project, DML_MAQL).get(); } + + @Test + public void shouldReadBatchErrorMessages() throws Exception { + onRequest() + .havingPathEqualTo("/gdc/md/PROJECT/etl/task/ID") + .respond() + .withStatus(200) + .withBody(readFromResource("/dataset/pullTaskStatusError.json")); + onRequest() + .havingPath(containsString("upload_status.json")) + .havingMethodEqualTo("GET") + .respond() + .withStatus(200) + .withBody(readFromResource("/dataset/batchFailStatus1.json")); + + final DatasetManifest manifest = MAPPER.readValue(readFromResource("/dataset/datasetManifest.json"), DatasetManifest.class); + final InputStream source = new ByteArrayInputStream(new byte[]{}); + manifest.setSource(source); + try { + gd.getDatasetService().loadDatasets(project, manifest).get(); + fail("Exception should be thrown"); + } catch (DatasetException e) { + assertThat(e.getMessage(), is("Load datasets [dataset.person] failed: [Manifest consist of columns that are not in single CSV file dataset.stats.csv: f_competitors.nm_name., Manifest consist of columns that are not in single CSV file dataset.stats.csv: f_competitors.nm_name.]")); + } + } + + @Test + public void shouldReadBatchErrorMessagesNoFailStatuses() throws Exception { + onRequest() + .havingPathEqualTo("/gdc/md/PROJECT/etl/task/ID") + .respond() + .withStatus(200) + .withBody(readFromResource("/dataset/pullTaskStatusError.json")); + onRequest() + .havingPath(containsString("upload_status.json")) + .havingMethodEqualTo("GET") + .respond() + .withStatus(200) + .withBody(readFromResource("/dataset/batchFailStatus2.json")); + + final DatasetManifest manifest = MAPPER.readValue(readFromResource("/dataset/datasetManifest.json"), DatasetManifest.class); + final InputStream source = new ByteArrayInputStream(new byte[]{}); + manifest.setSource(source); + try { + gd.getDatasetService().loadDatasets(project, manifest).get(); + fail("Exception should be thrown"); + } catch (DatasetException e) { + assertThat(e.getMessage(), is("Load datasets [dataset.person] failed: status: ERROR")); + } + } } \ No newline at end of file diff --git a/src/test/resources/corruptedPerson.csv b/src/test/resources/corruptedPerson.csv new file mode 100644 index 000000000..d31565503 --- /dev/null +++ b/src/test/resources/corruptedPerson.csv @@ -0,0 +1,5 @@ +f_person.f_shoesize,f_person.nm_name,d_person_role.nm_role,f_person.f_age,d_person_department.nm_department +37,Jane,manager,35,HR +42,developer,25,DevOps +35,Sandy,recruiter,27 +40,Jonathan,developer,50,DevOps diff --git a/src/test/resources/dataset/batchFailStatus1.json b/src/test/resources/dataset/batchFailStatus1.json new file mode 100755 index 000000000..cb135ab94 --- /dev/null +++ b/src/test/resources/dataset/batchFailStatus1.json @@ -0,0 +1,33 @@ +{ + "uploads" : [ + { + "date" : "2016-02-01 10:12:09", + "status" : "ERROR", + "error" : { + "parameters" : [ + "dataset.stats.csv", + "f_competitors.nm_name" + ], + "component" : "GDC::SliToDli", + "errorClass" : "GDC::Exception::User", + "message" : "Manifest consist of columns that are not in single CSV file %s: %s." + } + }, + { + "date" : "2016-02-01 10:12:09", + "status" : "ERROR", + "error" : { + "parameters" : [ + "dataset.stats.csv", + "f_competitors.nm_name" + ], + "component" : "GDC::SliToDli", + "errorClass" : "GDC::Exception::User", + "message" : "Manifest consist of columns that are not in single CSV file %s: %s." + } + } + ], + "messages" : [], + "status" : "ERROR", + "date" : "2016-02-01 10:12:09" +} \ No newline at end of file diff --git a/src/test/resources/dataset/batchFailStatus2.json b/src/test/resources/dataset/batchFailStatus2.json new file mode 100644 index 000000000..e789a3157 --- /dev/null +++ b/src/test/resources/dataset/batchFailStatus2.json @@ -0,0 +1,9 @@ +{ + "uploads": [], + "messages": [ + "test1", + "test2" + ], + "status": "ERROR", + "date": "2016-01-01 10:11:15" +} \ No newline at end of file