From ecfb06aa295f0eba444ff69e97878107c33bd665 Mon Sep 17 00:00:00 2001 From: fmendezh Date: Fri, 13 Sep 2024 11:53:41 +0200 Subject: [PATCH] https://github.com/gbif/pipelines/issues/1078 fixing IT test, some table outputs must be absent when the interpretation for that table is not requested --- .../ingest/utils/HdfsViewAvroUtils.java | 6 ++--- .../java/pipelines/HdfsViewPipelineIT.java | 22 +++++++++++++------ .../ingest/pipelines/HdfsViewPipelineIT.java | 20 +++++++++++------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/gbif/ingestion/ingest-gbif-beam/src/main/java/org/gbif/pipelines/ingest/utils/HdfsViewAvroUtils.java b/gbif/ingestion/ingest-gbif-beam/src/main/java/org/gbif/pipelines/ingest/utils/HdfsViewAvroUtils.java index f4636dd6d..d20d66a2c 100644 --- a/gbif/ingestion/ingest-gbif-beam/src/main/java/org/gbif/pipelines/ingest/utils/HdfsViewAvroUtils.java +++ b/gbif/ingestion/ingest-gbif-beam/src/main/java/org/gbif/pipelines/ingest/utils/HdfsViewAvroUtils.java @@ -38,7 +38,7 @@ private static void copyOccurrence(InterpretationPipelineOptions options) { copyAndOverwrite(options, OCCURRENCE); } - private static void cleanAndMoveTables(InterpretationPipelineOptions opt, RecordType coreType) { + private static void copyAndOverwriteTables(InterpretationPipelineOptions opt, RecordType coreType) { copyAndOverwrite(opt, coreType, MEASUREMENT_OR_FACT_TABLE, Extension.MEASUREMENT_OR_FACT); copyAndOverwrite(opt, coreType, IDENTIFICATION_TABLE, Extension.IDENTIFICATION); @@ -72,10 +72,10 @@ private static void cleanAndMoveTables(InterpretationPipelineOptions opt, Record private static void copyAll(InterpretationPipelineOptions options) { if (options.getCoreRecordType() == OCCURRENCE) { copyAndOverwrite(options, OCCURRENCE); - cleanAndMoveTables(options, OCCURRENCE); + copyAndOverwriteTables(options, OCCURRENCE); } else if (options.getCoreRecordType() == EVENT) { copyAndOverwrite(options, EVENT); - cleanAndMoveTables(options, EVENT); + copyAndOverwriteTables(options, EVENT); } } diff --git a/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/java/pipelines/HdfsViewPipelineIT.java b/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/java/pipelines/HdfsViewPipelineIT.java index 8de87d89c..93b9b9135 100644 --- a/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/java/pipelines/HdfsViewPipelineIT.java +++ b/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/java/pipelines/HdfsViewPipelineIT.java @@ -134,8 +134,10 @@ public void hdfsPipelineTest(String rootTestFolder, InterpretationType.RecordTyp assertFileExists(outputFn.apply("extendedmeasurementorfacttable")); assertFileExists(outputFn.apply("germplasmmeasurementtrialtable")); - assertFileExists(outputFn.apply("permittable")); - assertFileExists(outputFn.apply("loantable")); + + //Next two interpretations were not in the list of interpretation types param, --interpretationTypes + assertFileNotExists(outputFn.apply("permittable")); + assertFileNotExists(outputFn.apply("loantable")); } @Test @@ -204,11 +206,13 @@ public void singleHdfsPipelineTest( + "_147.avro"; assertFile(OccurrenceHdfsRecord.class, outputFn.apply(recordType.name().toLowerCase())); - assertFileExists(outputFn.apply("measurementorfacttable")); - assertFileExists(outputFn.apply("extendedmeasurementorfacttable")); - assertFileExists(outputFn.apply("germplasmmeasurementtrialtable")); - assertFileExists(outputFn.apply("permittable")); - assertFileExists(outputFn.apply("loantable")); + + //Tables were not requested in the interpretation types + assertFileNotExists(outputFn.apply("measurementorfacttable")); + assertFileNotExists(outputFn.apply("extendedmeasurementorfacttable")); + assertFileNotExists(outputFn.apply("germplasmmeasurementtrialtable")); + assertFileNotExists(outputFn.apply("permittable")); + assertFileNotExists(outputFn.apply("loantable")); } @SneakyThrows @@ -334,6 +338,10 @@ private void assertFileExists(String output) { Assert.assertTrue(new File(output).exists()); } + private void assertFileNotExists(String output) { + Assert.assertFalse(new File(output).exists()); + } + private void assertFile(Class clazz, String output) throws Exception { File file = new File(output); diff --git a/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/pipelines/HdfsViewPipelineIT.java b/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/pipelines/HdfsViewPipelineIT.java index 71780fe89..fa28fecb3 100644 --- a/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/pipelines/HdfsViewPipelineIT.java +++ b/gbif/ingestion/ingestion-integration-tests/src/test/java/org/gbif/pipelines/ingest/pipelines/HdfsViewPipelineIT.java @@ -209,8 +209,10 @@ public void pipelineAllTest() throws Exception { assertFileExists(outputFn.apply("germplasmmeasurementtrialtable")); assertFileExists(outputFn.apply("audubontable")); assertFileExists(outputFn.apply("extendedmeasurementorfacttable")); - assertFileExists(outputFn.apply("permittable")); - assertFileExists(outputFn.apply("loantable")); + + //Next two interpretations were not in the list of interpretation types param, --interpretationTypes + assertFileNotExists(outputFn.apply("permittable")); + assertFileNotExists(outputFn.apply("loantable")); } @Test @@ -379,17 +381,21 @@ public void pipelineHDFSTest(InterpretationType.RecordType recordType) throws Ex + "_147-00000-of-00001.avro"; assertFile(OccurrenceHdfsRecord.class, outputFn.apply(recordType.name().toLowerCase())); - assertFileExists(outputFn.apply("measurementorfacttable")); - assertFileExists(outputFn.apply("extendedmeasurementorfacttable")); - assertFileExists(outputFn.apply("germplasmmeasurementtrialtable")); - assertFileExists(outputFn.apply("permittable")); - assertFileExists(outputFn.apply("loantable")); + assertFileNotExists(outputFn.apply("measurementorfacttable")); + assertFileNotExists(outputFn.apply("extendedmeasurementorfacttable")); + assertFileNotExists(outputFn.apply("germplasmmeasurementtrialtable")); + assertFileNotExists(outputFn.apply("permittable")); + assertFileNotExists(outputFn.apply("loantable")); } private void assertFileExists(String output) { Assert.assertTrue(new File(output).exists()); } + private void assertFileNotExists(String output) { + Assert.assertFalse(new File(output).exists()); + } + private void assertFile(Class clazz, String output) throws Exception { File file = new File(output);