#1078 fixing IT test, some table outputs must be absent when the inte…

…rpretation for that table is not requested
gbif · Sep 13, 2024 · ecfb06a · ecfb06a
1 parent c2566db
commit ecfb06a
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 17 deletions.
diff --git a/...ion/ingest-gbif-beam/src/main/java/org/gbif/pipelines/ingest/utils/HdfsViewAvroUtils.java b/...ion/ingest-gbif-beam/src/main/java/org/gbif/pipelines/ingest/utils/HdfsViewAvroUtils.java
@@ -38,7 +38,7 @@ private static void copyOccurrence(InterpretationPipelineOptions options) {
     copyAndOverwrite(options, OCCURRENCE);
   }
 
-  private static void cleanAndMoveTables(InterpretationPipelineOptions opt, RecordType coreType) {
+  private static void copyAndOverwriteTables(InterpretationPipelineOptions opt, RecordType coreType) {
 
     copyAndOverwrite(opt, coreType, MEASUREMENT_OR_FACT_TABLE, Extension.MEASUREMENT_OR_FACT);
     copyAndOverwrite(opt, coreType, IDENTIFICATION_TABLE, Extension.IDENTIFICATION);
@@ -72,10 +72,10 @@ private static void cleanAndMoveTables(InterpretationPipelineOptions opt, Record
   private static void copyAll(InterpretationPipelineOptions options) {
     if (options.getCoreRecordType() == OCCURRENCE) {
       copyAndOverwrite(options, OCCURRENCE);
-      cleanAndMoveTables(options, OCCURRENCE);
+      copyAndOverwriteTables(options, OCCURRENCE);
     } else if (options.getCoreRecordType() == EVENT) {
       copyAndOverwrite(options, EVENT);
-      cleanAndMoveTables(options, EVENT);
+      copyAndOverwriteTables(options, EVENT);
     }
   }
 

diff --git a/...tion-tests/src/test/java/org/gbif/pipelines/ingest/java/pipelines/HdfsViewPipelineIT.java b/...tion-tests/src/test/java/org/gbif/pipelines/ingest/java/pipelines/HdfsViewPipelineIT.java
@@ -134,8 +134,10 @@ public void hdfsPipelineTest(String rootTestFolder, InterpretationType.RecordTyp
 
     assertFileExists(outputFn.apply("extendedmeasurementorfacttable"));
     assertFileExists(outputFn.apply("germplasmmeasurementtrialtable"));
-    assertFileExists(outputFn.apply("permittable"));
-    assertFileExists(outputFn.apply("loantable"));
+
+    //Next two interpretations were not in the list of interpretation types param, --interpretationTypes
+    assertFileNotExists(outputFn.apply("permittable"));
+    assertFileNotExists(outputFn.apply("loantable"));
   }
 
   @Test
@@ -204,11 +206,13 @@ public void singleHdfsPipelineTest(
                 + "_147.avro";
 
     assertFile(OccurrenceHdfsRecord.class, outputFn.apply(recordType.name().toLowerCase()));
-    assertFileExists(outputFn.apply("measurementorfacttable"));
-    assertFileExists(outputFn.apply("extendedmeasurementorfacttable"));
-    assertFileExists(outputFn.apply("germplasmmeasurementtrialtable"));
-    assertFileExists(outputFn.apply("permittable"));
-    assertFileExists(outputFn.apply("loantable"));
+
+    //Tables were not requested in the interpretation types
+    assertFileNotExists(outputFn.apply("measurementorfacttable"));
+    assertFileNotExists(outputFn.apply("extendedmeasurementorfacttable"));
+    assertFileNotExists(outputFn.apply("germplasmmeasurementtrialtable"));
+    assertFileNotExists(outputFn.apply("permittable"));
+    assertFileNotExists(outputFn.apply("loantable"));
   }
 
   @SneakyThrows
@@ -334,6 +338,10 @@ private void assertFileExists(String output) {
     Assert.assertTrue(new File(output).exists());
   }
 
+  private void assertFileNotExists(String output) {
+    Assert.assertFalse(new File(output).exists());
+  }
+
   private <T extends SpecificRecordBase> void assertFile(Class<T> clazz, String output)
       throws Exception {
     File file = new File(output);

diff --git a/...tegration-tests/src/test/java/org/gbif/pipelines/ingest/pipelines/HdfsViewPipelineIT.java b/...tegration-tests/src/test/java/org/gbif/pipelines/ingest/pipelines/HdfsViewPipelineIT.java
@@ -209,8 +209,10 @@ public void pipelineAllTest() throws Exception {
     assertFileExists(outputFn.apply("germplasmmeasurementtrialtable"));
     assertFileExists(outputFn.apply("audubontable"));
     assertFileExists(outputFn.apply("extendedmeasurementorfacttable"));
-    assertFileExists(outputFn.apply("permittable"));
-    assertFileExists(outputFn.apply("loantable"));
+
+    //Next two interpretations were not in the list of interpretation types param, --interpretationTypes
+    assertFileNotExists(outputFn.apply("permittable"));
+    assertFileNotExists(outputFn.apply("loantable"));
   }
 
   @Test
@@ -379,17 +381,21 @@ public void pipelineHDFSTest(InterpretationType.RecordType recordType) throws Ex
                 + "_147-00000-of-00001.avro";
 
     assertFile(OccurrenceHdfsRecord.class, outputFn.apply(recordType.name().toLowerCase()));
-    assertFileExists(outputFn.apply("measurementorfacttable"));
-    assertFileExists(outputFn.apply("extendedmeasurementorfacttable"));
-    assertFileExists(outputFn.apply("germplasmmeasurementtrialtable"));
-    assertFileExists(outputFn.apply("permittable"));
-    assertFileExists(outputFn.apply("loantable"));
+    assertFileNotExists(outputFn.apply("measurementorfacttable"));
+    assertFileNotExists(outputFn.apply("extendedmeasurementorfacttable"));
+    assertFileNotExists(outputFn.apply("germplasmmeasurementtrialtable"));
+    assertFileNotExists(outputFn.apply("permittable"));
+    assertFileNotExists(outputFn.apply("loantable"));
   }
 
   private void assertFileExists(String output) {
     Assert.assertTrue(new File(output).exists());
   }
 
+  private void assertFileNotExists(String output) {
+    Assert.assertFalse(new File(output).exists());
+  }
+
   private <T extends SpecificRecordBase> void assertFile(Class<T> clazz, String output)
       throws Exception {
     File file = new File(output);