prep for upgrade to latest gaffer tools

gchq · Jan 9, 2019 · 61e3a92 · 61e3a92
1 parent b411e8f
commit 61e3a92
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 13 deletions.
diff --git a/python-api/gafferpy-core/src/main/python/gafferpy_core/gaffer_operations.py b/python-api/gafferpy-core/src/main/python/gafferpy_core/gaffer_operations.py
@@ -2550,11 +2550,10 @@ def to_json(self):
 
         return operation
 
-class AddElementsFromLocalCsv(Operation):
-    CLASS = 'uk.gov.gchq.gaffer.simplified.operation.AddElementsFromLocalCsv'
+class AddElementsFromCsv(Operation):
+    CLASS = 'uk.gov.gchq.gaffer.operation.impl.add.AddElementsFromCsv'
 
     def __init__(self, filename=None, mappingsFile=None,
-                 delimiter=None, ignoreQuotes=None, quoteChar=None,
                  validate=None, skip_invalid_elements=None,
                  options=None):
         super().__init__(
@@ -2564,9 +2563,6 @@ def __init__(self, filename=None, mappingsFile=None,
 
         self.filename = filename
         self.mappingsFile = mappingsFile
-        self.delimiter = delimiter
-        self.ignoreQuotes = ignoreQuotes
-        self.quoteChar = quoteChar
         self.validate = validate
         self.skip_invalid_elements = skip_invalid_elements
 
@@ -2579,14 +2575,53 @@ def to_json(self):
         if self.mappingsFile is not None:
             operation['mappingsFile'] = self.mappingsFile
 
-        if self.delimiter is not None:
-            operation['delimiter'] = self.delimiter
+        if self.validate is not None:
+            operation['validate'] = self.validate
+
+        if self.skip_invalid_elements is not None:
+            operation['skipInvalidElements'] = self.skip_invalid_elements
+
+        return operation
+
+
+
+
+
+
+
+class AddElementsFromHdfsQuickstart(Operation):
+    CLASS = 'uk.gov.gchq.gaffer.hdfs.operation.AddElementsFromHdfsQuickstart'
+
+    def __init__(self, dataPath=None, elementGeneratorConfig=None,
+                 outputPath=None, failurePath=None,
+                 validate=None, skip_invalid_elements=None,
+                 options=None):
+        super().__init__(
+            self.CLASS,
+            options=options
+        )
 
-        if self.ignoreQuotes is not None:
-            operation['ignoreQuotes'] = self.ignoreQuotes
+        self.dataPath = dataPath
+        self.elementGeneratorConfig = elementGeneratorConfig
+        self.outputPath = outputPath
+        self.failurePath = failurePath
+        self.validate = validate
+        self.skip_invalid_elements = skip_invalid_elements
+
+    def to_json(self):
+        operation = super().to_json()
 
-        if self.quoteChar is not None:
-            operation['quoteChar'] = self.quoteChar
+        if self.dataPath is not None:
+            operation['dataPath'] = self.dataPath
+
+        if self.elementGeneratorConfig is not None:
+            operation['elementGeneratorConfig'] = self.elementGeneratorConfig
+
+        if self.outputPath is not None:
+            operation['outputPath'] = self.outputPath
+
+        if self.failurePath is not None:
+            operation['failurePath'] = self.failurePath
 
         if self.validate is not None:
             operation['validate'] = self.validate
@@ -2599,6 +2634,12 @@ def to_json(self):
 
 
 
+
+
+
+
+
+
 def load_operation_json_map():
     for name, class_obj in inspect.getmembers(
             sys.modules[__name__], inspect.isclass):

diff --git a/...c/test/java/uk/gov/gchq/gaffer/python/pyspark/serialiser/impl/PysparkSerialiserTests.java b/...c/test/java/uk/gov/gchq/gaffer/python/pyspark/serialiser/impl/PysparkSerialiserTests.java
@@ -16,6 +16,7 @@
 
 package uk.gov.gchq.gaffer.python.pyspark.serialiser.impl;
 
+import org.apache.commons.io.FileUtils;
 import org.junit.Test;
 import uk.gov.gchq.gaffer.data.element.Edge;
 import uk.gov.gchq.gaffer.data.element.Entity;
@@ -24,7 +25,11 @@
 import uk.gov.gchq.gaffer.jsonserialisation.JSONSerialiser;
 import uk.gov.gchq.gaffer.python.util.Constants;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
@@ -123,5 +128,4 @@ public void testPysparkElementJsonSerialiser(){
         assertEquals(edgeResult, serialiser.convert(edge));
         assertEquals(entityResult, serialiser.convert(entity));
     }
-
 }