diff --git a/python-api/docker/Dockerfile.quickstart.pyspark b/python-api/docker/Dockerfile.quickstart.pyspark new file mode 100644 index 000000000..797ef4ea7 --- /dev/null +++ b/python-api/docker/Dockerfile.quickstart.pyspark @@ -0,0 +1,26 @@ +FROM gaffer-quickstart-base + +WORKDIR /usr/gaffer + +RUN curl -L -O https://repo.anaconda.com/archive/Anaconda3-2018.12-Linux-x86_64.sh + +RUN yum -y install bzip2 + +RUN chmod u+x Anaconda3-2018.12-Linux-x86_64.sh + +RUN /bin/bash Anaconda3-2018.12-Linux-x86_64.sh -b -p /usr/gaffer/anaconda3 + +RUN curl -L -O https://archive.apache.org/dist/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz + +RUN tar -xvf spark-2.2.1-bin-hadoop2.7.tgz + +ENV ANACONDA_HOME="/usr/gaffer/anaconda3" +ENV SPARK_HOME="/usr/gaffer/spark-2.2.1-bin-hadoop2.7" +ENV PYSPARK_DRIVER_PYTHON=jupyter +ENV PYSPARK_DRIVER_PYTHON_OPTS='notebook --ip 0.0.0.0 --no-browser --allow-root' +ENV PATH=$ANACONDA_HOME/bin:$SPARK_HOME/bin:$ANACONDA_HOME/lib/:$PATH + +COPY gafferpy-build-1.7.1-SNAPSHOT-jar-with-dependencies.jar gafferpy-build-1.7.1-SNAPSHOT-jar-with-dependencies.jar +COPY gafferpy-build-1.7.1-SNAPSHOT-python-modules.zip gafferpy-build-1.7.1-SNAPSHOT-python-modules.zip + +CMD $GAFFER_HOME/bin/startup.sh --schema $SCHEMA --graphconfig $GRAPHCONFIG --storeproperties $STOREPROPERTIES --ui-config $UICONFIG ; pyspark --jars gafferpy-build-1.7.1-SNAPSHOT-jar-with-dependencies.jar --py-files gafferpy-build-1.7.1-SNAPSHOT-python-modules.zip --packages graphframes:graphframes:0.6.0-spark2.3-s_2.11 diff --git a/python-api/docker/docker-compose.yml b/python-api/docker/docker-compose.yml new file mode 100644 index 000000000..779a45600 --- /dev/null +++ b/python-api/docker/docker-compose.yml @@ -0,0 +1,8 @@ +version: "3" + +services: + gaffer-quickstart: + image: "gaffer-quickstart-pyspark" + ports: + - "8085:8085" + - "8888:8888" diff --git a/python-api/gafferpy-pyspark/src/main/java/uk/gov/gchq/gaffer/python/pyspark/serialiser/PysparkDataframeSerialiser.java b/python-api/gafferpy-pyspark/src/main/java/uk/gov/gchq/gaffer/python/pyspark/serialiser/PysparkDataframeSerialiser.java index b10ac386b..0f1de0fe2 100644 --- a/python-api/gafferpy-pyspark/src/main/java/uk/gov/gchq/gaffer/python/pyspark/serialiser/PysparkDataframeSerialiser.java +++ b/python-api/gafferpy-pyspark/src/main/java/uk/gov/gchq/gaffer/python/pyspark/serialiser/PysparkDataframeSerialiser.java @@ -25,7 +25,6 @@ public class PysparkDataframeSerialiser implements Converter> { - @Override public Map convert(final Element element) { return null;