Updated Gaffer version to 2.3.1 (#384)

.github/workflows/release.yaml

@@ -160,7 +160,7 @@ jobs:
     - name: Upload Asset
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: gh release upload ${{ needs.release-version.outputs.release_version }} ${{ env.ASSET_FILENAME }}
+      run: gh release upload ${{ needs.get-release-version.outputs.release_version }} ${{ env.ASSET_FILENAME }}
 
   update-helm-repo:
     runs-on: ubuntu-latest

README.md

@@ -1,20 +1,29 @@
 # Gaffer Docker
 
 This repo contains the code needed to run Gaffer using Docker or Kubernetes.
-There are two main sub-folders, 'docker' and 'kubernetes' which contain the project files you need for starting Gaffer using those services.
+There are two main sub-folders, 'docker' and 'kubernetes', which contain the
+project files you need for starting Gaffer using those services.
 
-# Running Gaffer Using Docker
+## Running Gaffer Using Docker
 
-For information on how to run Gaffer using Docker containers, please see the documentation: [Gaffer Docker Docs](https://gchq.github.io/gaffer-doc/latest/dev/docker/)
+For information on how to run Gaffer using Docker containers please see the
+documentation: [Gaffer Docker Docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gaffer-docker/gaffer-images.html)
 
-# Running Gaffer Using Kubernetes
+We also provide some example deployments with different store backings
+to help you get started learning and testing Gaffer. Please see the
+[example deployments](./docker/example-deployments/) directory for more
+details.
 
-For information on how to run Gaffer using Kubernetes, please see the documentation: [Gaffer Kubernetes Docs](https://gchq.github.io/gaffer-doc/latest/dev/kubernetes-guide/kubernetes/)
+## Running Gaffer Using Kubernetes
 
-# Versioning
+For information on how to run Gaffer using Kubernetes, please see the
+documentation: [Gaffer Kubernetes Docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/kubernetes-guide/running-on-kubernetes.html)
 
-Each of our images which is released will be tagged with the version of the software they represent. Every release,
-we update the `latest` tag for each image and add a new release which has the corresponding version tag.
+## Versioning
+
+Each of the released images will be tagged with the version of the
+software they represent. Every release we update the `latest` tag for each
+image and add a new release which has the corresponding version tag.
 
 If we release Gaffer version 2.1.2, the following images would be uploaded:
 
@@ -24,17 +33,20 @@ If we release Gaffer version 2.1.2, the following images would be uploaded:
 - gchq/gaffer:2.1.2
 - gchq/gaffer:2.1.2-accumulo-2.0.1
 
-We maintain mutable versions of latest, as well as the major, minor and bugfix versions of Gaffer. For reproducibility
-make sure to use the full version in your build metadata. For `gaffer`/`gaffer-rest` images, we also create a tag including the
-accumulo version, this allows for compatibility with Accumulo 1.9.3 in our tests. The `-accumulo-1.9.3` tagged images
-are not published but can be build locally if required.
+We maintain mutable versions of latest, as well as the major, minor and bugfix
+versions of Gaffer. For reproducibility make sure to use the full version in
+your build metadata. For `gaffer`/`gaffer-rest` images, we also create a tag
+including the accumulo version, this allows for compatibility with Accumulo
+1.9.3 in our tests. The `-accumulo-1.9.3` tagged images are not published but
+can be built locally if required.
 
 The release process is automated by GitHub actions.
 
-# Known Compatible Docker Versions
+## Known Compatible Docker Versions
 
 - 20.10.23
 
-# Contributing
+## Contributing
 
-We welcome contributions to this project. Detailed information on our ways of working can be found in our [developer docs](https://gchq.github.io/gaffer-doc/latest/dev/ways-of-working/).
+We welcome contributions to this project. Detailed information on our ways of
+working can be found in our [developer docs](https://gchq.github.io/gaffer-doc/latest/development-guide/ways-of-working.html).

cd/build_core_images.sh

@@ -26,7 +26,6 @@ pushd "${ROOT_DIR}" || exit 1
 # HADOOP_VERSION
 # GAFFER_VERSION
 # ACCUMULO_VERSION
-# TINKERPOP_VERSION
 if [[ -f "${1}" ]]; then
     source "${1}"
 else

cd/build_images.sh

@@ -28,7 +28,6 @@ pushd "${ROOT_DIR}" || exit 1
 # GAFFERPY_VERSION
 # ACCUMULO_VERSION
 # SPARK_VERSION
-# TINKERPOP_VERSION
 if [[ -f "${1}" ]]; then
     source "${1}"
 else
@@ -42,12 +41,11 @@ docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./doc
 # Builds all of the notebook related images:
 docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook
 docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build
-# Builds the Gaffer Gremlin server
-./docker/gaffer-gremlin/build.sh
 
-# Set $JHUB_OPTIONS_SERVER_VERSION
-source ./docker/gaffer-jhub-options-server/get-version.sh
-# Builds the jhub options server:
-docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build
+# Currently disabled due to npm depencency issue
+# # Set $JHUB_OPTIONS_SERVER_VERSION
+# source ./docker/gaffer-jhub-options-server/get-version.sh
+# # Builds the jhub options server:
+# docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build
 
 popd || exit 1

cd/publish_images.sh

@@ -102,9 +102,9 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}"
-pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}"
 pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}"
-pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}"
+# Currently disabled due to npm depencency issue
+# pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}"
 pushContainer gchq/spark-py "${SPARK_VERSION}"
 
 # Push legacy versions to Container Repositories

cd/update_versions.sh

@@ -31,7 +31,6 @@ source ./docker/gaffer-jhub-options-server/get-version.sh
 find . -type f -exec sed -i "s/GAFFER_VERSION=[0-9]\.[0-9]\.[0-9]/GAFFER_VERSION=${APP_VERSION}/g" {} +
 find . -type f -exec sed -E -i "s/GAFFERPY_VERSION=(gafferpy-)?[0-9]\.[0-9]\.[0-9]/GAFFERPY_VERSION=\1${APP_VERSION}/g" {} +
 find . -type f -exec sed -i "s/GAFFER_TESTER_VERSION=[0-9]\.[0-9]\.[0-9]/GAFFER_TESTER_VERSION=${APP_VERSION}/g" {} +
-sed -i'' -e "s/<gaffer.version>[0-9]\.[0-9]\.[0-9]/<gaffer.version>${APP_VERSION}/g" docker/gaffer-gremlin/pom.xml
 sed -i'' -e "s/BASE_IMAGE_TAG=[0-9]\.[0-9]\.[0-9]/BASE_IMAGE_TAG=${APP_VERSION}/g" docker/gaffer-kerberos/gaffer-krb/Dockerfile
 sed -i'' -e "s/BASE_IMAGE_TAG=[0-9]\.[0-9]\.[0-9]/BASE_IMAGE_TAG=${APP_VERSION}/g" docker/gaffer-kerberos/gaffer-rest-krb/Dockerfile
 

docker/accumulo1.env

@@ -3,9 +3,8 @@ HADOOP_VERSION=3.2.2
 ACCUMULO_VERSION=1.9.3
 HADOOP_CONF_DIR=/etc/hadoop/conf
 ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.3.0
-GAFFER_TESTER_VERSION=2.3.0
-GAFFERPY_VERSION=2.3.0
+GAFFER_VERSION=2.3.1
+GAFFER_TESTER_VERSION=2.3.1
+GAFFERPY_VERSION=2.3.1
 SPARK_VERSION=3.1.2
-TINKERPOP_VERSION=3.7.1
 KUBECTL_VERSION=1.23.0

docker/accumulo2.env

@@ -3,9 +3,8 @@ HADOOP_VERSION=3.3.3
 ACCUMULO_VERSION=2.0.1
 HADOOP_CONF_DIR=/etc/hadoop/conf
 ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.3.0
-GAFFER_TESTER_VERSION=2.3.0
-GAFFERPY_VERSION=2.3.0
+GAFFER_VERSION=2.3.1
+GAFFER_TESTER_VERSION=2.3.1
+GAFFERPY_VERSION=2.3.1
 SPARK_VERSION=3.1.2
-TINKERPOP_VERSION=3.7.1
 KUBECTL_VERSION=1.23.0

docker/example-deployments/README.md

@@ -0,0 +1,25 @@
+# Example Gaffer Deployments
+
+These directories have basic examples of deploying containerised Gaffer with
+different store backings. Some example notebooks are also available which
+primarily feature the use of the Gremlin interface for interacting with the
+Graph.
+
+All examples will provide a Gaffer REST API to start interacting with the
+deployed graph.
+
+## Modern Example (Accumulo)
+
+The modern example, which uses the [Tinkerpop modern dataset](https://tinkerpop.apache.org/docs/current/tutorials/the-gremlin-console/#toy-graphs),
+can be found under the `modern-example` directory which features an Accumulo
+store as its storage backing.
+
+## Federation Example
+
+A demo/example of using Gaffer REST with a Federated Store is available under
+the `federated-example` directory.
+
+## Proxy Example
+
+A demo/example of using Gaffer REST with a Proxy Store is available under the
+`proxy-example` directory.

docker/gaffer-gremlin/example/notebooks/graphnotebook-modern-example.ipynb → docker/example-deployments/example-notebooks/graphnotebook-modern-example.ipynb

@@ -38,7 +38,7 @@
     "%%graph_notebook_config\n",
     "{\n",
     "  \"host\": \"localhost\",\n",
-    "  \"port\": 8182,\n",
+    "  \"port\": 8080,\n",
     "  \"ssl\": false,\n",
     "  \"gremlin\": {\n",
     "    \"traversal_source\": \"g\",\n",

docker/gaffer-gremlin/example/notebooks/gremlinpython-modern-example.ipynb → docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb

@@ -21,10 +21,10 @@
     "# Common imports\n",
     "from gremlin_python.process.anonymous_traversal import traversal\n",
     "from gremlin_python.process.graph_traversal import __\n",
-    "from gremlin_python.process.strategies import *\n",
-    "from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection\n",
     "from gremlin_python.process.traversal import *\n",
+    "from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection\n",
     "from gremlin_python.driver.client import Client\n",
+    "from gremlin_python.driver.serializer import GraphSONSerializersV3d0\n",
     "import nest_asyncio\n",
     "\n",
     "from_ = Direction.OUT\n",
@@ -45,7 +45,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "g = traversal().with_remote(DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))"
+    "# Traversal root\n",
+    "g = traversal().with_remote(DriverRemoteConnection('ws://localhost:8080/gremlin', 'g', message_serializer=GraphSONSerializersV3d0()))\n",
+    "\n",
+    "# Client connection\n",
+    "client = Client('ws://localhost:8080/gremlin', 'g', message_serializer=GraphSONSerializersV3d0())"
    ]
   },
   {
@@ -77,7 +81,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "g.V('1', '2').hasLabel('person').outE('knows').values().is_(lt(1)).to_list()"
+    "g.V('1', '2').hasLabel('person').outE('knows').values().is_(P.lt(1)).to_list()"
    ]
   },
   {
@@ -96,13 +100,21 @@
     "g.V('1').repeat(__.both().simplePath()).until(__.has('name', 'lop')).limit(3).path().by(__.valueMap()).to_list()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Run a seeded cypher query via Gremlin. Submitted as a script rather than via gremlin-python bindings"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "g.V('1').outE().as_(\"a\").V('1').values(\"name\").to_list()"
+    "results =  client.submit(\"g.with(\\\"cypher\\\", \\\"MATCH (p:person) WHERE ID(p) = '1' RETURN p\\\").call()\")\n",
+    "results.all().result()"
    ]
   }
  ],

docker/gaffer-rest/federated-example/compose.yaml → docker/example-deployments/federated-example/compose.yaml

@@ -20,7 +20,8 @@ services:
     ports:
     - 8080:8080
     volumes:
-    - ./config/store/store.properties:/gaffer/store/store.properties:ro
-    - ./config/store/mergeFunctions.json:/gaffer/store/mergeFunctions.json:ro
-    - ./config/store/defaultGraphIds.json:/gaffer/store/defaultGraphIds.json:ro
-    - ./config/schema:/gaffer/schema:ro
+    - ./conf/gaffer/schema:/gaffer/schema:ro
+    - ./conf/gaffer/store/store.properties:/gaffer/store/store.properties:ro
+    - ./conf/gaffer/store/mergeFunctions.json:/gaffer/store/mergeFunctions.json:ro
+    - ./conf/gaffer/store/defaultGraphIds.json:/gaffer/store/defaultGraphIds.json:ro
+    - ./conf/gaffer/gafferpop.properties:/gaffer/gafferpop.properties:ro

docker/gaffer-gremlin/compose.yaml → docker/example-deployments/federated-example/conf/gaffer/gafferpop.properties

@@ -1,3 +1,4 @@
+#
 # Copyright 2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,15 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-version: "3.7"
-
-services:
-  gaffer-gremlin:
-    image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}
-    build:
-      context: .
-      args:
-        BASE_IMAGE_TAG: ${TINKERPOP_VERSION}
-    ports:
-    - 8182:8182
+#
+# Default operation config
+gaffer.elements.getalllimit=5000
+gaffer.elements.hasstepfilterstage=PRE_AGGREGATION

docker/example-deployments/modern-example/README.md

@@ -0,0 +1,26 @@
+# Gaffer Modern Example
+
+This is an example deployment of Gaffer on Accumulo. This uses a REST API
+deployment of Gaffer with full Accumulo and Hadoop storage backing.
+
+The example uses the
+[Tinkerpop Modern Graph](https://tinkerpop.apache.org/docs/current/tutorials/the-gremlin-console/#toy-graphs)
+as its data set to demonstrate basic querying of data. The data can be queried
+via the REST API using standard Gaffer operations or by using the Gremlin
+websocket (see [the docs](https://gchq.github.io/gaffer-doc/latest/user-guide/query/gremlin/gremlin.html)
+for more info).
+
+## Running the Example
+
+To run the example please use the provided start script with an environment file
+to specify the image versions e.g. `accumulo2.env`:
+
+```bash
+./create-deployment.sh ../../accumulo2.env
+```
+
+With the demo deployment a standard Gaffer REST API will be started (available
+on port 8080 by default). Basic configuration for Accumulo and Hadoop have been
+added but can be modified to test out different
+scenarios, please see the [documentation](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-stores/accumulo-store.html)
+for more information.

docker/gaffer-gremlin/example/compose.yaml → docker/example-deployments/modern-example/compose.yaml

@@ -173,16 +173,5 @@ services:
     volumes:
       - ./conf/gaffer/schema:/gaffer/schema:ro
       - ./conf/gaffer/store-accumulo.properties:/gaffer/store/store.properties:ro
+      - ./conf/gaffer/gafferpop.properties:/gaffer/gafferpop.properties:ro
 
-  # Gremlin server
-  gaffer-gremlin:
-    image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}
-    depends_on:
-      accumulo-tserver:
-        condition: service_healthy
-    ports:
-    - 8182:8182
-    volumes:
-    - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro
-    - ./conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro
-    - ./conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro

docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties → docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties

@@ -1,5 +1,4 @@
-#
-# Copyright 2023-2024 Crown Copyright
+# Copyright 2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
-gaffer.graphId=graph1
-gaffer.storeproperties=conf/gaffer/store.properties
-gaffer.schemas=conf/gaffer/schema/
-gaffer.userId=user01
+# Default operation config
+gaffer.elements.getalllimit=5000
+gaffer.elements.hasstepfilterstage=PRE_AGGREGATION

docker/example-deployments/modern-example/conf/gaffer/schema/elements.json

@@ -0,0 +1,49 @@
+{
+  "entities": {
+      "id": {
+          "vertex": "string.id"
+      },
+      "person": {
+          "vertex": "string.id",
+          "properties": {
+              "name": "property.string",
+              "age": "property.integer"
+          }
+      },
+      "software": {
+          "vertex": "string.id",
+          "properties": {
+              "name": "property.string",
+              "lang": "property.string"
+          }
+      }
+  },
+  "edges": {
+      "knows": {
+          "source": "string.id",
+          "destination": "string.id",
+          "directed": "true",
+          "properties": {
+              "weight": "property.double"
+          }
+      },
+      "created": {
+          "source": "string.id",
+          "destination": "string.id",
+          "directed": "true",
+          "properties": {
+              "weight": "property.double"
+          }
+      },
+      "dependsOn": {
+          "source": "string.id",
+          "destination": "string.id",
+          "directed": "true"
+      },
+      "encapsulates": {
+          "source": "string.id",
+          "destination": "string.id",
+          "directed": "true"
+      }
+  }
+}