WIP: Synapse 3.0.0 (SYN-197)

.bumpversion.cfg

@@ -1,8 +1,12 @@
 [bumpversion]
-current_version = 2.187.0
+current_version = 3.0.0
 commit = True
 tag = True
 tag_message = 
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(a(?P<alpha>\d+))?
+serialize = 
+	{major}.{minor}.{patch}a{alpha}
+	{major}.{minor}.{patch}
 
 [bumpversion:file:pyproject.toml]
 search = version = '{current_version}'

.circleci/config.yml

@@ -252,6 +252,85 @@ commands:
             . venv/bin/activate
             twine upload dist/*
 
+  deploy_to_devpi:
+    description: "Publish wheel packages to devpi"
+    parameters:
+      devpi_base_url:
+        description: Base URL for the devpi service
+        type: string
+      devpi_index:
+        description: Index to push packages to and install requirements from.
+        type: string
+      python_tag:
+        description: Python tag to use when building wheels
+        type: string
+      tag_verify:
+        description: Check if the git tag matches the package version.
+        type: boolean
+        default: true
+      pip_install_args:
+        description: Additional args to provide to pip install when installing requirements.
+        type: string
+        default: ''
+      python_smoke_code:
+        description: Code used to smoke test the package before publication.
+        type: string
+    steps:
+      - restore_cache:
+          keys:
+            - v4-venv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }}
+
+      - run:
+          name: install python dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            python3 -m pip install -U wheel pip
+            python3 -m pip install -U -r requirements.txt --index-url="<< parameters.devpi_base_url >><< parameters.devpi_index >>/+simple" << parameters.pip_install_args >>
+            python3 -m pip install -U twine build
+
+      - save_cache:
+          paths:
+            - ./venv
+          key: v4-venv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }}
+
+      - when:
+          condition: << parameters.tag_verify >>
+          steps:
+            - run:
+                name: verify git tag vs. version
+                command: |
+                  . venv/bin/activate
+                  python ./scripts/verify_version.py
+
+      - run:
+          name: create packages
+          command: |
+            . venv/bin/activate
+            python3 scripts/replace_commit.py
+            export DIST_EXTRA_CONFIG=/tmp/build-opts.cfg
+            echo -e "[bdist_wheel]\npython_tag=<< parameters.python_tag >>" > $DIST_EXTRA_CONFIG
+            python -m build --wheel
+
+      - run:
+          name: smoke packages
+          command: |
+            mkdir -p /tmp/wheeltest
+            python3 -m venv /tmp/wheeltest/venv
+            cp dist/*.whl /tmp/wheeltest
+            cd /tmp/wheeltest
+            . ./venv/bin/activate
+            python3 -m pip install -U wheel pip twine
+            python3 -m twine check *.whl
+            python3 -m pip install *.whl
+            python3 -c "<< parameters.python_smoke_code >>"
+            deactivate
+
+      - run:
+          name: upload to devpi
+          command: |
+            . venv/bin/activate
+            twine upload --repository-url "<< parameters.devpi_base_url >><< parameters.devpi_index >>" dist/*
 
   do_docker_prep:
     description: "Install packages and set commit."
@@ -528,6 +607,26 @@ jobs:
           registry: ${DOCKER_PRIMARY_REGISTRY}
           secondaryregistry: ${DOCKER_SECONDARY_REGISTRY}
 
+  devpi_synapse_3xx:
+    docker:
+      - image: cimg/python:3.11
+    steps:
+      - checkout
+      - run:
+          name: Version bump and date the build.
+          command: |
+            python3 -m pip install bump2version
+            DATE=`date '+%Y%m%d%H%M'`
+            CURRENT_VERSION=`grep "current_version =" .bumpversion.cfg | cut -f 2 -d "=" | tr -d '[:space:]'`
+            # The "patch" value is used here just to fulfill a required argument.
+            python3 -m bumpversion --allow-dirty --no-tag --no-commit --new-version "${CURRENT_VERSION}a${DATE}" patch
+      - deploy_to_devpi:
+          devpi_base_url: ${DEVPI_BASE_URL}
+          devpi_index: vertex/synapse-3xx
+          python_tag: py311
+          tag_verify: false
+          python_smoke_code: import synapse; print(synapse.version)
+
 workflows:
   version: 2
   run_tests:
@@ -576,6 +675,21 @@ workflows:
             branches:
               ignore: /.*/
 
+      - devpi_synapse_3xx:
+          context:
+            - Devpi00Access
+          requires:
+            - doctests
+            - python311
+            - python311_replay
+            - python_package_smoketest
+          filters:
+            tags:
+              ignore: /.*/
+            branches:
+              only:
+                - synapse-3xx
+
       - gh-release/dorelease:
           requires:
             - doctests

docs/docdata/foopkg/foopkg.yml

@@ -1,6 +1,6 @@
 name: foopkg
 version: 1.0.0
-synapse_version: '>=2.144.0,<3.0.0'
+synapse_version: '>=3.0.0,<4.0.0'
 
 onload: $lib.import(foomod).onload()
 

docs/synapse/adminguide.rstorm

@@ -436,12 +436,12 @@ analysts may be allowed to apply tags representing certain assessments (such as
 |                                                                 |                                 |
 | node data)                                                      |                                 |
 +-----------------------------------------------------------------+---------------------------------+
-| **Only** add ``inet:ipv4`` nodes                                |                                 |
-|                                                                 | ``node.add.inet:ipv4``          |
+| **Only** add ``inet:ip`` nodes                                |                                 |
+|                                                                 | ``node.add.inet:ip``          |
 | (but not set properties, or work with tags or edges)            |                                 |
 +-----------------------------------------------------------------+---------------------------------+
-| **Only** add (set) the ``:asn`` property of ``inet:ipv4`` nodes |                                 |
-|                                                                 | ``node.prop.set.inet:ipv4:asn`` |
+| **Only** add (set) the ``:asn`` property of ``inet:ip`` nodes |                                 |
+|                                                                 | ``node.prop.set.inet:ip:asn`` |
 | (but not create nodes or work with other properties, tags,      |                                 |
 |                                                                 |                                 |
 | edges, etc.)                                                    |                                 |

docs/synapse/datamodel_deprecation.rst

@@ -68,9 +68,9 @@ Types
 - `ps:persona`
     - Please use the `ps:person` or `ps:contact` types.
 - `ps:person:has`
-    - Please use `edge:has` or a light edge.
+    - Please use a light edge.
 - `ps:persona:has`
-    - Please use `ps:person` or `ps:context` in combination with an `edge:has` or a light edge.
+    - Please use `ps:person` or `ps:context` in combination with a light edge.
 - `inet:ssl:cert`
     - `inet:ssl:cert` is deprecated in favor of `inet:tls:servercert` and `inet:tls:clientcert`.
 
@@ -115,7 +115,7 @@ Properties
     - `:naics` and `:sic`
         - The `ou:org:naics` and `ou:org:sic` properties has been collected into the `ou:org:industries` property, which is an array of type `ou:industry`.
     - `:has`
-        - Please use an `edge:has` node or a light edge.
+        - Please use a light edge.
 
 - `risk:attack`
     - `:actor:org`

docs/synapse/devguides/architecture.rst

@@ -13,9 +13,6 @@ such as the implementations of the Cortex, Axon, Cryotank, as well as the Telepa
 There are also a set of common helper functions (common.py_) and exceptions (exc.py_). There are several submodules
 available as well:
 
-synapse.cmds
-  Command implementations for the Cmdr CLI tool
-
 synapse.data
   Data files stored in the library.
 
@@ -78,9 +75,8 @@ The ``Cell`` (cell.py_) is a ``Base`` implementation which has several component
 - It is a ``Base``, so it benefits from all the components a ``Base`` has.
 - It contains support for configuration directives at start time, so a cell can have well defined configuration
   options availble to it.
-- It has persistent storage available via two different mechanisms, a LMDB slab for arbitrary data that is local to the
-  cell, and a ``Hive`` for key-value data storage that can be remotely read and written.
-- It handles user authentication and authorization via user data stored in the ``Hive``.
+- It has persistent storage available via a LMDB slab for arbitrary data that is local to the cell.
+- It handles user authentication and authorization via user data stored in the slab.
 - The ``Cell`` is Telepath aware, and will start his own ``Daemon`` that allows remote access.  By default, the ``Cell``
   has a PF Unix socket available for access, so local telepath access is trivial.
 - Since the ``Cell`` is Telepath aware, there is a base ``CellApi`` that implements his RPC routines.  ``Cell``

docs/synapse/devguides/power-ups.rst

@@ -37,10 +37,7 @@ that gets processed and loaded into your Cortex.
     name: acme-hello
     version: 0.0.1
 
-    synapse_version: '>=2.144.0,<3.0.0'
-
-    genopts:
-        dotstorm: true # Specify that storm command/module files end with ".storm"
+    synapse_version: '>=3.0.0,<4.0.0'
 
     author:
       url: https://acme.newp
@@ -383,10 +380,10 @@ To define **Optic** actions, you declare them in the **Storm Package** YAML file
           - name: Hello Omgopts
             storm: acme.hello.omgopts --debug
             descr: This description is displayed as the tooltip in the menu
-            forms: [ inet:ipv4, inet:fqdn ]
+            forms: [ inet:ip, inet:fqdn ]
 
 By specifying the ``forms:`` key, you can control which node actions will be presented on different forms. For example,
-if you are writing a DNS power-up, you may want to limit the specified actions to ``inet:ipv4``, ``inet:ipv6``, and ``inet:fqdn``
+if you are writing a DNS power-up, you may want to limit the specified actions to ``inet:ip`` and ``inet:fqdn``
 nodes.
 
 When selected, the query specified in the ``storm:`` key will be run with the currently selected nodes as input. For example,