From 2aedf902f454358135ef37183b9bce738856fc34 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 17 Jul 2024 09:29:09 -0400 Subject: [PATCH] Migration helpers for inet:web -> inet:service --- synapse/lib/stormlib/gen.py | 36 ++++++++++++++++++++++++++ synapse/lib/stormlib/model.py | 48 +++++++++++++++++++++++++++++++++++ synapse/models/inet.py | 13 ++++++++-- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/synapse/lib/stormlib/gen.py b/synapse/lib/stormlib/gen.py index 358f82c1f2..e18db6c84c 100644 --- a/synapse/lib/stormlib/gen.py +++ b/synapse/lib/stormlib/gen.py @@ -136,6 +136,21 @@ class LibGen(s_stormtypes.Lib): {'name': 'name', 'type': 'str', 'desc': 'The name of the place.'}, ), 'returns': {'type': 'node', 'desc': 'A geo:place node with the given name.'}}}, + + {'name': 'inetServicePlatformByFqdn', 'desc': 'Generates an inet:service:platform by FQDN.', + 'type': {'type': 'function', '_funcname': '_storm_query', + 'args': ( + {'name': 'fqdn', 'type': 'str', 'desc': 'The FQDN used to identify the platform.'}, + ), + 'returns': {'type': 'node', 'desc': 'An inet:servce:platform node.'}}}, + + {'name': 'inetServiceAccountByFqdnUser', 'desc': 'Generates an inet:service:account from an FQDN and username.', + 'type': {'type': 'function', '_funcname': '_storm_query', + 'args': ( + {'name': 'fqdn', 'type': 'str', 'desc': 'The FQDN used to identify a platform.'}, + {'name': 'user', 'type': 'str', 'desc': 'The user name which identifies the platform user.'}, + ), + 'returns': {'type': 'node', 'desc': 'An inet:servce:account node.'}}}, ) _storm_lib_path = ('gen',) @@ -425,6 +440,27 @@ class LibGen(s_stormtypes.Lib): [ geo:place=(gen, name, $geoname) :name=$geoname ] return($node) } + + function inetServicePlatformByFqdn(fqdn) { + $fqdn = $lib.cast(inet:fqdn, $fqdn) + + inet:fqdn=$fqdn -> inet:servce:platform:fqdns + return($node) + + [ inet:service:platform=(byfqdn, $fqdn) :fqdns+=$fqdn ] + return($node) + } + + function inetServiceAccountByFqdnUser(fqdn, user) { + $user = $lib.cast(inet:user, $user) + $platform = $inetServicePlatformByFqdn($fqdn) + + inet:service:account:user=$user +:platform=$platform + return($node) + + [ inet:service:account=($platform, $user) :platform=$platform :user=$user ] + return($node) + } ''' stormcmds = ( diff --git a/synapse/lib/stormlib/model.py b/synapse/lib/stormlib/model.py index 0eab3b9047..6455f918be 100644 --- a/synapse/lib/stormlib/model.py +++ b/synapse/lib/stormlib/model.py @@ -919,8 +919,56 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin): 'desc': 'Do not copy nodedata to the risk:vulnerable node.'}, ), 'returns': {'type': 'list', 'desc': 'A list of idens for the risk:vulnerable nodes.'}}}, + + {'name': 'inetWebToService', 'desc': ''' + Generate inet:service model elements from existing inet:web nodes. + + The following forms are migrated: + inet:web:acct -> inet:service:account + inet:web:post -> inet:service:message + inet:web:mesg -> inet:service:message + inet:web:group -> inet:service:group + inet:web:logon -> inet:service:login + inet:web:channel -> inet:service:channel + inet:web:instance -> inet:service:instance + inet:web:member -> inet:service:group:member / inet:service:channel:member + inet:web:attachment -> inet:service:message:attachment + inet:web:post:link -> inet:service:message:link + ''', + 'type': {'type': 'function', '_funcname': 'stormcode', 'args': (), 'returns': {'type': 'null'}}, ) _storm_lib_path = ('model', 'migration', 's') + _storm_query = ''' + function __webAcctToService() { + inet:web:acct + $acct = $lib.gen.inetServiceAccountByFqdnUser(:fqdn, :user) + $lib.model.migration.copyData($node, $acct) + $lib.model.migration.copyTags($node, $acct) + $lib.model.migration.copyEdges($node, $acct) + fini { return() } + } + function __webPostToService(acct, post) { + inet:web:post + + $iden = $node.repr() + $props = $node.props + + -> {[ inet:service:message=$iden ]} + if ($props.acct) { + [ :account=$lib.gen.inetServiceByFqdnUser($props.acct.0, $props.acct.1) ] + } + + if $props.url {[ :url=$props.url ]} + if $props.text {[ :text=$props.text ]} + if $props.time {[ :period=($props.time, ?) ]} + if $props.client {[ :client=$props.client ]} + if $props.deleted {[ :status=removed ]} + } + function inetWebToService() { + $__webAcctToService() + $__webPostToService() + } + ''' def getObjLocals(self): return { diff --git a/synapse/models/inet.py b/synapse/models/inet.py index c48763f38d..319daf9ec3 100644 --- a/synapse/models/inet.py +++ b/synapse/models/inet.py @@ -2550,8 +2550,11 @@ def getModelDefs(self): 'doc': 'The time the web search was issued.', }), ('acct', ('inet:web:acct', {}), { - 'doc': 'The account that the query was issued as.', - }), + 'doc': 'The account that the query was issued as.'}), + + ('account', ('inet:service:account', {}), { + 'doc': 'The service account which ran the query.'}), + ('host', ('it:host', {}), { 'doc': 'The host that issued the query.', }), @@ -3510,6 +3513,9 @@ def getModelDefs(self): 'ex': 'https://twitter.com', 'doc': 'The primary URL of the platform.'}), + ('fqdns', ('array', {'type': 'inet:fqdn', 'uniq': True, 'sorted': True}), { + 'doc': 'An array of FQDNs used by the platform.'}), + ('name', ('str', {'onespace': True, 'lower': True}), { 'ex': 'twitter', 'doc': 'A friendly name for the platform.'}), @@ -3538,6 +3544,9 @@ def getModelDefs(self): 'ex': 'https://v.vtx.lk/slack', 'doc': 'The primary URL which identifies the service instance.'}), + ('fqdns', ('array', {'type': 'inet:fqdn', 'uniq': True, 'sorted': True}), { + 'doc': 'An array of FQDNs used by the instance.'}), + ('name', ('str', {'lower': True, 'onespace': True}), { 'ex': 'synapse users slack', 'doc': 'The name of the service instance.'}),