diff --git a/Extractor/Config/ExtractionConfig.cs b/Extractor/Config/ExtractionConfig.cs index 8bcce0c40..d55ba5c9d 100644 --- a/Extractor/Config/ExtractionConfig.cs +++ b/Extractor/Config/ExtractionConfig.cs @@ -15,12 +15,21 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +using Cognite.Extensions.DataModels.QueryBuilder; using Cognite.Extractor.Common; +using Cognite.OpcUa.Nodes; using Cognite.OpcUa.NodeSources; using Microsoft.Extensions.Logging; using Opc.Ua; +using Serilog.Debugging; +using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; using YamlDotNet.Serialization; namespace Cognite.OpcUa.Config @@ -386,20 +395,157 @@ public bool NamespacePublicationDate public List GetTargets => ToBeSubscribed; } - public class RawNodeFilter + public interface IFieldFilter + { + bool IsMatch(string raw); + } + + public class RegexFieldFilter : IFieldFilter + { + private readonly Regex filter; + + public RegexFieldFilter(string regex) + { + filter = new Regex(regex, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant); + } + public bool IsMatch(string raw) + { + return filter.IsMatch(raw); + } + + public string Raw => filter.ToString(); + + public override string ToString() + { + return Raw; + } + } + + public class ListFieldFilter : IFieldFilter + { + private readonly HashSet entries; + public string? OriginalFile { get; } + + public ListFieldFilter(IEnumerable items, string? originalFile) + { + entries = new HashSet(items); + OriginalFile = originalFile; + } + + public bool IsMatch(string raw) + { + return entries.Contains(raw); + } + + public IEnumerable Raw => entries; + + public override string ToString() + { + var builder = new StringBuilder(); + builder.Append("Any of: ["); + var first = true; + foreach (var entry in entries) + { + if (!first) + { + builder.Append(", "); + } + first = false; + builder.AppendFormat("\"{0}\"", entry); + } + builder.Append("]"); + return base.ToString(); + } + } + + public class FieldFilterConverter : IYamlTypeConverter + { + public bool Accepts(Type type) + { + return typeof(IFieldFilter).IsAssignableFrom(type); + } + + public object? ReadYaml(IParser parser, Type type) + { + if (parser.TryConsume(out var scalar)) + { + return new RegexFieldFilter(scalar.Value); + } + if (parser.TryConsume(out _)) + { + var items = new List(); + while (!parser.Accept(out _)) + { + var seqScalar = parser.Consume(); + items.Add(seqScalar.Value); + } + + parser.Consume(); + + return new ListFieldFilter(items, null); + } + if (parser.TryConsume(out _)) + { + var key = parser.Consume(); + if (key.Value != "file") + { + throw new YamlException("Expected object containing \"file\""); + } + var value = parser.Consume(); + var lines = File.ReadAllLines(value.Value); + parser.Consume(); + return new ListFieldFilter(lines.Where(line => !string.IsNullOrWhiteSpace(line)), value.Value); + } + + throw new YamlException("Expected a string, object, or list of strings"); + } + + public void WriteYaml(IEmitter emitter, object? value, Type type) + { + if (value is RegexFieldFilter regexFilter) + { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, regexFilter.Raw, ScalarStyle.DoubleQuoted, false, true)); + } + else if (value is ListFieldFilter listFilter) + { + if (listFilter.OriginalFile != null) + { + emitter.Emit(new MappingStart()); + emitter.Emit(new Scalar("file")); + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, listFilter.OriginalFile, ScalarStyle.DoubleQuoted, false, true)); + emitter.Emit(new MappingEnd()); + } + else + { + emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true, SequenceStyle.Block, Mark.Empty, Mark.Empty)); + foreach (var entry in listFilter.Raw) + { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, entry, ScalarStyle.DoubleQuoted, false, true)); + } + emitter.Emit(new SequenceEnd()); + } + } + else + { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, "", ScalarStyle.DoubleQuoted, false, true)); + } + } + } + + public class NodeFilter { /// /// Regex on node DisplayName. /// - public string? Name { get; set; } + public IFieldFilter? Name { get; set; } /// /// Regex on node Description. /// - public string? Description { get; set; } + public IFieldFilter? Description { get; set; } /// /// Regex on node id. Ids on the form "i=123" or "s=string" are matched. /// - public string? Id { get; set; } + public IFieldFilter? Id { get; set; } /// /// Whether the node is an array. If this is set, the filter only matches varables. /// @@ -407,11 +553,11 @@ public class RawNodeFilter /// /// Regex on the full namespace of the node id. /// - public string? Namespace { get; set; } + public IFieldFilter? Namespace { get; set; } /// /// Regex on the id of the type definition. On the form "i=123" or "s=string". /// - public string? TypeDefinition { get; set; } + public IFieldFilter? TypeDefinition { get; set; } /// /// The "historizing" attribute on variables. If this is set, the filter only matches variables. /// @@ -424,7 +570,165 @@ public class RawNodeFilter /// /// Another instance of NodeFilter which is applied to the parent node. /// - public RawNodeFilter? Parent { get; set; } + public NodeFilter? Parent { get; set; } + + /// + /// Return a representation if the identifier of , + /// on the form i=123, or s=string, etc. + /// + /// Identifier to get representation of + /// String representation of identifier of + private static string GetIdString(NodeId id) + { + var builder = new StringBuilder(); + NodeId.Format(builder, id.Identifier, id.IdType, 0); + return builder.ToString(); + } + + /// + /// Test for match using only basic properties available in when reading from the server. + /// Will always return false if there are filters on not yet available fields. + /// + /// DisplayName + /// Raw NodeId + /// TypeDefinition Id + /// Source namespacetable + /// NodeClass + /// True if match + public bool IsBasicMatch(string name, NodeId id, NodeId typeDefinition, NamespaceTable namespaces, NodeClass nc) + { + if (Description != null || IsArray != null || Parent != null || Historizing != null) return false; + return MatchBasic(name, id ?? NodeId.Null, typeDefinition, namespaces, nc); + } + + /// + /// Test for match using only basic properties available in when reading from the server. + /// + /// DisplayName + /// Raw NodeId + /// TypeDefinition Id + /// Source namespacetable + /// NodeClass + /// True if match + private bool MatchBasic(string? name, NodeId id, NodeId? typeDefinition, NamespaceTable namespaces, NodeClass nc) + { + if (Name != null && (string.IsNullOrEmpty(name) || !Name.IsMatch(name))) return false; + if (Id != null) + { + if (id == null || id.IsNullNodeId) return false; + var idstr = GetIdString(id); + if (!Id.IsMatch(idstr)) return false; + } + if (Namespace != null && namespaces != null) + { + var ns = namespaces.GetString(id.NamespaceIndex); + if (string.IsNullOrEmpty(ns)) return false; + if (!Namespace.IsMatch(ns)) return false; + } + if (TypeDefinition != null) + { + if (typeDefinition == null || typeDefinition.IsNullNodeId) return false; + var tdStr = GetIdString(typeDefinition); + if (!TypeDefinition.IsMatch(tdStr)) return false; + } + if (NodeClass != null) + { + if (nc != NodeClass.Value) return false; + } + return true; + } + /// + /// Return true if the given node matches the filter. + /// + /// Node to test + /// Currently active namespace table + /// True if match + public bool IsMatch(BaseUANode node, NamespaceTable ns) + { + if (node == null || !MatchBasic(node.Name, node.Id, node.TypeDefinition, ns, node.NodeClass)) return false; + if (Description != null && (string.IsNullOrEmpty(node.Attributes.Description) || !Description.IsMatch(node.Attributes.Description))) return false; + if (node is UAVariable variable) + { + if (IsArray != null && variable.IsArray != IsArray) return false; + if (Historizing != null && variable.FullAttributes.Historizing != Historizing) return false; + } + else if (IsArray != null || Historizing != null) return false; + if (Parent != null && (node.Parent == null || !Parent.IsMatch(node.Parent, ns))) return false; + return true; + } + + public bool IsBasic => Description == null && IsArray == null && Parent == null && Historizing == null; + + /// + /// Create string representation, for logging. + /// + /// StringBuilder to write to + /// Level of nesting, for clean indentation. + public void Format(StringBuilder builder, int idx) + { + if (Name != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("Name: {0}", Name); + builder.AppendLine(); + } + if (Description != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("Description: {0}", Description); + builder.AppendLine(); + } + if (Id != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("Id: {0}", Id); + builder.AppendLine(); + } + if (IsArray != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("IsArray: {0}", IsArray); + builder.AppendLine(); + } + if (Historizing != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("Historizing: {0}", Historizing); + builder.AppendLine(); + } + if (Namespace != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("Namespace: {0}", Namespace); + builder.AppendLine(); + } + if (TypeDefinition != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("TypeDefinition: {0}", TypeDefinition); + builder.AppendLine(); + } + if (NodeClass != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.AppendFormat("NodeClass: {0}", NodeClass); + builder.AppendLine(); + } + if (Parent != null) + { + builder.Append(' ', (idx + 1) * 4); + builder.Append("Parent:"); + builder.AppendLine(); + Parent.Format(builder, idx + 1); + } + } + + public override string ToString() + { + var builder = new StringBuilder(); + Format(builder, 0); + return builder.ToString(); + } } public class RawNodeTransformation { @@ -435,7 +739,7 @@ public class RawNodeTransformation /// /// NodeFilter. All non-null filters must match each node for the transformation to be applied. /// - public RawNodeFilter? Filter { get; set; } + public NodeFilter? Filter { get; set; } } public enum StatusCodeMode diff --git a/Extractor/Extractor.csproj b/Extractor/Extractor.csproj index 82c5ab6f1..093aecd2e 100644 --- a/Extractor/Extractor.csproj +++ b/Extractor/Extractor.csproj @@ -13,7 +13,7 @@ - + diff --git a/Extractor/NodeTransformation.cs b/Extractor/NodeTransformation.cs index 6e2cd957d..b42d15df6 100644 --- a/Extractor/NodeTransformation.cs +++ b/Extractor/NodeTransformation.cs @@ -19,216 +19,11 @@ You should have received a copy of the GNU General Public License using Cognite.OpcUa.Nodes; using Microsoft.Extensions.Logging; using Opc.Ua; -using Serilog.Debugging; -using System; using System.Collections.Generic; using System.Text; -using System.Text.RegularExpressions; namespace Cognite.OpcUa { - /// - /// Class used to apply a complex filter to nodes. - /// - public class NodeFilter - { - private Regex? Name { get; } - private Regex? Description { get; } - private Regex? Id { get; } - private bool? IsArray { get; } - private NodeClass? NodeClass { get; } - private Regex? Namespace { get; } - private Regex? TypeDefinition { get; } - private NodeFilter? Parent { get; } - private bool? Historizing { get; } - public NodeFilter(RawNodeFilter? filter) - { - // Filter with no elements applies to everything, which may be bizarre, but that's on the user. - if (filter == null) return; - Name = CreateRegex(filter.Name); - Description = CreateRegex(filter.Description); - Id = CreateRegex(filter.Id); - Namespace = CreateRegex(filter.Namespace); - TypeDefinition = CreateRegex(filter.TypeDefinition); - IsArray = filter.IsArray; - NodeClass = filter.NodeClass; - Historizing = filter.Historizing; - if (filter.Parent != null) - { - Parent = new NodeFilter(filter.Parent); - } - } - - /// - /// Return a representation if the identifier of , - /// on the form i=123, or s=string, etc. - /// - /// Identifier to get representation of - /// String representation of identifier of - private static string GetIdString(NodeId id) - { - var builder = new StringBuilder(); - NodeId.Format(builder, id.Identifier, id.IdType, 0); - return builder.ToString(); - } - - /// - /// Create regex from configured string. - /// - /// Raw string to create regex for. - /// Created regex. - private static Regex? CreateRegex(string? raw) - { - if (string.IsNullOrEmpty(raw)) return null; - - return new Regex(raw, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant); - } - - /// - /// Test for match using only basic properties available in when reading from the server. - /// Will always return false if there are filters on not yet available fields. - /// - /// DisplayName - /// Raw NodeId - /// TypeDefinition Id - /// Source namespacetable - /// NodeClass - /// True if match - public bool IsBasicMatch(string name, NodeId id, NodeId typeDefinition, NamespaceTable namespaces, NodeClass nc) - { - if (Description != null || IsArray != null || Parent != null || Historizing != null) return false; - return MatchBasic(name, id ?? NodeId.Null, typeDefinition, namespaces, nc); - } - - public bool IsBasic => Description == null && IsArray == null && Parent == null && Historizing == null; - - /// - /// Test for match using only basic properties available in when reading from the server. - /// - /// DisplayName - /// Raw NodeId - /// TypeDefinition Id - /// Source namespacetable - /// NodeClass - /// True if match - private bool MatchBasic(string? name, NodeId id, NodeId? typeDefinition, NamespaceTable namespaces, NodeClass nc) - { - if (Name != null && (string.IsNullOrEmpty(name) || !Name.IsMatch(name))) return false; - if (Id != null) - { - if (id == null || id.IsNullNodeId) return false; - var idstr = GetIdString(id); - if (!Id.IsMatch(idstr)) return false; - } - if (Namespace != null && namespaces != null) - { - var ns = namespaces.GetString(id.NamespaceIndex); - if (string.IsNullOrEmpty(ns)) return false; - if (!Namespace.IsMatch(ns)) return false; - } - if (TypeDefinition != null) - { - if (typeDefinition == null || typeDefinition.IsNullNodeId) return false; - var tdStr = GetIdString(typeDefinition); - if (!TypeDefinition.IsMatch(tdStr)) return false; - } - if (NodeClass != null) - { - if (nc != NodeClass.Value) return false; - } - return true; - } - /// - /// Return true if the given node matches the filter. - /// - /// Node to test - /// Currently active namespace table - /// True if match - public bool IsMatch(BaseUANode node, NamespaceTable ns) - { - if (node == null || !MatchBasic(node.Name, node.Id, node.TypeDefinition, ns, node.NodeClass)) return false; - if (Description != null && (string.IsNullOrEmpty(node.Attributes.Description) || !Description.IsMatch(node.Attributes.Description))) return false; - if (node is UAVariable variable) - { - if (IsArray != null && variable.IsArray != IsArray) return false; - if (Historizing != null && variable.FullAttributes.Historizing != Historizing) return false; - } - else if (IsArray != null || Historizing != null) return false; - if (Parent != null && (node.Parent == null || !Parent.IsMatch(node.Parent, ns))) return false; - return true; - } - /// - /// Create string representation, for logging. - /// - /// StringBuilder to write to - /// Level of nesting, for clean indentation. - public void Format(StringBuilder builder, int idx) - { - if (Name != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("Name: {0}", Name); - builder.AppendLine(); - } - if (Description != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("Description: {0}", Description); - builder.AppendLine(); - } - if (Id != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("Id: {0}", Id); - builder.AppendLine(); - } - if (IsArray != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("IsArray: {0}", IsArray); - builder.AppendLine(); - } - if (Historizing != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("Historizing: {0}", Historizing); - builder.AppendLine(); - } - if (Namespace != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("Namespace: {0}", Namespace); - builder.AppendLine(); - } - if (TypeDefinition != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("TypeDefinition: {0}", TypeDefinition); - builder.AppendLine(); - } - if (NodeClass != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.AppendFormat("NodeClass: {0}", NodeClass); - builder.AppendLine(); - } - if (Parent != null) - { - builder.Append(' ', (idx + 1) * 4); - builder.Append("Parent:"); - builder.AppendLine(); - Parent.Format(builder, idx + 1); - } - } - - public override string ToString() - { - var builder = new StringBuilder(); - Format(builder, 0); - return builder.ToString(); - } - } - /// /// Describes a transformation to the source hierarchy. Consists of a filter and a transformation type. /// @@ -239,7 +34,7 @@ public class NodeTransformation private readonly int index; public NodeTransformation(RawNodeTransformation raw, int index) { - Filter = new NodeFilter(raw.Filter); + Filter = raw.Filter ?? new NodeFilter(); Type = raw.Type; this.index = index; } diff --git a/Extractor/UAExtractor.cs b/Extractor/UAExtractor.cs index ef9f8afe6..99fc35a92 100644 --- a/Extractor/UAExtractor.cs +++ b/Extractor/UAExtractor.cs @@ -722,9 +722,9 @@ private void BuildTransformations() log.LogWarning("Property Id filter is deprecated, use transformations instead"); transformations.Add(new NodeTransformation(new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Id = Config.Extraction.PropertyIdFilter + Id = new RegexFieldFilter(Config.Extraction.PropertyIdFilter) }, Type = TransformationType.Property }, idx++)); @@ -734,9 +734,9 @@ private void BuildTransformations() log.LogWarning("Property Name filter is deprecated, use transformations instead"); transformations.Add(new NodeTransformation(new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = Config.Extraction.PropertyNameFilter + Name = new RegexFieldFilter(Config.Extraction.PropertyNameFilter) }, Type = TransformationType.Property }, idx++)); @@ -744,12 +744,11 @@ private void BuildTransformations() if (Config.Extraction.IgnoreName != null && Config.Extraction.IgnoreName.Any()) { log.LogWarning("Ignore name is deprecated, use transformations instead"); - var filterStr = string.Join('|', Config.Extraction.IgnoreName.Select(str => $"^{str}$")); transformations.Add(new NodeTransformation(new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = filterStr + Name = new ListFieldFilter(Config.Extraction.IgnoreName, null) }, Type = TransformationType.Ignore }, idx++)); @@ -760,9 +759,9 @@ private void BuildTransformations() var filterStr = string.Join('|', Config.Extraction.IgnoreNamePrefix.Select(str => $"^{str}")); transformations.Add(new NodeTransformation(new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = filterStr + Name = new RegexFieldFilter(filterStr) }, Type = TransformationType.Ignore }, idx++)); diff --git a/ExtractorLauncher/Program.cs b/ExtractorLauncher/Program.cs index 79dbf2bd2..a8c998d1f 100644 --- a/ExtractorLauncher/Program.cs +++ b/ExtractorLauncher/Program.cs @@ -15,6 +15,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +using Cognite.Extractor.Configuration; using Cognite.Extractor.Utils.CommandLine; using Cognite.OpcUa.Config; using Cognite.OpcUa.Service; @@ -94,6 +95,12 @@ public static class Program public static CancellationToken? RootToken { get; set; } public static async Task Main(string[] args) { + try + { + ConfigurationUtils.AddTypeConverter(new FieldFilterConverter()); + } + catch { } + return await GetCommandLineOptions().InvokeAsync(args); } diff --git a/Test/CDFMockHandler.cs b/Test/CDFMockHandler.cs index c8fbc5859..c26e1a301 100644 --- a/Test/CDFMockHandler.cs +++ b/Test/CDFMockHandler.cs @@ -60,7 +60,6 @@ public class CDFMockHandler public Dictionary Views { get; } = new(); public Dictionary Containers { get; } = new(); public Dictionary Instances { get; } = new(); - public List Callbacks { get; } = new List(); private long assetIdCounter = 1; private long timeseriesIdCounter = 1; @@ -122,15 +121,6 @@ private async Task MessageHandler(HttpRequestMessage req, C return GetFailedRequest(HttpStatusCode.InternalServerError); } - if (req.RequestUri.AbsolutePath == $"/api/playground/projects/{project}/functions/1234/call") - { - var funcContent = await req.Content.ReadAsStringAsync(cancellationToken); - var res = HandleCallFunction(funcContent); - res.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); - res.Headers.Add("x-request-id", (requestIdCounter++).ToString(CultureInfo.InvariantCulture)); - return res; - } - string reqPath = req.RequestUri.AbsolutePath.Replace($"/api/v1/projects/{project}", "", StringComparison.InvariantCulture); log.LogInformation("Request to {Path}", reqPath); @@ -986,27 +976,6 @@ private HttpResponseMessage HandleDeleteRelationships(string content) }; } - - private HttpResponseMessage HandleCallFunction(string content) - { - var options = new JsonSerializerSettings - { - ContractResolver = new DefaultContractResolver - { - NamingStrategy = new CamelCaseNamingStrategy() - } - }; - var data = JsonConvert.DeserializeObject>(content, options); - - Callbacks.Add(data.Data); - - return new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent("{}") - }; - } - public AssetDummy MockAsset(string externalId) { var asset = new AssetDummy diff --git a/Test/Integration/DataPointTests.cs b/Test/Integration/DataPointTests.cs index 3dc25bef9..b12674b9a 100644 --- a/Test/Integration/DataPointTests.cs +++ b/Test/Integration/DataPointTests.cs @@ -322,9 +322,9 @@ public async Task TestDataPointsAsEvents() { new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Variable bool" + Name = new RegexFieldFilter("Variable bool") }, Type = TransformationType.AsEvents } @@ -832,9 +832,9 @@ async Task Reset() { new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Id = $"i={ids.DoubleVar1.Identifier}$" + Id = new RegexFieldFilter($"i={ids.DoubleVar1.Identifier}$") }, Type = TransformationType.DropSubscriptions } diff --git a/Test/Integration/EventTests.cs b/Test/Integration/EventTests.cs index 17b8a808d..7af91f033 100644 --- a/Test/Integration/EventTests.cs +++ b/Test/Integration/EventTests.cs @@ -237,9 +237,9 @@ async Task Reset() { new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Id = $"i={ids.Obj1.Identifier}$" + Id = new RegexFieldFilter($"i={ids.Obj1.Identifier}$") }, Type = TransformationType.DropSubscriptions } diff --git a/Test/Integration/NodeExtractionTests.cs b/Test/Integration/NodeExtractionTests.cs index e697aed79..485cc126c 100644 --- a/Test/Integration/NodeExtractionTests.cs +++ b/Test/Integration/NodeExtractionTests.cs @@ -1068,9 +1068,9 @@ public async Task TestPropertyInheritance() { new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "^CustomRoot$" + Name = new RegexFieldFilter("^CustomRoot$") }, Type = TransformationType.Property } @@ -1115,9 +1115,9 @@ public async Task TestArrayPropertiesWithoutMaxArraySize() { new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "^CustomRoot$" + Name = new RegexFieldFilter("^CustomRoot$") }, Type = TransformationType.Property } @@ -1162,11 +1162,11 @@ public async Task TestLateIgnore() { new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Parent = new RawNodeFilter + Parent = new NodeFilter { - Name = "^CustomRoot$" + Name = new RegexFieldFilter("^CustomRoot$") } }, Type = TransformationType.Ignore diff --git a/Test/Unit/CDFPusherTest.cs b/Test/Unit/CDFPusherTest.cs index b5d4af13f..485feb8e0 100644 --- a/Test/Unit/CDFPusherTest.cs +++ b/Test/Unit/CDFPusherTest.cs @@ -687,223 +687,6 @@ public async Task TestUpdateRawTimeseries() Assert.True(CommonTestUtils.TestMetricValue("opcua_node_ensure_failures_cdf", 1)); } - - [Fact] - public async Task TestNodeCallback() - { - tester.Config.Cognite.BrowseCallback = new BrowseCallbackConfig - { - Id = 1234, - ReportOnEmpty = true - }; - - CommonTestUtils.ResetMetricValue("opcua_node_ensure_failures_cdf"); - tester.Config.Cognite.MetadataTargets = new MetadataTargetsConfig - { - Clean = new CleanMetadataTargetConfig - { - Relationships = true, - Assets = true, - Timeseries = true - }, - }; - - (handler, pusher) = tester.GetCDFPusher(); - using var extractor = tester.BuildExtractor(true, null, pusher); - - var dt = new UADataType(DataTypes.Double); - - var update = new UpdateConfig(); - await pusher.PushNodes(Enumerable.Empty(), Enumerable.Empty(), - Enumerable.Empty(), update, tester.Source.Token); - - Assert.Single(handler.Callbacks); - var res = handler.Callbacks[0]; - Assert.Equal(0, res.AssetsCreated); - Assert.Equal(0, res.AssetsUpdated); - Assert.Equal(0, res.TimeSeriesCreated); - Assert.Equal(0, res.TimeSeriesUpdated); - Assert.Equal(0, res.RelationshipsCreated); - Assert.Null(res.RawDatabase); - Assert.Null(res.AssetsTable); - Assert.Null(res.TimeSeriesTable); - Assert.Null(res.RelationshipsTable); - Assert.Equal("gp.", res.IdPrefix); - - var organizes = tester.Client.TypeManager.GetReferenceType(ReferenceTypeIds.Organizes); - - var source = new UAObject(new NodeId("source", 0), "Source", "Source", null, NodeId.Null, null); - var target = new UAObject(new NodeId("target", 0), "Target", "Target", null, NodeId.Null, null); - var sourceVar = new UAVariable(new NodeId("source2", 0), "Source", "Source", null, NodeId.Null, null); - var targetVar = new UAVariable(new NodeId("target2", 0), "Target", "Target", null, NodeId.Null, null); - - // Create one of each. - var node = new UAObject(tester.Server.Ids.Base.Root, "BaseRoot", null, null, NodeId.Null, null); - var variable = new UAVariable(tester.Server.Ids.Base.DoubleVar1, "Variable 1", null, null, new NodeId("parent", 0), null); - variable.FullAttributes.DataType = dt; - var rel = new UAReference(organizes, true, source, target); - - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(2, handler.Callbacks.Count); - res = handler.Callbacks[1]; - Assert.Equal(1, res.AssetsCreated); - Assert.Equal(0, res.AssetsUpdated); - Assert.Equal(1, res.TimeSeriesCreated); - Assert.Equal(0, res.TimeSeriesUpdated); - Assert.Equal(1, res.RelationshipsCreated); - - // Update each without modifying "update" - node.Attributes.Description = "Some description"; - variable.Attributes.Description = "Some description"; - - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(3, handler.Callbacks.Count); - res = handler.Callbacks[2]; - Assert.Equal(0, res.AssetsCreated); - Assert.Equal(0, res.AssetsUpdated); - Assert.Equal(0, res.TimeSeriesCreated); - Assert.Equal(0, res.TimeSeriesUpdated); - Assert.Equal(0, res.RelationshipsCreated); - - // Modify "update", also add another reference. - rel = new UAReference(organizes, true, sourceVar, targetVar); - update.Variables.Description = true; - update.Objects.Description = true; - - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(4, handler.Callbacks.Count); - res = handler.Callbacks[3]; - Assert.Equal(0, res.AssetsCreated); - Assert.Equal(1, res.AssetsUpdated); - Assert.Equal(0, res.TimeSeriesCreated); - Assert.Equal(1, res.TimeSeriesUpdated); - Assert.Equal(1, res.RelationshipsCreated); - - // Again, this time nothing changes - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(5, handler.Callbacks.Count); - res = handler.Callbacks[4]; - Assert.Equal(0, res.AssetsCreated); - Assert.Equal(0, res.AssetsUpdated); - Assert.Equal(0, res.TimeSeriesCreated); - Assert.Equal(0, res.TimeSeriesUpdated); - Assert.Equal(0, res.RelationshipsCreated); - } - [Fact] - public async Task TestRawNodeCallback() - { - tester.Config.Cognite.BrowseCallback = new BrowseCallbackConfig - { - Id = 1234, - ReportOnEmpty = true - }; - - tester.Config.Cognite.MetadataTargets = new MetadataTargetsConfig - { - Raw = new RawMetadataTargetConfig - { - Database = "metadata", - AssetsTable = "assets", - TimeseriesTable = "timeseries", - RelationshipsTable = "relationships" - } - }; - - var organizes = tester.Client.TypeManager.GetReferenceType(ReferenceTypeIds.Organizes); - - var source = new UAObject(new NodeId("source", 0), "Source", "Source", null, NodeId.Null, null); - var target = new UAObject(new NodeId("target", 0), "Target", "Target", null, NodeId.Null, null); - var sourceVar = new UAVariable(new NodeId("source2", 0), "Source", "Source", null, NodeId.Null, null); - var targetVar = new UAVariable(new NodeId("target2", 0), "Target", "Target", null, NodeId.Null, null); - - (handler, pusher) = tester.GetCDFPusher(); - using var extractor = tester.BuildExtractor(true, null, pusher); - CommonTestUtils.ResetMetricValue("opcua_node_ensure_failures_cdf"); - - var dt = new UADataType(DataTypes.Double); - - var update = new UpdateConfig(); - await pusher.PushNodes(Enumerable.Empty(), Enumerable.Empty(), - Enumerable.Empty(), update, tester.Source.Token); - - Assert.Single(handler.Callbacks); - var res = handler.Callbacks[0]; - Assert.Equal(0, res.RawAssetsCreated); - Assert.Equal(0, res.RawAssetsUpdated); - Assert.Equal(0, res.RawTimeseriesCreated); - Assert.Equal(0, res.RawTimeseriesUpdated); - Assert.Equal(0, res.MinimalTimeSeriesCreated); - Assert.Equal(0, res.RawRelationshipsCreated); - Assert.Equal("metadata", res.RawDatabase); - Assert.Equal("assets", res.AssetsTable); - Assert.Equal("timeseries", res.TimeSeriesTable); - Assert.Equal("relationships", res.RelationshipsTable); - Assert.Equal("gp.", res.IdPrefix); - - // Create one of each. - var node = new UAObject(tester.Server.Ids.Base.Root, "BaseRoot", null, null, NodeId.Null, null); - var variable = new UAVariable(tester.Server.Ids.Base.DoubleVar1, "Variable 1", null, null, new NodeId("parent", 0), null); - variable.FullAttributes.DataType = dt; - var rel = new UAReference(organizes, true, source, targetVar); - - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(2, handler.Callbacks.Count); - res = handler.Callbacks[1]; - Assert.Equal(1, res.RawAssetsCreated); - Assert.Equal(0, res.RawAssetsUpdated); - Assert.Equal(1, res.RawTimeseriesCreated); - Assert.Equal(1, res.MinimalTimeSeriesCreated); - Assert.Equal(0, res.TimeSeriesUpdated); - Assert.Equal(1, res.RawRelationshipsCreated); - - // Update each without modifying "update" - node.Attributes.Description = "Some description"; - variable.Attributes.Description = "Some description"; - - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(3, handler.Callbacks.Count); - res = handler.Callbacks[2]; - Assert.Equal(0, res.RawAssetsCreated); - Assert.Equal(0, res.RawAssetsUpdated); - Assert.Equal(0, res.RawTimeseriesCreated); - Assert.Equal(0, res.RawTimeseriesUpdated); - Assert.Equal(0, res.MinimalTimeSeriesCreated); - Assert.Equal(0, res.RawRelationshipsCreated); - - // Modify "update", also add another reference. - rel = new UAReference(organizes, true, sourceVar, targetVar); - update.Variables.Description = true; - update.Objects.Description = true; - - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(4, handler.Callbacks.Count); - res = handler.Callbacks[3]; - Assert.Equal(0, res.RawAssetsCreated); - Assert.Equal(1, res.RawAssetsUpdated); - Assert.Equal(0, res.RawTimeseriesCreated); - Assert.Equal(1, res.RawTimeseriesUpdated); - Assert.Equal(0, res.MinimalTimeSeriesCreated); - Assert.Equal(1, res.RawRelationshipsCreated); - - // Again, this time nothing changes - await pusher.PushNodes(new[] { node }, new[] { variable }, new[] { rel }, update, tester.Source.Token); - - Assert.Equal(5, handler.Callbacks.Count); - res = handler.Callbacks[4]; - Assert.Equal(0, res.RawAssetsCreated); - Assert.Equal(0, res.RawAssetsUpdated); - Assert.Equal(0, res.RawTimeseriesCreated); - Assert.Equal(0, res.RawTimeseriesUpdated); - Assert.Equal(0, res.MinimalTimeSeriesCreated); - Assert.Equal(0, res.RawRelationshipsCreated); - } #endregion #region other-methods diff --git a/Test/Unit/FDMTests.cs b/Test/Unit/FDMTests.cs index 1e34c825c..d8117617c 100644 --- a/Test/Unit/FDMTests.cs +++ b/Test/Unit/FDMTests.cs @@ -171,9 +171,9 @@ public async Task TestDeleteNodesAndEdges() .Append(new RawNodeTransformation { Type = Cognite.OpcUa.TransformationType.Ignore, - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "DeviceThree" + Name = new RegexFieldFilter("DeviceThree") } }); diff --git a/Test/Unit/TransformationTest.cs b/Test/Unit/TransformationTest.cs index 88c297f52..3bf5fb0a4 100644 --- a/Test/Unit/TransformationTest.cs +++ b/Test/Unit/TransformationTest.cs @@ -1,14 +1,21 @@ -using Cognite.Extractor.Testing; +using Cognite.Extractor.Configuration; +using Cognite.Extractor.Testing; using Cognite.OpcUa; using Cognite.OpcUa.Config; using Cognite.OpcUa.Nodes; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Opc.Ua; +using System; +using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; using Xunit; using Xunit.Abstractions; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.TypeInspectors; namespace Test.Unit { @@ -26,13 +33,18 @@ public TransformationTest(ITestOutputHelper output) var services = new ServiceCollection(); services.AddTestLogging(output); log = services.BuildServiceProvider().GetRequiredService>(); + try + { + ConfigurationUtils.AddTypeConverter(new FieldFilterConverter()); + } + catch { } } [Fact] public void TestNameFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - Name = "Test" + Name = new RegexFieldFilter("Test") }; var nodes = new[] { @@ -41,7 +53,6 @@ public void TestNameFilter() new UAObject(new NodeId(3), "Test", null, null, new NodeId("parent", 0), null), new UAObject(new NodeId(4), "Other", null, null, new NodeId("parent", 0), null), }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, node.FullAttributes.TypeDefinition?.Id, nss, node.NodeClass)).ToList(); @@ -55,9 +66,9 @@ public void TestNameFilter() [Fact] public void TestDescriptionFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - Description = "Test" + Description = new RegexFieldFilter("Test") }; var nodes = new[] { @@ -70,7 +81,6 @@ public void TestDescriptionFilter() nodes[1].Attributes.Description = "Some Other test"; nodes[2].Attributes.Description = null; nodes[3].Attributes.Description = ""; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, node.FullAttributes.TypeDefinition?.Id, nss, node.NodeClass)).ToList(); @@ -84,9 +94,9 @@ public void TestDescriptionFilter() [Fact] public void TestIdFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - Id = "id|1|i=3|s=4" + Id = new RegexFieldFilter("id|1|i=3|s=4") }; var nodes = new[] { @@ -95,7 +105,6 @@ public void TestIdFilter() new UAObject(new NodeId(3), "Test", null, null, new NodeId("parent", 0), null), new UAObject(new NodeId(4), "Other", null, null, new NodeId("parent", 0), null), }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, node.FullAttributes.TypeDefinition?.Id, nss, node.NodeClass)).ToList(); @@ -108,9 +117,9 @@ public void TestIdFilter() [Fact] public void TestNamespaceFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - Namespace = "test-|uri" + Namespace = new RegexFieldFilter("test-|uri") }; var nodes = new[] { @@ -119,7 +128,6 @@ public void TestNamespaceFilter() new UAObject(new NodeId(3, 2), "Test", null, null, new NodeId("parent", 0), null), new UAObject(new NodeId(4, 3), "Other", null, null, new NodeId("parent", 0), null), }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, node.FullAttributes.TypeDefinition?.Id, nss, node.NodeClass)).ToList(); @@ -133,9 +141,9 @@ public void TestNamespaceFilter() [Fact] public void TestTypeDefinitionFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - TypeDefinition = "i=1|test" + TypeDefinition = new RegexFieldFilter("i=1|test") }; var nodes = new[] @@ -145,7 +153,6 @@ public void TestTypeDefinitionFilter() new UAObject(new NodeId(3), "Test", null, null, new NodeId("parent", 0), null), new UAObject(new NodeId(4), "Other", null, null, new NodeId("parent", 0), new UAObjectType(new NodeId("test", 0))), }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, node.FullAttributes.TypeDefinition?.Id, nss, node.NodeClass)).ToList(); @@ -169,7 +176,7 @@ private static NodeId GetTypeDefinition(BaseUANode node) [InlineData(false)] public void TestIsArrayFilter(bool isArray) { - var raw = new RawNodeFilter + var filter = new NodeFilter { IsArray = isArray }; @@ -185,7 +192,6 @@ public void TestIsArrayFilter(bool isArray) (nodes[2].Attributes as Cognite.OpcUa.Nodes.VariableAttributes).ArrayDimensions = new[] { 4 }; (nodes[4].Attributes as Cognite.OpcUa.Nodes.VariableAttributes).ArrayDimensions = new[] { 4 }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, GetTypeDefinition(node), nss, node.NodeClass)).ToList(); @@ -207,11 +213,11 @@ public void TestIsArrayFilter(bool isArray) [Fact] public void TestParentFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - Parent = new RawNodeFilter + Parent = new NodeFilter { - Name = "parent1" + Name = new RegexFieldFilter("parent1") } }; @@ -225,7 +231,6 @@ public void TestParentFilter() new UAObject(new NodeId(3, 2), "Test", null, parent2, new NodeId("parent2", 0), null) { Parent = parent2 }, new UAObject(new NodeId(4, 3), "Other", null, parent2, new NodeId("parent2", 0), null) { Parent = parent2 }, }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, node.FullAttributes.TypeDefinition?.Id, nss, node.NodeClass)).ToList(); @@ -239,7 +244,7 @@ public void TestParentFilter() [Fact] public void TestNodeClassFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { NodeClass = NodeClass.Object }; @@ -251,7 +256,6 @@ public void TestNodeClassFilter() new UAObject(new NodeId(3), "Test", null, null, new NodeId("parent", 0), null), new UAObjectType(new NodeId(4), "Other", null, null, new NodeId("parent", 0)), }; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, GetTypeDefinition(node), nss, node.NodeClass)).ToList(); @@ -267,7 +271,7 @@ public void TestNodeClassFilter() [InlineData(false)] public void TestHistorizingFilter(bool historizing) { - var raw = new RawNodeFilter + var filter = new NodeFilter { Historizing = historizing }; @@ -283,7 +287,6 @@ public void TestHistorizingFilter(bool historizing) (nodes[2].Attributes as Cognite.OpcUa.Nodes.VariableAttributes).Historizing = true; (nodes[4].Attributes as Cognite.OpcUa.Nodes.VariableAttributes).Historizing = true; - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, GetTypeDefinition(node), nss, node.NodeClass)).ToList(); @@ -305,19 +308,19 @@ public void TestHistorizingFilter(bool historizing) [Fact] public void TestMultipleFilter() { - var raw = new RawNodeFilter + var filter = new NodeFilter { - Id = "i=1", - Description = "target", - TypeDefinition = "i=1", + Id = new RegexFieldFilter("i=1"), + Description = new RegexFieldFilter("target"), + TypeDefinition = new RegexFieldFilter("i=1"), IsArray = true, Historizing = true, - Name = "target", - Parent = new RawNodeFilter + Name = new RegexFieldFilter("target"), + Parent = new NodeFilter { - Name = "parent1" + Name = new RegexFieldFilter("parent1") }, - Namespace = "test-", + Namespace = new RegexFieldFilter("test-"), NodeClass = NodeClass.Variable }; var parent1 = new UAObject(new NodeId("parent1", 0), "parent1", null, null, NodeId.Null, null); @@ -358,7 +361,6 @@ public void TestMultipleFilter() nodes.Add(node); } - var filter = new NodeFilter(raw); var matched = nodes.Where(node => filter.IsMatch(node, nss)).ToList(); var matchedBasic = nodes.Where(node => filter.IsBasicMatch(node.Name, node.Id, GetTypeDefinition(node), nss, node.NodeClass)).ToList(); @@ -371,9 +373,9 @@ public void TestIgnoreTransformation() { var raw = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Test" + Name = new RegexFieldFilter("Test") }, Type = TransformationType.Ignore }; @@ -402,9 +404,9 @@ public void TestPropertyTransformation() { var raw = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Test" + Name = new RegexFieldFilter("Test") }, Type = TransformationType.Property }; @@ -432,17 +434,17 @@ public void TestTimeSeriesTransformation() { var raw = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Test" + Name = new RegexFieldFilter("Test") }, Type = TransformationType.Property }; var raw2 = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Other" + Name = new RegexFieldFilter("Other") }, Type = TransformationType.TimeSeries }; @@ -473,9 +475,9 @@ public void TestAsEventTransformation() { var raw = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Test" + Name = new RegexFieldFilter("Test") }, Type = TransformationType.AsEvents }; @@ -504,24 +506,24 @@ public void TestLogTransformation() { var raw = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "name", - Namespace = "namespace", + Name = new RegexFieldFilter("name"), + Namespace = new RegexFieldFilter("namespace"), NodeClass = NodeClass.Variable, - Description = "description", - TypeDefinition = "typeDefinition", - Id = "id", + Description = new RegexFieldFilter("description"), + TypeDefinition = new RegexFieldFilter("typeDefinition"), + Id = new RegexFieldFilter("id"), IsArray = true, Historizing = true, - Parent = new RawNodeFilter + Parent = new NodeFilter { - Name = "name2", - Namespace = "namespace2", + Name = new RegexFieldFilter("name2"), + Namespace = new RegexFieldFilter("namespace2"), NodeClass = NodeClass.Object, - Description = "description2", - TypeDefinition = "typeDefinition2", - Id = "id2", + Description = new RegexFieldFilter("description2"), + TypeDefinition = new RegexFieldFilter("typeDefinition2"), + Id = new RegexFieldFilter("id2"), IsArray = false, Historizing = false } @@ -558,9 +560,9 @@ public void TestIncludeTransformation() { var raw = new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "Test" + Name = new RegexFieldFilter("Test") }, Type = TransformationType.Include }; @@ -587,5 +589,63 @@ public void TestIncludeTransformation() Assert.False(nodes[2].Ignore); Assert.False(nodes[3].Ignore); } + + [Fact] + public void TestFromYaml() + { + var extFile = "transform_entries.txt"; + File.WriteAllLines(extFile, new[] { + "foo", + "bar", + "", + " ", + "baz" + }); + var data = @" + transformations: + - type: Property + filter: + name: ""regex"" + description: + - desc1 + - desc2 + id: + file: transform_entries.txt + "; + var conf = ConfigurationUtils.ReadString(data, false); + + Assert.Single(conf.Transformations); + var tf = conf.Transformations.First(); + var desc = Assert.IsType(tf.Filter.Description); + Assert.Equal(2, desc.Raw.Count()); + Assert.Null(desc.OriginalFile); + Assert.Contains("desc1", desc.Raw); + Assert.Contains("desc2", desc.Raw); + var name = Assert.IsType(tf.Filter.Name); + Assert.Equal("regex", name.Raw); + var nodeId = Assert.IsType(tf.Filter.Id); + Assert.Equal(3, nodeId.Raw.Count()); + Assert.Contains("foo", nodeId.Raw); + Assert.Contains("bar", nodeId.Raw); + Assert.Contains("baz", nodeId.Raw); + + //var str = serializer.Serialize(conf.Transformations); + var str = ConfigurationUtils.ConfigToString( + conf.Transformations, + Enumerable.Empty(), + Enumerable.Empty(), + Enumerable.Empty(), + false); + Assert.Equal( +@"- type: ""Property"" + filter: + name: ""regex"" + description: + - ""desc1"" + - ""desc2"" + id: + file: ""transform_entries.txt"" +", str); + } } } diff --git a/Test/Unit/UAClientTest.cs b/Test/Unit/UAClientTest.cs index 6865df1d0..7dfa6ccb6 100644 --- a/Test/Unit/UAClientTest.cs +++ b/Test/Unit/UAClientTest.cs @@ -561,9 +561,9 @@ public async Task TestBrowseIgnoreName() tester.Client.Browser.Transformations = new TransformationCollection(new List { new NodeTransformation(new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "WideRoot" + Name = new RegexFieldFilter("WideRoot") }, Type = TransformationType.Ignore }, 0) @@ -590,9 +590,9 @@ public async Task TestBrowseIgnorePrefix() tester.Client.Browser.Transformations = new TransformationCollection(new List { new NodeTransformation(new RawNodeTransformation { - Filter = new RawNodeFilter + Filter = new NodeFilter { - Name = "^Sub|^Deep" + Name = new RegexFieldFilter("^Sub|^Deep") }, Type = TransformationType.Ignore }, 0) diff --git a/Test/Utils/BaseExtractorTestFixture.cs b/Test/Utils/BaseExtractorTestFixture.cs index 4947591e7..fe3adc486 100644 --- a/Test/Utils/BaseExtractorTestFixture.cs +++ b/Test/Utils/BaseExtractorTestFixture.cs @@ -44,6 +44,11 @@ protected BaseExtractorTestFixture(PredefinedSetup[] setups = null) // Set higher min thread count, this is required due to running both server and client in the same process. // The server uses the threadPool in a weird way that can cause starvation if this is set too low. ThreadPool.SetMinThreads(20, 20); + try + { + ConfigurationUtils.AddTypeConverter(new FieldFilterConverter()); + } + catch { } Services = new ServiceCollection(); Config = Services.AddConfig("config.test.yml", 1); Config.Source.EndpointUrl = $"opc.tcp://localhost:{Port}"; diff --git a/config/config.example.yml b/config/config.example.yml index f3654155a..67ae20250 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -400,32 +400,6 @@ cognite: extraction-pipeline: # ExternalId of extraction pipeline # external-id: - # Call a cognite function with the number of assets, timeseries and relationships created/updated - # after each browse/rebrowse operation. - # The function is called with the following object: - # { - # "idPrefix": [Configured extraction.idPrefix], - # "assetsCreated": [Number of new assets or raw rows in the assets table created], - # "assetsUpdated": [Number of assets updated or raw rows in the asset table modified], - # "timeSeriesCreated": [Number of new time series or raw rows in the time series table created], - # "timeSeriesUpdated": [Number of time series updated or raw rows in the time series table modified], - # "minimalTimeSeriesCreated": [Number of time series created, only used if timeseries are written to raw], - # "relationshipsCreated": [Number of new relationships, or raw rows in the relationships table created], - # "rawDatabase": [Name of the configured raw database], - # "assetsTable": [Name of the configured raw table for assets], - # "timeSeriesTable": [Name of the configured raw table for time series], - # "relationshipsTable": [Name of the configured raw table for relationships] - # } - # "Minimal time series" here refers to timeseries that are created with no metadata when timeseries metadata is written to raw. - # This requires functions:WRITE scoped to the function given by external-id or id, and READ if external-id is used. - browse-callback: - # Function external id. If this is set the functions:read capability is required. - external-id: - # Function internal id. - id: - # True to always report, even if nothing was modified. The counts are always reported, - # so you can handle this logic yourself. - report-on-empty: false # Push to an influx-database. Data-variables are mapped to series with the given id. # Events are mapped to series with ids on the form [id].[eventId] where id is given by the source node. @@ -789,6 +763,14 @@ extraction: # "Ignore" filters first. This is also worth noting when it comes to TimeSeries transformations, which can undo Property # transformations. # It is possible to have multiple of each filter type. + # + # name, description, id, namespace, and type-definition can be one of + # - A regex string + # - A list of strings, in which case the filter is a match if any of these are equal to the value being matched. + # - An object on the form + # name: + # file: ... + # where "file" is a path to a local file containing newline-separated values to be matched exactly. transformations: # Type, either "Ignore", "Property", "DropSubscriptions", "TimeSeries", "AsEvents", or "Include" # - type: diff --git a/manifest.yml b/manifest.yml index 0ea7f4306..70d892ad9 100644 --- a/manifest.yml +++ b/manifest.yml @@ -67,6 +67,11 @@ schema: - "https://raw.githubusercontent.com/" versions: + "2.30.0": + description: Add support for configuring filters from a list of values. + changelog: + added: + - Add support for configuring filters from a list of values contained either directly in config, or in a local file. "2.29.1": description: Log more related to continuation points. changelog: diff --git a/schema/cognite_config.schema.json b/schema/cognite_config.schema.json index ec3b58640..aa16bad12 100644 --- a/schema/cognite_config.schema.json +++ b/schema/cognite_config.schema.json @@ -254,26 +254,6 @@ } } }, - "browse-callback": { - "type": "object", - "unevaluatedProperties": false, - "description": "Specify a CDF function that is called after nodes are pushed to CDF, reporting the number of changed and created nodes. The function is called with a JSON object. \nThe counts for assets, time series, and relationships also include rows created in CDF Raw tables if raw is enabled as a metadata target.", - "properties": { - "id": { - "type": "integer", - "description": "Internal ID of function to call." - }, - "external-id": { - "type": "string", - "description": "External ID of function to call" - }, - "report-on-empty": { - "type": "boolean", - "description": "Call callback even if zero items are created or updated." - } - }, - "$extraDescription": "The function will be called with the following JSON object:\n\n```json\n{\n \"idPrefix\": \"Configured ID prefix\",\n \"assetsCreated\": \"The number of assets reported to CDF\",\n \"assetsUpdated\": \"The number of assets updated in CDF\",\n \"timeSeriesCreated\": \"The number of time series created in CDF\",\n \"timeSeriesUpdated\": \"The number of time series created in CDF\",\n \"minimalTimeSeriesCreated\": \"The number of time series created with no metadata. Only used if time series metadata is not written to CDF clean\",\n \"relationshipsCreated\": \"The number of new relationships created in CDF\",\n \"rawDatabase\": \"Name of the configured CDF RAW database\",\n \"assetsTable\": \"Name of the configured CDF RAW table for assets\",\n \"timeSeriesTable\": \"Name of the configured CDF RAW table for time series\",\n \"relationshipsTable\": \"Name of the configured CDF RAW table for relationships.\"\n}\n```" - }, "delete-relationships": { "type": "boolean", "description": "If this is set to `true`, relationships deleted from the source will be hard-deleted in CDF. Relationships do not have metadata, so soft-deleting them is not possible." diff --git a/schema/extraction_config.schema.json b/schema/extraction_config.schema.json index 80a554d35..8295dcad2 100644 --- a/schema/extraction_config.schema.json +++ b/schema/extraction_config.schema.json @@ -338,28 +338,28 @@ "unevaluatedProperties": false, "properties": { "name": { - "type": "string", - "description": "Regex on node `DisplayName`." + "$ref": "field-filter", + "description": "Regex or list of values matching node `DisplayName`." }, "description": { - "type": "string", - "description": "Regex on node `Description`." + "$ref": "field-filter", + "description": "Regex or list of values matching node `Description`." }, "id": { - "type": "string", - "description": "Regex on node ID. IDs on the form `i=123` or `s=string` are matched." + "$ref": "field-filter", + "description": "Regex or list of values matching node ID. IDs on the form `i=123` or `s=string` are matched." }, "is-array": { "type": "boolean", "description": "Match on whether a node is an array. If this is set to `true` or `false`, the filter will only match variables." }, "namespace": { - "type": "string", - "description": "Regex on the full namespace of the node ID." + "$ref": "field-filter", + "description": "Regex or list of values matching the full namespace of the node ID." }, "type-definition": { - "type": "string", - "description": "Regex on the node ID of the type definition. On the form `i=123` or `s=string`." + "$ref": "field-filter", + "description": "Regex or list of values matching the node ID of the type definition. On the form `i=123` or `s=string`." }, "historizing": { "type": "boolean", @@ -382,6 +382,37 @@ "description": "Another filter instance which is applied to the parent node." } } + }, + "field-filter": { + "$id": "field-filter", + "oneOf": [ + { + "type": "object", + "unevaluatedProperties": false, + "title": "File", + "description": "Match field exactly from a list of values read from a local file", + "properties": { + "file": { + "type": "string", + "description": "Path to a local file containing a new-line separated list of values." + } + } + }, + { + "type": "array", + "title": "Values", + "description": "Match field exactly from a list of values", + "items": { + "type": "string", + "description": "Value to match" + } + }, + { + "type": "string", + "title": "Regex", + "description": "Regex on the value." + } + ] } } } \ No newline at end of file