Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add option to transform filter to use a list of values #663

Merged
merged 4 commits into from
Jun 24, 2024
Merged

Add option to transform filter to use a list of values #663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8b95ca4
Add option to transform filter to use a list of values
einarmo Jun 20, 2024
fc2734e
Version bump
einarmo Jun 20, 2024
c6d8ab8
Fix tests
einarmo Jun 20, 2024
6ba0786
Fix tests
einarmo Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
320 changes: 312 additions & 8 deletions Extractor/Config/ExtractionConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,21 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */

using Cognite.Extensions.DataModels.QueryBuilder;
using Cognite.Extractor.Common;
using Cognite.OpcUa.Nodes;
using Cognite.OpcUa.NodeSources;
using Microsoft.Extensions.Logging;
using Opc.Ua;
using Serilog.Debugging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using YamlDotNet.Core;
using YamlDotNet.Core.Events;
using YamlDotNet.Serialization;

namespace Cognite.OpcUa.Config
Expand Down Expand Up @@ -386,32 +395,169 @@ public bool NamespacePublicationDate

public List<string> GetTargets => ToBeSubscribed;
}
public class RawNodeFilter
public interface IFieldFilter
{
bool IsMatch(string raw);
}

public class RegexFieldFilter : IFieldFilter
{
private readonly Regex filter;

public RegexFieldFilter(string regex)
{
filter = new Regex(regex, RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.CultureInvariant);
}
public bool IsMatch(string raw)
{
return filter.IsMatch(raw);
}

public string Raw => filter.ToString();

public override string ToString()
{
return Raw;
}
}

public class ListFieldFilter : IFieldFilter
{
private readonly HashSet<string> entries;
public string? OriginalFile { get; }

public ListFieldFilter(IEnumerable<string> items, string? originalFile)
{
entries = new HashSet<string>(items);
OriginalFile = originalFile;
}

public bool IsMatch(string raw)
{
return entries.Contains(raw);
}

public IEnumerable<string> Raw => entries;

public override string ToString()
{
var builder = new StringBuilder();
builder.Append("Any of: [");
var first = true;
foreach (var entry in entries)
{
if (!first)
{
builder.Append(", ");
}
first = false;
builder.AppendFormat("\"{0}\"", entry);
}
builder.Append("]");
return base.ToString();
}
}

public class FieldFilterConverter : IYamlTypeConverter
{
public bool Accepts(Type type)
{
return typeof(IFieldFilter).IsAssignableFrom(type);
}

public object? ReadYaml(IParser parser, Type type)
{
if (parser.TryConsume<Scalar>(out var scalar))
{
return new RegexFieldFilter(scalar.Value);
}
if (parser.TryConsume<SequenceStart>(out _))
{
var items = new List<string>();
while (!parser.Accept<SequenceEnd>(out _))
{
var seqScalar = parser.Consume<Scalar>();
items.Add(seqScalar.Value);
}

parser.Consume<SequenceEnd>();

return new ListFieldFilter(items, null);
}
if (parser.TryConsume<MappingStart>(out _))
{
var key = parser.Consume<Scalar>();
if (key.Value != "file")
{
throw new YamlException("Expected object containing \"file\"");
}
var value = parser.Consume<Scalar>();
var lines = File.ReadAllLines(value.Value);
parser.Consume<MappingEnd>();
return new ListFieldFilter(lines.Where(line => !string.IsNullOrWhiteSpace(line)), value.Value);
}

throw new YamlException("Expected a string, object, or list of strings");
}

public void WriteYaml(IEmitter emitter, object? value, Type type)
{
if (value is RegexFieldFilter regexFilter)
{
emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, regexFilter.Raw, ScalarStyle.DoubleQuoted, false, true));
}
else if (value is ListFieldFilter listFilter)
{
if (listFilter.OriginalFile != null)
{
emitter.Emit(new MappingStart());
emitter.Emit(new Scalar("file"));
emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, listFilter.OriginalFile, ScalarStyle.DoubleQuoted, false, true));
emitter.Emit(new MappingEnd());
}
else
{
emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true, SequenceStyle.Block, Mark.Empty, Mark.Empty));
foreach (var entry in listFilter.Raw)
{
emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, entry, ScalarStyle.DoubleQuoted, false, true));
}
emitter.Emit(new SequenceEnd());
}
}
else
{
emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, "", ScalarStyle.DoubleQuoted, false, true));
}
}
}

public class NodeFilter
{
/// <summary>
/// Regex on node DisplayName.
/// </summary>
public string? Name { get; set; }
public IFieldFilter? Name { get; set; }
/// <summary>
/// Regex on node Description.
/// </summary>
public string? Description { get; set; }
public IFieldFilter? Description { get; set; }
/// <summary>
/// Regex on node id. Ids on the form "i=123" or "s=string" are matched.
/// </summary>
public string? Id { get; set; }
public IFieldFilter? Id { get; set; }
/// <summary>
/// Whether the node is an array. If this is set, the filter only matches varables.
/// </summary>
public bool? IsArray { get; set; }
/// <summary>
/// Regex on the full namespace of the node id.
/// </summary>
public string? Namespace { get; set; }
public IFieldFilter? Namespace { get; set; }
/// <summary>
/// Regex on the id of the type definition. On the form "i=123" or "s=string".
/// </summary>
public string? TypeDefinition { get; set; }
public IFieldFilter? TypeDefinition { get; set; }
/// <summary>
/// The "historizing" attribute on variables. If this is set, the filter only matches variables.
/// </summary>
Expand All @@ -424,7 +570,165 @@ public class RawNodeFilter
/// <summary>
/// Another instance of NodeFilter which is applied to the parent node.
/// </summary>
public RawNodeFilter? Parent { get; set; }
public NodeFilter? Parent { get; set; }

/// <summary>
/// Return a representation if the identifier of <paramref name="id"/>,
/// on the form i=123, or s=string, etc.
/// </summary>
/// <param name="id">Identifier to get representation of</param>
/// <returns>String representation of identifier of <paramref name="id"/></returns>
private static string GetIdString(NodeId id)
{
var builder = new StringBuilder();
NodeId.Format(builder, id.Identifier, id.IdType, 0);
return builder.ToString();
}

/// <summary>
/// Test for match using only basic properties available in when reading from the server.
/// Will always return false if there are filters on not yet available fields.
/// </summary>
/// <param name="name">DisplayName</param>
/// <param name="id">Raw NodeId</param>
/// <param name="typeDefinition">TypeDefinition Id</param>
/// <param name="namespaces">Source namespacetable</param>
/// <param name="nc">NodeClass</param>
/// <returns>True if match</returns>
public bool IsBasicMatch(string name, NodeId id, NodeId typeDefinition, NamespaceTable namespaces, NodeClass nc)
{
if (Description != null || IsArray != null || Parent != null || Historizing != null) return false;
return MatchBasic(name, id ?? NodeId.Null, typeDefinition, namespaces, nc);
}

/// <summary>
/// Test for match using only basic properties available in when reading from the server.
/// </summary>
/// <param name="name">DisplayName</param>
/// <param name="id">Raw NodeId</param>
/// <param name="typeDefinition">TypeDefinition Id</param>
/// <param name="namespaces">Source namespacetable</param>
/// <param name="nc">NodeClass</param>
/// <returns>True if match</returns>
private bool MatchBasic(string? name, NodeId id, NodeId? typeDefinition, NamespaceTable namespaces, NodeClass nc)
{
if (Name != null && (string.IsNullOrEmpty(name) || !Name.IsMatch(name))) return false;
if (Id != null)
{
if (id == null || id.IsNullNodeId) return false;
var idstr = GetIdString(id);
if (!Id.IsMatch(idstr)) return false;
}
if (Namespace != null && namespaces != null)
{
var ns = namespaces.GetString(id.NamespaceIndex);
if (string.IsNullOrEmpty(ns)) return false;
if (!Namespace.IsMatch(ns)) return false;
}
if (TypeDefinition != null)
{
if (typeDefinition == null || typeDefinition.IsNullNodeId) return false;
var tdStr = GetIdString(typeDefinition);
if (!TypeDefinition.IsMatch(tdStr)) return false;
}
if (NodeClass != null)
{
if (nc != NodeClass.Value) return false;
}
return true;
}
/// <summary>
/// Return true if the given node matches the filter.
/// </summary>
/// <param name="node">Node to test</param>
/// <param name="ns">Currently active namespace table</param>
/// <returns>True if match</returns>
public bool IsMatch(BaseUANode node, NamespaceTable ns)
{
if (node == null || !MatchBasic(node.Name, node.Id, node.TypeDefinition, ns, node.NodeClass)) return false;
if (Description != null && (string.IsNullOrEmpty(node.Attributes.Description) || !Description.IsMatch(node.Attributes.Description))) return false;
if (node is UAVariable variable)
{
if (IsArray != null && variable.IsArray != IsArray) return false;
if (Historizing != null && variable.FullAttributes.Historizing != Historizing) return false;
}
else if (IsArray != null || Historizing != null) return false;
if (Parent != null && (node.Parent == null || !Parent.IsMatch(node.Parent, ns))) return false;
return true;
}

public bool IsBasic => Description == null && IsArray == null && Parent == null && Historizing == null;

/// <summary>
/// Create string representation, for logging.
/// </summary>
/// <param name="builder">StringBuilder to write to</param>
/// <param name="idx">Level of nesting, for clean indentation.</param>
public void Format(StringBuilder builder, int idx)
{
if (Name != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("Name: {0}", Name);
builder.AppendLine();
}
if (Description != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("Description: {0}", Description);
builder.AppendLine();
}
if (Id != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("Id: {0}", Id);
builder.AppendLine();
}
if (IsArray != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("IsArray: {0}", IsArray);
builder.AppendLine();
}
if (Historizing != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("Historizing: {0}", Historizing);
builder.AppendLine();
}
if (Namespace != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("Namespace: {0}", Namespace);
builder.AppendLine();
}
if (TypeDefinition != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("TypeDefinition: {0}", TypeDefinition);
builder.AppendLine();
}
if (NodeClass != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.AppendFormat("NodeClass: {0}", NodeClass);
builder.AppendLine();
}
if (Parent != null)
{
builder.Append(' ', (idx + 1) * 4);
builder.Append("Parent:");
builder.AppendLine();
Parent.Format(builder, idx + 1);
}
}

public override string ToString()
{
var builder = new StringBuilder();
Format(builder, 0);
return builder.ToString();
}
}
public class RawNodeTransformation
{
Expand All @@ -435,7 +739,7 @@ public class RawNodeTransformation
/// <summary>
/// NodeFilter. All non-null filters must match each node for the transformation to be applied.
/// </summary>
public RawNodeFilter? Filter { get; set; }
public NodeFilter? Filter { get; set; }
}

public enum StatusCodeMode
Expand Down
2 changes: 1 addition & 1 deletion Extractor/Extractor.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AdysTech.InfluxDB.Client.Net.Core" Version="0.25.0" />
<PackageReference Include="Cognite.ExtractorUtils" Version="1.24.0" />
<PackageReference Include="Cognite.ExtractorUtils" Version="1.24.3" />
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Client" Version="1.5.374.36" />
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Configuration" Version="1.5.374.36" />
Expand Down
Loading
Loading