Initial preparation for supporting writing to any/all cdf targets

.config/dotnet-tools.json

@@ -9,4 +9,4 @@
       ]
     }
   }
-}
+}

Extractor/BrowseCallback.cs

@@ -27,10 +27,15 @@ public class BrowseReport
     {
         public string? IdPrefix { get; set; }
         public int AssetsUpdated { get; set; }
+        public int RawAssetsUpdated { get; set; }
         public int AssetsCreated { get; set; }
+        public int RawAssetsCreated { get; set; }
         public int TimeSeriesUpdated { get; set; }
+        public int RawTimeseriesUpdated { get; set; }
         public int TimeSeriesCreated { get; set; }
+        public int RawTimeseriesCreated { get; set; }
         public int RelationshipsCreated { get; set; }
+        public int RawRelationshipsCreated { get; set; }
         public int MinimalTimeSeriesCreated { get; set; }
         public string? RawDatabase { get; set; }
         public string? AssetsTable { get; set; }

Extractor/Config/CogniteConfig.cs

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 
 using Cognite.Extensions;
 using Cognite.Extractor.Utils;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.ComponentModel.DataAnnotations;
@@ -56,6 +57,7 @@ public class CognitePusherConfig : CogniteConfig, IPusherConfig
         /// similarly to raw-metadata, and datapoints will be pushed. Nothing will be written to raw, and no assets will be created.
         /// Events will be created, but without asset context.
         /// </summary>
+        [Obsolete("Deprecated!")]
         public bool SkipMetadata { get; set; }
         /// <summary>
         /// Store assets and/or timeseries data in raw. Assets will not be created at all,
@@ -65,6 +67,7 @@ public class CognitePusherConfig : CogniteConfig, IPusherConfig
         /// of the source node is added to metadata if applicable.
         /// Use different table names for assets and timeseries.
         /// </summary>
+        [Obsolete("Deprecated! Use MetadataTargetsConfig.RawMetadataTargetConfig instead.")]
         public RawMetadataConfig? RawMetadata { get; set; }
         /// <summary>
         /// Map metadata to asset/timeseries attributes. Each of "assets" and "timeseries" is a map from property DisplayName to
@@ -114,7 +117,13 @@ public double? NonFiniteReplacement
         /// <summary>
         /// Configuration for writing to a custom OPC-UA flexible data model.
         /// </summary>
+        [Obsolete("Deprecated! Use MetadataTargetsConfig.FdmDestinationConfig instead.")]
         public FdmDestinationConfig? FlexibleDataModels { get; set; }
+
+        /// <summary>
+        /// This is the implementation of the metadata targets 
+        /// </summary>
+        public MetadataTargetsConfig? MetadataTargets { get; set; }
     }
     public class RawMetadataConfig
     {
@@ -136,6 +145,34 @@ public class RawMetadataConfig
         /// </summary>
         public string? RelationshipsTable { get; set; }
     }
+    public class MetadataTargetsConfig 
+    {
+        /// <summary>
+        /// Raw metadata targets config
+        /// </summary>
+        public RawMetadataTargetConfig? Raw { get; set; }
+        /// <summary>
+        /// Clean metadata targets config
+        /// </summary>
+        public CleanMetadataTargetConfig? Clean { get; set; }
+        /// <summary>
+        /// FDM destination config
+        /// </summary>
+        public FdmDestinationConfig? DataModels { get; set; }
+    }
+    public class RawMetadataTargetConfig
+    {
+        public string? Database { get; set; }
+        public string? AssetsTable { get; set; }
+        public string? TimeseriesTable { get; set; }
+        public string? RelationshipsTable { get; set; }
+    }
+    public class CleanMetadataTargetConfig
+    {
+        public bool Assets { get; set; }
+        public bool Timeseries { get; set; }
+        public bool Relationships { get; set; }
+    }
     public class MetadataMapConfig
     {
         public Dictionary<string, string>? Assets { get; set; }

Extractor/NodeSources/NodeSetSource.cs

@@ -349,7 +349,7 @@ private async Task InitNodes(IEnumerable<BaseUANode> nodes, CancellationToken to
 
             await InitNodes(NodeList, token);
 
-            var usesFdm = Config.Cognite?.FlexibleDataModels?.Enabled ?? false;
+            var usesFdm = Config.Cognite?.MetadataTargets?.DataModels?.Enabled ?? false;
 
             if (Config.Extraction.Relationships.Enabled)
             {

Extractor/NodeSources/UANodeSource.cs

@@ -110,7 +110,7 @@ private async Task InitNodes(IEnumerable<BaseUANode> nodes, CancellationToken to
 
             await InitNodes(NodeList, token);
 
-            var usesFdm = Config.Cognite?.FlexibleDataModels?.Enabled ?? false;
+            var usesFdm = Config.Cognite?.MetadataTargets?.DataModels?.Enabled ?? false;
 
             if (Config.Extraction.Relationships.Enabled)
             {

Extractor/Nodes/UADataType.cs

@@ -38,7 +38,7 @@ public DataTypeAttributes() : base(NodeClass.DataType)
 
         public override IEnumerable<uint> GetAttributeSet(FullConfig config)
         {
-            if (config.Cognite?.FlexibleDataModels?.Enabled ?? false)
+            if (config.Cognite?.MetadataTargets?.DataModels?.Enabled ?? false)
             {
                 yield return Attributes.DataTypeDefinition;
             }

Extractor/Nodes/UAVariable.cs

@@ -457,22 +457,9 @@ public TimeSeriesCreate ToTimeseries(
             UAExtractor extractor,
             long? dataSetId,
             IDictionary<NodeId, long>? nodeToAssetIds,
-            Dictionary<string, string>? metaMap,
-            bool minimal = false)
+            Dictionary<string, string>? metaMap)
         {
             string? externalId = GetUniqueId(client);
-
-            if (minimal)
-            {
-                return new TimeSeriesCreate
-                {
-                    ExternalId = externalId,
-                    IsString = FullAttributes.DataType.IsString,
-                    IsStep = FullAttributes.DataType.IsStep,
-                    DataSetId = dataSetId
-                };
-            }
-
             var writePoco = new TimeSeriesCreate
             {
                 Description = FullAttributes.Description,
@@ -502,6 +489,19 @@ public TimeSeriesCreate ToTimeseries(
 
             return writePoco;
         }
+
+        public TimeSeriesCreate ToMinimalTimeseries(IUAClientAccess client, long? dataSetId)
+        {
+            string? externalId = GetUniqueId(client);
+
+            return new TimeSeriesCreate
+            {
+                ExternalId = externalId,
+                IsString = FullAttributes.DataType.IsString,
+                IsStep = FullAttributes.DataType.IsStep,
+                DataSetId = dataSetId
+            };
+        }
         #endregion
     }