From c9ce0925ae095455a9664ac4451e5a1fac047e96 Mon Sep 17 00:00:00 2001 From: Einar Date: Tue, 20 Feb 2024 11:17:55 +0100 Subject: [PATCH] Backport schema changes from docs (#603) * Update schema * Fixes from review --- schema/cognite_config.schema.json | 183 ++++++++----- .../continuation_point_throttling.schema.json | 12 +- schema/event_config.schema.json | 90 +++--- schema/extraction_config.schema.json | 177 ++++++------ schema/failure_buffer_config.schema.json | 32 ++- schema/full_config.schema.json | 5 + schema/history_config.schema.json | 55 ++-- schema/influx_config.schema.json | 29 +- schema/logger_config.schema.json | 6 +- schema/metrics_config.schema.json | 4 +- schema/mqtt_config.schema.json | 126 +++++---- schema/proto_node_id.schema.json | 3 +- schema/source_config.schema.json | 257 +++++++++++------- schema/state_storage_config.schema.json | 14 +- schema/subscription_config.schema.json | 26 +- schema/ua_retries_config.schema.json | 4 +- 16 files changed, 583 insertions(+), 440 deletions(-) diff --git a/schema/cognite_config.schema.json b/schema/cognite_config.schema.json index 2fd416a15..89506db57 100644 --- a/schema/cognite_config.schema.json +++ b/schema/cognite_config.schema.json @@ -5,7 +5,7 @@ "description": "Configuration for pushing data to Cognite Data Fusion (CDF)", "allOf": [ { - "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.16.0/schema/cognite_config.schema.json" + "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.19.0/schema/cognite_config.schema.json" } ], "unevaluatedProperties": false, @@ -13,34 +13,43 @@ "data-set": { "type": "object", "unevaluatedProperties": false, + "description": "Data set used for new time series, assets, events, and relationships. Existing objects will not be updated", "properties": { "id": { "type": "integer", - "description": "Data set used for new objects. Existing objects will not be updated" + "description": "Data set internal ID" }, "external-id": { "type": "string", - "description": "Data set used for new objects. Requires the capability `datasets:Read` for the given data set" + "description": "Data set external ID" } - } - }, - "debug": { - "type": "boolean", - "description": "Debug mode. If `true`, the extractor will not push to target" + }, + "oneOf": [ + { + "required": [ + "id" + ] + }, + { + "required": [ + "external-id" + ] + } + ] }, "read-extracted-ranges": { "type": "boolean", - "description": "Whether to read start/end-points on startup, where possible. At least one pusher should be able to do this, or `state-store` should be enabled, otherwise back/frontfill will run for the entire history every restart", + "description": "Specifies whether to read start/end-points for datapoints on startup, where possible. It is generally recommended to use the [`state-storage`](#state-storage) instead of this.", "default": true }, "metadata-targets": { "type": "object", - "description": "Metadata targets for objects, variable and references in CDF.", + "description": "Configure targets for node metadata. This configures which resources other than time series datapoints are created. By default, if this is left out, data is written to assets and time series metadata. Note that this behavior is deprecated, in the future leaving this section out will result in no metadata being written at all.", "unevaluatedProperties": false, "properties": { "raw": { "type": "object", - "description": "Details of CDF raw (database and table).", + "description": "Write metadata to the CDF staging area (Raw).", "unevaluatedProperties": false, "required": [ "database" @@ -48,93 +57,117 @@ "properties": { "database": { "type": "string", - "description": "Database in raw" + "description": "The CDF Raw database to write to." }, "assets-table": { "type": "string", - "descripion": "Assets table in raw" + "descripion": "Name of the Raw table to write assets to." }, "timeseries-table": { "type": "string", - "description": "Timeseries table in raw" + "description": "Name of the Raw table to write time series metadata to." }, "relationships-table": { "type": "string", - "description": "Relationships table in raw" + "description": "Name of the Raw table to write relationships metadata to." } } }, "clean": { "unevaluatedProperties": false, - "description": "Enable or disable assets, timeseries and/or relationships", + "description": "Write metadata to CDF clean, assets, time series, and relationships.", "type": "object", "properties": { "assets": { "type": "boolean", - "description": "Indicates enabled or disabled" + "description": "Set to `true` to enable creating CDF assets from OPC UA nodes." }, "timeseries": { "type": "boolean", - "description": "Indicates enabled or disabled" + "description": "Set to `true` to enable adding metadata to CDF time series based on OPC UA properties." }, "relationships": { "type": "boolean", - "description": "Indicates enabled or disabled" + "description": "Set to `true` to enable creating relationships from OPC UA references. Requires [`extraction.relationships`](#extraction.relationships) to be enabled." } } }, "data-models": { "unevaluatedProperties": false, - "description": "configuration for flexible data models in CDF", + "description": "ALPHA: Write metadata to CDF Data Models.\n\nThis will create CDF data models based on the OPC UA type hierarchy, then populate them with data from the OPC UA node hierarchy. Note that this _requires_ [`extraction.relationships.enabled`](#extraction.relationships) and [`extraction.relationships.hierarchical`](#extraction.relationships) to be set to `true`, and there must be exactly one root node with ID `i=84`.\n\nNote that this feature is in _alpha_ there may be changes that require you to delete the data model from CDF, and breaking changes to the configuration schema. These changes will not be considered breaking changes to the extractor.", "type": "object", "required": [ "enabled", - "exclude-no-referenced", - "types-to-map", - "skip-simple-types", - "skip-types-on-equal-count", - "ignore-mandatory" + "model-space", + "instance-space", + "model-version" ], "properties": { - "space": { - "type": "string", - "description": "Space instance in CDF" - }, "enabled": { "type": "boolean", - "description": "Should enable" + "description": "Set this to `true` to enable writing to CDF Data Models." }, - "exclude-no-referenced": { - "type": "boolean", - "description": "Exclude any node referenced by custom nodes" + "model-space": { + "type": "string", + "description": "Set the space to create data models in. The space will be created if it does not exist." + }, + "instance-space": { + "type": "string", + "description": "Set the space instances will be created in. The space will be created if it does not exist. May be the same as `model-space`." + }, + "model-version": { + "type": "string", + "description": "Version used for created data model and all created views." }, "types-to-map": { "type": "string", - "description": "Types to map to fdm", + "description": "Configure which types to map to Data Models.\n\n`Referenced` means that only types that are referenced by instance nodes will be created.\n`Custom` means that all types not in the base namespace will be created.\n`All` means that all types will be created.\n\nNote: Setting this to `All` is rarely useful, and may produce impractically large models.", "enum": [ - "referenced", - "custom", - "all" - ] + "Referenced", + "Custom", + "All" + ], + "default": "Custom" }, "skip-simple-types": { "type": "boolean", - "description": "Do not create views without an associated container" - }, - "skip-types-on-equal-count": { - "type": "boolean", - "description": "Skip views if number of views in CDF is equal to number of views in server" + "description": "Do not create views without their own connections or properties. Simplifies the model greatly, but reduces the number of distinct types in your model.", + "default": false }, "ignore-mandatory": { "type": "boolean", - "description": "Allow optional mandatory options" + "description": "Let mandatory options be nullable. Many servers do not obey `Mandatory` requirements in their own models, which breaks when they are ingested into CDF, where nullable constraints are enforced.", + "default": false }, "connection-target-map": { "type": "object", - "description": "target connections", - "[A-z0-9-_.]+": { - "type": "string" - } + "description": "Target connections on the form `\"Type.Property\": \"Target\"`. This is useful for certain schemas. This overrides the expected type of specific CDF Connections, letting you override incorrect schemas. For example, the published nodeset file for ISA-95 incorrectly states that the `EquipmentClass` reference for `EquipmentType` is an `Object`, while it should be an `ObjectClass`.", + "patternProperties": { + "[A-z0-9-_.]+": { + "type": "string", + "description": "NodeClass to override connection with.", + "enum": [ + "ObjectType", + "Object", + "VariableType", + "Variable", + "ReferenceType", + "DataType", + "View", + "Method" + ] + } + }, + "examples": [ + { + "EquipmentType.EquipmentClass": "ObjectType" + } + ] + }, + "enable-deletes": { + "type": "boolean", + "described": "Enable deleting Data Model nodes if deletes are enabled and the node is deleted in the source. This will only delete nodes extracted as part of the instance hierarchy. Types, and type-related nodes will not be deleted.", + "default": false } } } @@ -142,7 +175,7 @@ }, "metadata-mapping": { "type": "object", - "description": "Map metadata to asset/timeseries attributes. This lets you assign properties in OPC-UA directly to fields like `description` and `unit` in CDF", + "description": "Define mappings between properties in OPC UA and CDF attributes. For example, it is quite common for variables in OPC UA to have a property named `EngineeringUnits`, which ideally should be mapped to `unit` in CDF. This property lets you do that.", "examples": [ { "timeseries": { @@ -158,10 +191,11 @@ "properties": { "assets": { "type": "object", - "description": "Map metadata for assets", + "description": "Map metadata for assets.", "patternProperties": { ".*": { "type": "string", + "description": "Target asset attribute", "enum": [ "description", "name", @@ -172,10 +206,11 @@ }, "timeseries": { "type": "object", - "description": "Map metadata for timeseries", + "description": "Map metadata for time series.", "patternProperties": { ".*": { "type": "string", + "description": "Target time series attribute", "enum": [ "description", "name", @@ -189,7 +224,7 @@ }, "raw-node-buffer": { "type": "object", - "description": "Read from CDF instead of OPC-UA when starting, to speed up start on slow servers. Requires `extraction-data-types.expand-node-ids` and `append-internal-values` to be set to `true`. This should generaly be enabled along with `skip-metadata` or `raw-metadata`. If `browse-on-empty` is set to `true` and `raw-metadata` is configured with the same database and tables the extractor will read into raw on first run, then use raw as source for later runs. The Raw database can be deleted to reset on next read.", + "description": "Read from CDF instead of OPC-UA when starting, to speed up start on slow servers. Requires [`extraction.data-types.expand-node-ids`](#extraction.data-types) and [`extraction.data-types.append-internal-values`](#extraction.data-types) to be set to `true`.\n\nThis should generaly be enabled along with [`metadata-targets.raw`](#cognite.metadata-targets.raw) or with no metadata targets at all.\n\nIf `browse-on-empty` is set to `true` and `metadata-targets.raw` is configured with the same database and tables the extractor will read into raw on first run, then use raw as source for later runs. The Raw database can be deleted, and it will be re-created on extractor restart.", "unevaluatedProperties": false, "required": [ "enable", @@ -198,56 +233,64 @@ "properties": { "enable": { "type": "boolean", - "description": "Enable the raw node buffer" + "description": "Set to `true` to enable loading nodes from CDF Raw.", + "default": false }, "database": { "type": "string", - "description": "Raw database to read from" + "description": "CDF RAW database to read from." }, "assets-table": { "type": "string", - "description": "Table to read assets from, used for events" + "description": "CDF RAW table to read assets from, used for events. This is not useful if there are no custom nodes generating events in the server." }, "timeseries-table": { "type": "string", - "description": "Table to read timeseries from" + "description": "CDF RAW table to read time series from." }, "browse-on-empty": { "type": "boolean", - "description": "Run normal browse if nothing is found when reading from CDF, either because the tables are empty, or because they do not exist. This applies if no valid nodes are found, meaning it will apply if there are nodes, but none of them are valid extraction targets" + "description": "Run normal browse if nothing is found when reading from CDF, either because the tables are empty, or because they do not exist. Note that nodes may be present in the CDF RAW tables. Browse will run if no _valid_ nodes are found, even if there are nodes present in RAW." } } }, - "non-finite-replacement": { - "type": [ - "number", - "null" - ], - "description": "Replacement for NaN values. Deprecated in favor of `nan-replacement`", - "deprecated": true - }, "browse-callback": { "type": "object", "unevaluatedProperties": false, - "description": "Specify a CDF function that is called after nodes are pushed to CDF, reporting the number of changed and created nodes", + "description": "Specify a CDF function that is called after nodes are pushed to CDF, reporting the number of changed and created nodes. The function is called with a JSON object. \nThe counts for assets, time series, and relationships also include rows created in CDF Raw tables if raw is enabled as a metadata target.", "properties": { "id": { "type": "integer", - "description": "InternalId of function to call" + "description": "Internal ID of function to call." }, "external-id": { "type": "string", - "description": "ExternalId of function to call" + "description": "External ID of function to call" }, "report-on-empty": { "type": "boolean", "description": "Call callback even if zero items are created or updated." } - } + }, + "examples": [ + { + "idPrefix": "Configured ID prefix", + "assetsCreated": "The number of assets reported to CDF", + "assetsUpdated": "The number of assets updated in CDF", + "timeSeriesCreated": "The number of time series created in CDF", + "timeSeriesUpdated": "The number of time series created in CDF", + "minimalTimeSeriesCreated": "The number of time series created with no metadata. Only used if time series metadata is not written to CDF clean", + "relationshipsCreated": "The number of new relationships created in CDF", + "rawDatabase": "Name of the configured CDF RAW database", + "assetsTable": "Name of the configured CDF RAW table for assets", + "timeSeriesTable": "Name of the configured CDF RAW table for time series", + "relationshipsTable": "Name of the configured CDF RAW table for relationships." + } + ] }, "delete-relationships": { "type": "boolean", - "description": "If relationships are enabled, and written to clean, and deletes are neabled. This needs to be set to true in order to hard delete the relationships." + "description": "If this is set to `true`, relationships deleted from the source will be hard-deleted in CDF. Relationships do not have metadata, so soft-deleting them is not possible." } } } \ No newline at end of file diff --git a/schema/continuation_point_throttling.schema.json b/schema/continuation_point_throttling.schema.json index f74394255..53aa34b11 100644 --- a/schema/continuation_point_throttling.schema.json +++ b/schema/continuation_point_throttling.schema.json @@ -2,12 +2,16 @@ "$id": "continuation_point_throttling.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "allOf": [{ "$ref": "ua-throttling-config" }], + "allOf": [ + { + "$ref": "ua-throttling-config" + } + ], "unevaluatedProperties": false, "properties": { "max-node-parallelism": { "type": "integer", - "description": "Maximum number of concurrent nodes accross all parallel requests" + "description": "Maximum number of concurrent nodes accross all parallel requests." } }, "$defs": { @@ -18,11 +22,11 @@ "properties": { "max-per-minute": { "type": "integer", - "description": "Maximum number of requests per minute, approximately" + "description": "Maximum number of requests per minute, approximately." }, "max-parallelism": { "type": "integer", - "description": "Maximum number of parallel requests" + "description": "Maximum number of parallel requests." } } } diff --git a/schema/event_config.schema.json b/schema/event_config.schema.json index d53329856..2b6c887b6 100644 --- a/schema/event_config.schema.json +++ b/schema/event_config.schema.json @@ -2,72 +2,88 @@ "$id": "event_config.schema.json", "type": "object", "unevaluatedProperties": false, - "description": "Configuration for extraction OPC-UA events and alarams as CDF events or litedb time series", - "required": ["enabled"], + "description": "Configuration for extracting OPC UA events and alarams as CDF events or litedb time series", + "required": [ + "enabled" + ], "properties": { + "enabled": { + "type": "boolean", + "description": "Set to `true` to enable reading events from the server" + }, + "history": { + "type": "boolean", + "description": "Set to `true` to enable reading event history" + }, + "all-events": { + "type": "boolean", + "description": "Set to `true` to read base OPC UA events, in addition to custom events.", + "default": true + }, + "read-server": { + "type": "boolean", + "description": "Set to `true` to enable checking the server node when looking for event emitters", + "default": true + }, + "destination-name-map": { + "type": "object", + "description": "Map source browse names to other values in the destination. For CDF, internal properties may be overwritten. By default `Message` is mapped to `description`, `SourceNode` is used to set the `assetIds`, and `EventType` is mapped to `type`. These may be replaced by overrides in `destination-name-map`.\n\nIf `StartTime`, `EndTime`, or `SubType` are specified, either on the event in OPC-UA itself, or translated through this map, they are set directly on the CDF event. `StartTime` and `EndTime` should be either `DateTime`, or a number corresponding to the number of milliseconds since 01/01/1970. If no `StartTime` or `EndTime` are specified, both are set to the `Time` property.", + "examples": [ + { + "MyProperty": "SubType", + "MyEndTime": "EndTime" + } + ], + "patternProperties": { + ".*": { + "type": "string", + "description": "Target field to map to.", + "enum": [ + "StartTime", + "EndTime", + "SubType", + "Type" + ] + } + } + }, "event-ids": { "type": "array", - "description": "Event ids to map. Custom events must be subtypes of `BaseEventType`. This is used to specify which events should be extracted, instead of just extracting all events", + "description": "Allow-list of event type IDs to map. If this is included, only the events with type equal to one of these node IDs will be included.", "items": { "$ref": "proto_node_id.schema.json" } }, + "exclude-event-filter": { + "type": "string", + "description": "Regex filter on event type `DisplayName`, matches will not be extracted." + }, "emitter-ids": { "type": "array", - "description": "Id of nodes to be subscribed to as event emitters. Empty defaults to the server node. This is used to add extra emitters that are not in the extracted node hierarchy, or that does not correctly specify the `EventNotifier` property", + "description": "List of event emitters to extract from. The default behavior is to extract events from nodes based on the `EventNotifier` attribute. This option ignores `EventNotifier` and extracts events from the given list of nodes.", "items": { "$ref": "proto_node_id.schema.json" } }, "historizing-emitter-ids": { "type": "array", - "description": "Should be a subset of the `emitter-ids` property. Used to make the extractor read history from certain emitter nodes. Requires `events.history` to be `true`", + "description": "List of node IDs that must be a subset of the `emitter-ids` property. These emitters will have their event history read. The server must support historical events. The `events.history` property must be `true` for this to work.", "items": { "$ref": "proto_node_id.schema.json" } }, - "enabled": { - "type": "boolean", - "description": "`true` enables reading events from the server" - }, "discover-emitters": { "type": "boolean", - "description": "Automatically treat nodes with suitable `EventNotifier` as event emitters", - "default": true - }, - "all-events": { - "type": "boolean", - "description": "Enable reading both custom and base OPC-UA events", - "default": true - }, - "history": { - "type": "boolean", - "description": "`true` enables reading event history" - }, - "exclude-event-filter": { - "type": "string", - "description": "Regex filter on event type `DisplayName`, matches will be ignored" - }, - "read-server": { - "type": "boolean", - "description": "`true` enables checking the server node when looking for event emitters", + "description": "Automatically treat nodes with suitable `EventNotifier` as event emitters when they are discovered in the node hierarchy.", "default": true }, "exclude-properties": { "type": "array", - "description": "List of `BrowseName`s of event properties that are to be excluded from automatic mapping to destination metadata. `LocalTime` and `ReceiveTime` are excluded by default. Be aware that a maximum of 16 metadata entries are allowed in CDF", + "description": "List of `BrowseName`s of event properties that are to be excluded from automatic mapping to destination metadata. By default, `LocalTime` and `ReceiveTime` are excluded. Be aware that a maximum of 16 metadata entries are allowed in CDF.", "items": { "type": "string", "description": "`BrowseName` of an event property" } - }, - "destination-name-map": { - "type": "object", - "description": "Map source browse names to other values in the destination. For CDF, internal properties may be overwritten, by default `Message` is mapped to `description`, `SourceNode` is used to set the `assetIds`, and `EventType` is mapped to `type`. These may be replaced by overrides in `destination-name-map`. If `StartTime`, `EndTime`, or `SubType` are specified, either on the event in OPC-UA itself, or translated through this map, they are set directly on the CDF event. `StartTime` and `EndTime` should be either `DateTime`, or a number corresponding to the number of milliseconds since 01/01/1970. If no `StartTime` or `EndTime` are specified, both are set to the `Time` property.", - "examples": [{ - "MyProperty": "SubType", - "MyEndTime": "EndTime" - }] } } } \ No newline at end of file diff --git a/schema/extraction_config.schema.json b/schema/extraction_config.schema.json index 0b8e16210..cc0208ef1 100644 --- a/schema/extraction_config.schema.json +++ b/schema/extraction_config.schema.json @@ -2,30 +2,12 @@ "$id": "extraction_config.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "description": "Configuration for how metadata should be extracted from the server, and generally about the startup phase of the extractor", + "description": "Configuration for general extraction options, such as data types, mapping, and filters.\n### External ID generation\nIDs used in OPC UA are special `nodeId` objects with an identifier and a namespace that must be converted to a string before they are written to CDF. A direct conversion, however, has several potential problems.\n * The namespace index is by default part of the node, but it may change between server restarts. Only the namespace itself is fixed.\n * The namespace table may be modified, in which case all old node IDs are invalidated.\n * Node IDs are not unique between different OPC UA servers.\n * Node identifiers can be duplicated accross namespaces.\n\n The solution is to create a node ID on the following form:\n ```[id-prefix][namespace][identifierType]=[identifier as string]([optional array index])```.\nFor example, the node with node ID `ns=1;i=123` with ID prefix `gp:` would be mapped to `gp:http://my.namespace.url:i=123`.\n\nYou can optionally override this behavior for individual nodes by using `node-map`(#extraction.node-map).", "unevaluatedProperties": false, "properties": { "id-prefix": { "type": "string", - "description": "Global prefix for `externalId`s in destinations. Should be unique to prevent name conflicts. The generated `externalId` is on the form `[id-prefix][namespace][identifier]`" - }, - "ignore-name-prefix": { - "type": "array", - "description": "DEPRECATED. Specify a list of prefixes on `DisplayName` to ignore", - "deprecated": true, - "items": { - "type": "string", - "description": "`DisplayName` prefix" - } - }, - "ignore-name": { - "type": "array", - "description": "DEPRECATED. Specify a list `DisplayName`s to ignore", - "deprecated": true, - "items": { - "type": "string", - "description": "`DisplayName` to ignore" - } + "description": "Global prefix for `externalId`s in destinations. Should be unique for each extractor deployment to prevent name conflicts." }, "root-node": { "$ref": "proto_node_id.schema.json", @@ -40,124 +22,121 @@ "uniqueItems": true, "items": { "$ref": "proto_node_id.schema.json", - "description": "Root node. Defaults to the Objects node", - "default": { - "node-id": "i=86" - } + "description": "Root node." } }, "node-map": { "type": "object", - "description": "Override mappings between OPC-UA node ids and externalIds, allowing e.g. the `root-node` to be mapped to a particular asset in CDF. Applies to all `externalId`s", + "description": "Map from external IDs to OPC UA node IDs. This can, for example, be used to place the node hierarchy as a child of a specific asset in the asset hierarchy.", "unevaluatedProperties": false, "patternProperties": { ".*": { - "$ref": "proto_node_id.schema.json" + "$ref": "proto_node_id.schema.json", + "description": "Target node ID for mapping external ID." } } }, "namespace-map": { "type": "object", - "description": "Map OPC-UA namespaces to prefixes in CDF. If not mapped, the full namespace URI is used. This saves space compared to using the full URI, and might make ids more readable", + "description": "Map OPC-UA namespaces to prefixes in CDF. If not mapped, the full namespace URI is used. This saves space compared to using the full URI, and might make IDs more readable.", "unevaluatedProperties": false, "patternProperties": { ".*": { "type": "string", - "description": "Mapped namespace uri" + "description": "Namespace URI to map." } } }, "auto-rebrowse-period": { "type": "string", - "description": "Time in minutes between each callto browse the OPC-UA node hierarchy, then push nodes to destinations. Note that this is a heavy operation, so this number should not be set too low. Format is `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`. You can also use a cron expression on the form `[minute] [hour] [day of month] [month] [day of week]`", + "description": "Time in minutes between each automatic re-browse of the node hierarchy. Format is as given in [Timestamps and intervals](#timestamps-and-intervals), this option accepts cron expressions. Set this to `0` to disable automatic re-browsing of the server.", "default": "0m" }, "enable-audit-discovery": { "type": "boolean", - "description": "Enable using audit events to discover new nodes on the server. If this is set to true, the client will expect `AuditAddNodes` or `AuditAddReferences` events on the server node. These will be used to add new nodes automatically be recursively browsing from each node mentioned in the event" + "description": "Enable this to make the extractor listen to `AuditAddNodes` and `AuditAddReferences` events from the server, and use that to identify when new nodes are added to the server. This is more efficient than browsing the node hierarchy, but does not work with data models and requires that the server supports auditing.", + "default": false }, "data-push-delay": { "type": "string", - "description": "Delay between each push of data points to targets. Format is `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`", - "default": "1000ms" + "description": "Time between each push to destinations. Format is as given in [Timestamps and intervals](#timestamps-and-intervals).", + "default": "1s" }, "update": { "type": "object", - "description": "Update data in destinations on rebrowse or restart. Set `auto-rebrowse-period` to some value to do this periodically. `Context` here refers to the structure of the node graph in OPC-UA, i.e. hierarchical references. `Metadata` refers to any information obtained from OPC-UA properties. Enabling anything here will increase the startup and rebrowse-time of the extractor", + "description": "Update data in destinations on re-browse or restart. Set `auto-rebrowse-period` to do this periodically.", "unevaluatedProperties": false, "properties": { "objects": { "$ref": "type-update-config", - "description": "Configuration for updating objects and object types" + "description": "Configuration for updating objects and object types." }, "variables": { "$ref": "type-update-config", - "description": "Configuration for updating variables and variable types" + "description": "Configuration for updating variables and variable types." } } }, "data-types": { "$ref": "data-type-config" }, - "property-name-filter": { - "type": "string", - "deprecated": true, - "description": "DEPRECATED. Regex filter on DisplayName to treat variables as properties" - }, - "property-id-filter": { - "type": "string", - "deprecated": true, - "description": "DEPRECATED. Regex filter on NodeId to treat variables as properties" - }, "relationships": { "type": "object", - "description": "Configuration for translating OPC-UA references to relationships in CDF", + "description": "Map OPC UA references to relationships in CDF, or edges in CDF data models. Generated relationships will have external ID on the form `[prefix][reference type];[source][target]`\n\nOnly relationships between mapped nodes are extracted.", "unevaluatedProperties": false, "properties": { "enabled": { "type": "boolean", - "description": "`true` to enable mapping OPC-UA references to relationships in CDF" + "description": "Set to `true` to enable mapping OPC-UA references to relationships in CDF", + "default": false }, "hierarchical": { "type": "boolean", - "description": "`true` to enable also mapping the hierarchical references over. These are normally mapped to `assetId`/`parentId` relations in CDF, but when doing that the type of the reference is lost. Requires `enabled` to be `true`" + "description": "Set to `true` to enable also mapping the hierarchical references over. These are normally mapped to `assetId`/`parentId` relations in CDF, but when doing that the type of the reference is lost. Requires `enabled` to be `true`.", + "default": false }, "inverse-hierarchical": { "type": "boolean", - "description": "`true` to create inverse relationships for each fo the hierarchical references. For efficiency these are not read, but inferred from forward references. Does nothing if `hierarchical` is `false`" + "description": "Set to `true` to create inverse relationships for each fo the hierarchical references. For efficiency these are not read, but inferred from forward references. Does nothing if `hierarchical` is `false`.", + "default": false } - } + }, + "required": [ + "enabled" + ] }, "node-types": { "type": "object", - "description": "Configuration for reading OPC-UA node types", + "description": "Configuration for mapping OPC UA types to CDF in some way.", "unevaluatedProperties": false, "properties": { "metadata": { "type": "boolean", - "description": "Add the `TypeDefinition` as a metadata field to all nodes" + "description": "Add the `TypeDefinition` as a metadata field to all objects and variables." }, "as-nodes": { "type": "boolean", - "description": "Allow reading object- and variable types as normal nodes and map them to destinations. They will need to be in the mapped hierarchy, so you will need to add a root node that they are descended from." + "description": "Allow reading object- and variable types as normal nodes and map them to assets or data model nodes in CDF. They will need to be in the mapped hierarchy, so you will need to add a root node that they are descended from." } } }, "map-variable-children": { "type": "boolean", - "description": "If `true`, the extractor will try reading children of variables and map those to timeseries as well" + "description": "Set to `true` to make the extractor read children of variables and potentially map those to timeseries as well.", + "default": false }, "transformations": { "type": "array", - "description": "A list of transformations to be applied to the source nodes before pushing. The possible transformations are `Ignore`, `Property`, `DropSubscriptions`, `TimeSeries`. For efficiency put `Ignore` filters first. This is also worth noting when it comes to `TimeSeries` transformations, which can undo `Property` transformations", + "description": "A list of transformations to be applied to the source nodes before pushing. Transformations are applied sequentially, so it can help performance to put `Ignore` filters first, and `TimeSeries` filters can undo `Property` transformations.", "uniqueItems": true, "items": { "type": "object", "unevaluatedProperties": false, + "description": "A single transformation.", "properties": { "type": { "type": "string", - "description": "Transformation type. `Ignore` ignores the node and all descendants, `DropSubscriptions` prevents the extractor from subscribing to this node, `Property` converts timeseries and their descendants into metadata, `TimeSeries` converts metadata into timeseries.", + "description": "Transformation type. `Ignore` ignores the node and all descendants, `DropSubscriptions` prevents the extractor from subscribing to this node, `Property` converts time series and their descendants into metadata, `TimeSeries` converts metadata into time series, `DropSubscriptions` prevents the extractor from subscribing to events or data points on matching nodes.", "enum": [ "Ignore", "DropSubscriptions", @@ -167,23 +146,24 @@ }, "filter": { "$ref": "raw-node-filter", - "description": "Filter to match. All non-null filters must match each node for the transformation to be applied" + "description": "Filter to match. All non-null filters must match each node for the transformation to be applied." } } } }, "deletes": { "type": "object", - "description": "Configure soft deletes. When this is enabled, all read nodes are written to a state store after browse, and nodes that are missing on subsequent browses are marked as deleted from CDF, with a configurable marker. A notable exception is relationships in CDF, which has no metadata, so these are hard deleted if cognite.delete-relationships is enabled.", + "description": "Configure soft deletes. When this is enabled, all read nodes are written to a state store after browse, and nodes that are missing on subsequent browses are marked as deleted from CDF, with a configurable marker. A notable exception is relationships in CDF, which has no metadata, so these are hard deleted if [`cognite.delete-relationships`](#cognite) is enabled.", "unevaluatedProperties": false, "properties": { "enabled": { "type": "boolean", - "description": "True to enable deletes. This requires a state store to be configured." + "description": "Set this to `true` to enable deletes. This requires [`state-storage`](#state-storage) to be configured." }, "delete-marker": { "type": "string", - "description": "Name of marked indicated that a node is deleted. Added to metadata, or as a column in Raw." + "description": "Name of marker used to indicate that a node is deleted. Added to metadata, or as a column in Raw.", + "default": "deleted" } } } @@ -196,19 +176,19 @@ "properties": { "description": { "type": "boolean", - "description": "`true` to update description" + "description": "Set to `true` to update description" }, "context": { "type": "boolean", - "description": "`true` to update context, i.e. the position of the node in the node hierarchy" + "description": "Set to `true` to update context, i.e. the position of the node in the node hierarchy" }, "metadata": { "type": "boolean", - "description": "`true` to update metadata" + "description": "Set to `true` to update metadata, including fields like `unit`." }, "name": { "type": "boolean", - "description": "`true` to update name" + "description": "Set to `true` to update name." } } }, @@ -216,19 +196,21 @@ "type": "object", "$id": "proto-data-type", "unevaluatedProperties": false, - "description": "Definition of an OPC-UA data type", + "description": "Definition of an OPC UA data type.", "properties": { "node-id": { "$ref": "proto_node_id.schema.json", - "description": "NodeId of the data type" + "description": "Node ID of the data type" }, "is-step": { "type": "boolean", - "description": "True if is-step should be set on timeseries in CDF" + "description": "Set to `true` if is-step should be set on timeseries in CDF", + "default": false }, "enum": { "type": "boolean", - "description": "True if this is an enumeration" + "description": "Set to `true` if this is data type is an enumeration", + "default": false } } }, @@ -236,11 +218,11 @@ "type": "object", "$id": "data-type-config", "unevaluatedProperties": false, - "description": "Configuration related to how data types are mapped to CDF", + "description": "Configuration related to how data types and arrays should be handled by the OPC UA extractor.", "properties": { "custom-numeric-types": { "type": "array", - "description": "Add custom numeric types using ther NodeId. This overwrites default behavior, so it is possible to make the `Integer` data type discrete, even if it is not considered so by default. Note that the type in question needs to have a sensible numerical conversion in C#, unless it is an array type, in which case each element needs a conversion", + "description": "Set a list of manually defined numeric OPC UA data types. This can be used to make custom types be treated as numbers. Conversion from the actual returned values is done using the `C#` `Convert` functionality. If no valid conversion exists, this will fail.", "uniqueItems": true, "items": { "$ref": "proto-data-type" @@ -248,7 +230,7 @@ }, "ignore-data-types": { "type": "array", - "description": "List of NodeIds corresponding to data types that should be ignored. Time series with these data types will not be mapped to destinations", + "description": "List of node IDs of data types that should not be extracted to CDF. Time series with one of these data types are not extracted.", "uniqueItems": true, "items": { "$ref": "proto_node_id.schema.json" @@ -256,11 +238,11 @@ }, "unknown-as-scalar": { "type": "boolean", - "description": "Assume imprecise `ValueRanks` without `ArrayDimensions` are all scalar, and create timeseries in destinations accordingly. If such a variable produces an array, only the first element is mapped to CDF" + "description": "Assume non-specific `ValueRanks` in OPC UA (`ScalarOrOneDimension` or `Any`) are scalar if they do not have an `ArrayDimensions` attribute. If such a variable produces an array, only the first element will be extracted." }, "max-array-size": { "type": "integer", - "description": "Maximum size of array variables. Only arrays with the `ArrayDimensions` property in OPC-UA will be mapped. Leave this at 0 to only allow scalar values. Note that some server implementations have issues with the `ArrayDimensions` propertiy, so it is not fetched at all if `max-array-size` is 0. -1 indicates that there is no limit to array length, but variables must still be one-dimensional and contain a fixed number of elements." + "description": "Maximum length of arrays mapped to CDF, if this is set to 0, only scalar variables are mapped. Set this to `-1` to indicate that there is no upper limit.\n\n WARNING: If you set this to `-1`, and encounter a very large array, such as an image, the extractor might create hundreds of thousands of time series in CDF.\n\nIn general, extracting arrays require variables to have an `ArrayDimensions` attribute with length equal to 1. See `estimate-array-sizes` for workarounds." }, "allow-string-variables": { "type": "boolean", @@ -268,31 +250,31 @@ }, "auto-identify-types": { "type": "boolean", - "description": "Map out the data type hierarchy before starting. This is useful if there are custom numeric types, or any enum types are used. This must be enabled for enum metadata and `enums-as-strings` to work. If this is false, any custom numeric types have to be added manually" + "description": "Map out the data type hierarchy before starting. This is useful if there are custom numeric types, or any enum types are used. This must be enabled for enum metadata and `enums-as-strings` to work. If this is false, any custom numeric types have to be added manually." }, "enums-as-strings": { "type": "boolean", - "description": "If this is false and `auto-identify-types` is true, or there are manually added enums in `custom-numeric-types`, enums will be mapped to numeric time series, and labels are added as metadata fields. If this is true, labels are not mapped to metadata, and enums will be mapped to string timeseries with values equal to mapped label values" + "description": "If this is `false` and `auto-identify-types` is `true`, or there are manually added enums in `custom-numeric-types`, enums will be mapped to numeric time series, and labels are added as metadata fields.\n\nIf this is `true`, labels are not mapped to metadata, and enums will be mapped to string time series with values equal to mapped label values." }, "data-type-metadata": { "type": "boolean", - "description": "Add a metadata property `dataType`, which contains the id of the OPC-UA data type" + "description": "Add a metadata property `dataType`, which contains the node ID of the OPC-UA data type." }, "null-as-numeric": { "type": "boolean", - "description": "Set this to `true` to treat Null NodeIds as numeric instead of string" + "description": "Set this to `true` to treat `null` node IDs as numeric instead of ignoring them. Note that `null` node IDs violate the OPC UA standard." }, "expand-node-ids": { "type": "boolean", - "description": "Add full JSON node-ids to data pushed to Raw" + "description": "Add attributes such as `NodeId`, `ParentNodeId`, and `TypeDefinitionId` to nodes written to CDF Raw as full node IDs with reversible encoding." }, "append-internal-values": { "type": "boolean", - "description": "Add attributes generally used internally like `AccessLevel`, `Historizing`, `ArrayDimensions`, `ValueRank`, etc. to data pushed to Raw" + "description": "Add attributes generally used internally like `AccessLevel`, `Historizing`, `ArrayDimensions`, and `ValueRank` to data pushed to CDF Raw." }, "estimate-array-sizes": { "type": "boolean", - "description": "If `max-array-size` is set, this looks for the `MaxArraySize` property on each node with `one-dimensional` ValueRank. If it is not found, the extractor tries to read the value of the node, and look at the current array length. ArrayDimensions is still the preferred way to identify array sizes, this is not guaranteed to generate reasonable or useful values" + "description": "If `max-array-size` is set, this looks for the `MaxArraySize` property on each node with `one-dimensional` ValueRank. If it is not found, the extractor tries to read the value of the node, and look at the current array length.\n\n`ArrayDimensions` is still the preferred way to identify array sizes, this is not guaranteed to generate reasonable or useful values." } } }, @@ -303,41 +285,35 @@ "properties": { "name": { "type": "string", - "description": "Regex on node `DisplayName`" + "description": "Regex on node `DisplayName`." }, "description": { "type": "string", - "description": "Regex on node `Description`" + "description": "Regex on node `Description`." }, "id": { "type": "string", - "description": "Regex on node id. Ids on the form `i=123` or `s=string` are matched" + "description": "Regex on node ID. IDs on the form `i=123` or `s=string` are matched." }, "is-array": { - "type": [ - "boolean", - "null" - ], - "description": "Whether the node is an array. If this is set, the filter only considers variables" + "type": "boolean", + "description": "Match on whether a node is an array. If this is set to `true` or `false`, the filter will only match variables." }, "namespace": { "type": "string", - "description": "Regex on the full namespace of the node id" + "description": "Regex on the full namespace of the node ID." }, "type-definition": { "type": "string", - "description": "Regex on the id of the type definition. On the form `i=123` or `s=string`" + "description": "Regex on the node ID of the type definition. On the form `i=123` or `s=string`." }, "historizing": { - "type": [ - "boolean", - "null" - ], - "description": "Whether the node is historizing. If this is set, the filter only considers variables" + "type": "boolean", + "description": "Whether the node is historizing. If this is set, the filter will only match variables." }, "node-class": { "type": "string", - "description": "The OPC-UA node class, exact match.", + "description": "The OPC UA node class, exact match.", "enum": [ "Object", "ObjectType", @@ -349,7 +325,7 @@ }, "parent": { "$ref": "raw-node-filter", - "description": "Another filter instance which is applied to the parent node" + "description": "Another filter instance which is applied to the parent node." } } }, @@ -360,14 +336,15 @@ "properties": { "namespace-publication-date": { "type": "boolean", - "description": "Indicate if the extractor should subscribe to namespacePublicationDate on the Server node." + "description": "Indicate if the extractor should subscribe to namespacePublicationDate on the Server node.", + "default": false }, "namespaces": { "type": "array", - "description": "A list of namespaces uris used to filter which namespacePublicationDates the extractor will subscribe to.", + "description": "A list of namespace URIs used to filter which `namespace publication data` nodes the extractor will subscribe to.", "items": { "type": "string", - "description": "The uri of a namespace." + "description": "A namespace URI." } } } diff --git a/schema/failure_buffer_config.schema.json b/schema/failure_buffer_config.schema.json index 9ace6536c..759066323 100644 --- a/schema/failure_buffer_config.schema.json +++ b/schema/failure_buffer_config.schema.json @@ -2,32 +2,38 @@ "$id": "failure_buffer_config.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "description": "Configuration to store data points in a local file when connection to CDF is lost", + "description": "If the connection to CDF goes down, the OPC UA extractor supports buffering data points and events in a local file or InfluxDB. This is helpful if the connection is unstable, and the server does not provide its own historical data.", "unevaluatedProperties": false, "properties": { "enabled": { "type": "boolean", - "description": "Set to `true` to enable the failure buffer" - }, - "influx-state-store": { - "type": "boolean", - "description": "If `state-storage` is configured, this can be used to store the ranges of points buffered in influxdb, so that they can be recovered even if the extractor goes down" - }, - "influx": { - "type": "boolean", - "description": "Use influxdb as a buffer. Requires `influx` to be configured. This is intended to be used if there is a local influxdb instance running. If points are received on non-historical points while the connection to CDF is down, they are read from influxdb once the connection is restored" + "description": "Set to `true` to enable the failure buffer.", + "default": false }, "datapoint-path": { "type": "string", - "description": "Path to a binary file. There is no safety, and a bad write can corrupt the file, but it is very fast." + "description": "Path to a binary file where data points are buffered. Buffering to a file is very fast, and can handle a large amount of data. The file will be created for you if it does not already exist.", + "examples": [ + "buffer.bin" + ] }, "event-path": { "type": "string", - "description": "Path to a binary file for storing events. The two buffer paths must be different." + "description": "Path to a binary file for storing events. Buffering to a file is very fast and can handle a large amount of data. The file will be created for you if it does not already exist. Must be different from `datapoint-path`." }, "max-buffer-size": { "type": "integer", - "description": "Maximum size of buffer file in bytes" + "description": "Set the maximum size in bytes of the buffer file. If the size of the file exceeds this number, no new datapoints or events will be written to their respective buffer files, and any further ephemeral data is lost. Note that if both datapoint and event buffers are enabled, the potential disk usage is twice this number." + }, + "influx-state-store": { + "type": "boolean", + "description": "Set to `true` to enable storing the state of the InfluxDB buffer. This makes the InfluxDB buffer persistent even if the OPC UA extractor stops before it is emptied. Requires [`state-storage`](#state-storage) to be configured.", + "default": false + }, + "influx": { + "type": "boolean", + "description": "Set to `true` to enable buffering data in InfluxDB. This requires a configured influx pusher. See [`influx`](#influx). This serves as an alternative to a local file, but should only be used if pushing to InfluxDB is already an requirement.", + "default": false } } } \ No newline at end of file diff --git a/schema/full_config.schema.json b/schema/full_config.schema.json index 7d825c625..72331b85d 100644 --- a/schema/full_config.schema.json +++ b/schema/full_config.schema.json @@ -10,6 +10,11 @@ "minimum": 0, "description": "Version of the config file, the extractor specifies which config file versions are accepted in each version of the extractor." }, + "dry-run": { + "type": "boolean", + "default": false, + "description": "Set this to `true` to prevent the extractor from writing anything to CDF. This is useful for debugging the extractor configuration." + }, "source": { "$ref": "source_config.schema.json" }, diff --git a/schema/history_config.schema.json b/schema/history_config.schema.json index ca29fc4a0..8a7151388 100644 --- a/schema/history_config.schema.json +++ b/schema/history_config.schema.json @@ -6,74 +6,81 @@ "properties": { "enabled": { "type": "boolean", - "description": "Enable/disable history synchronization from the OPC-UA server to CDF" + "description": "Set to `true` to enable reading history from the server. If this is `false` no history will be read, overriding configuration set elsewhere.", + "default": false }, "data": { "type": "boolean", - "description": "Enable or disable reading data point history. `enabled` must be `true`. By default nodes with AccessLevel ReadHistory are read" + "description": "Set to `false` to disable reading history for data points.", + "default": true }, "backfill": { "type": "boolean", - "description": "Enable/disable backfill behavior. If this is enabled, the extractor will read history backwards as well as forwards, which makes it catch up to fresh data more quickly. This is especially useful if there is a lot of history" + "description": "Enable backfill, meaning that data is read backward and forward through history. This makes it so that the extractor will prioritize reading recent values, then keep reading old data in the background while loading live data. This is potentially useful if there is a lot of history.", + "default": false }, "require-historizing": { "type": "boolean", - "description": "`true` to require `Historizing` to be `true` on timeseries in order to read history. `Historizing` means that the node writes history, but not all servers correctly set it" + "description": "Set this to `true` to require the `Historizing` attribute to be `true` on OPC UA variables in order to read history from them.", + "default": false + }, + "restart-period": { + "type": "string", + "description": "Time in seconds between restarts of history. Setting this too low may impact performance. Leave at 0 to disable periodic restarts. Format is as given in [Timestamps and intervals](#timestamps-and-intervals). This option also allows using cron expressions." }, "data-chunk": { "type": "integer", - "description": "Max number of datapoints per variable for each history read request. 0 means that the server decides", + "description": "Maximum number of datapoints per variable for each `HistoryRead` service call Generally, this is limited by the server, so it can safely be set to 0, which means the server decides the number of points to return.", "minimum": 0, "maximum": 100000, "default": 1000 }, "data-nodes-chunk": { "type": "integer", - "description": "Max number of simultaneous nodes per history read request for datapoints", + "description": "Maximum number of variables to query in each `HistoryRead` service call. If [`granularity`](#history.granularity) is set, this is applied _after_ calculating chunks based on history granularity.", "minimum": 1, "maximum": 10000, "default": 100 }, "event-chunk": { "type": "integer", - "description": "Maximum number of events per node for each history read request. 0 means that the server decides", + "description": "Maximum number of events per node for each `HistoryRead` service call for events. Generally, this is limited by the server, so it can safely be set to 0, which means the server decides the number of events to return.", "minimum": 0, "maximum": 100000, "default": 1000 }, "event-nodes-chunk": { "type": "integer", - "description": "Maximum number of simultaneous nodes per history read request for events", + "description": "Maximum number of nodes to query in each `HistoryRead` service call for events.", "minimum": 1, "maximum": 10000, "default": 100 }, - "max-read-length": { + "granularity": { "type": "string", - "description": "Maximum length of time for each history read request. If this is set greater than zero, history will be read in chunks of this size until the end. Format is `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`", - "default": "0" + "description": "Granularity in seconds for chunking history read operations. Variables with latest timestamp within the same granularity chunk will have their history read together. Reading more variables per operation is more efficient, but if the granularity is set too high, then the same history may be read multiple times. A good choice for this value is `60 * average_update_frequency`.\n\nFormat is as given in [Timestamps and intervals](#timestamps-and-intervals).", + "default": "600s" }, "start-time": { "type": "string", - "description": "The earliest timestamp history will be read from, in milliseconds since 01/01/1970. Alternatively use syntax `N[timeunit](-ago)` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`. `-ago` indicates that this is set in the past, if left out it will be that duration in the future", - "default": "0" + "description": "Earliest timestamp to read history from in milliseconds since January 1, 1970. Format is as given in [Timestamps and intervals](#timestamps-and-intervals), `-ago` can be added to make a relative timestamp in the past.", + "default": "0", + "examples": [ + "3d-ago" + ] }, "end-time": { "type": "string", - "description": "The latest timestamp history will be read from. In milliseconds since 01/01/1970. Default is current time. Alternatively use syntax `N[timeunit](-ago)` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`. `-ago` indicates that this is set in the past, if left out it will be that duration in the future" + "description": "The latest timestamp history will be read from. In milliseconds since 01/01/1970. Alternatively use syntax `N[timeunit](-ago)` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`. `-ago` indicates that this is set in the past, if left out it will be that duration in the future." }, - "granularity": { + "max-read-length": { "type": "string", - "description": "Granularity to use when doing history read, in seconds. Nodes with last known timestamp within this range of each other will be read together. Should not be smaller than usual average update rate. Leave at 0 to always read a single node each time. Format is `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`", - "default": "600s" + "description": "Maximum length in time of each history read. If this is greater than zero, history will be read in chunks until the end. This is a workaround for server issues, do _not_ use this unless you have concrete issues with continuation points. Format is as given in [Timestamps and intervals](#timestamps-and-intervals).", + "default": "0" }, "ignore-continuation-points": { "type": "boolean", - "description": "Set to `true` to read history without using ContinuationPoints, instead using the `Time` field on events, and `SourceTimestamp` on datapoints to incrementally increase the start time of the request until no data is returned" - }, - "restart-period": { - "type": "string", - "description": "Time in seconds between restarts of history. Setting this too low may impact performance. Leave at 0 to disable periodic restarts. Format is `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`. You can also use a cron expression on the form `[minute] [hour] [day of month] [month] [day of week]`" + "description": "Set to `true` to read history without using `ContinuationPoints`, instead using the `Time` field on events, and `SourceTimestamp` on datapoints to incrementally increase the start time of the request until no data is returned. This is a workaround for server issues, do _not_ use this unless you have concrete issues with continuation points." }, "throttling": { "$ref": "continuation_point_throttling.schema.json", @@ -81,12 +88,12 @@ }, "log-bad-values": { "type": "boolean", - "description": "Log bad history datapoints: Count per read at Debug level, and each individual datapoint at Verbose level", + "description": "Log the number of bad datapoints per `HistoryRead` call at Debug level, and each individual bad datapoint at Verbose level", "default": true }, "error-threshold": { "type": "number", - "description": "Threshold in percent for a history run to be considered failed. Example: 10.0 -> History read operation would be considered failed if more than 10% of nodes fail to read at some point. Retries still apply, this only applies to nodes that fail even after retries.", + "description": "Threshold in percent for a history run to be considered failed. Example: 10.0 -> History read operation would be considered failed if more than 10% of nodes fail to read at some point. Retries still apply, this only applies to nodes that fail even after retries.\n\nThis is safe in terms data loss. A node that has failed during history will not receive state updates from live values, next time history is read, the extractor will continue from where it last successfully read history.", "default": 10 } } diff --git a/schema/influx_config.schema.json b/schema/influx_config.schema.json index 940212706..8db7e7083 100644 --- a/schema/influx_config.schema.json +++ b/schema/influx_config.schema.json @@ -3,46 +3,41 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "unevaluatedProperties": false, - "description": "Write datapoints and events to an influxdb database", - "required": ["host", "database"], + "description": "Configuration for pushing to an [InfluxDB](https://www.influxdata.com/) database. Data points and events will be pushed, but no context or metadata.", + "required": [ + "host", + "database" + ], "properties": { "host": { "type": "string", - "description": "URL of the host influxdb server" + "description": "URL of the host InfluxDB server" }, "username": { "type": "string", - "description": "Username for the influxdb server" + "description": "The username for connecting to the InfluxDB database." }, "password": { "type": "string", - "description": "Password for the influxdb server" + "description": "The password for connecting to the InfluxDB database." }, "database": { "type": "string", - "description": "Database to connect to on the influxdb server" + "description": "The database to connect to on the InfluxDB server. The database will not be created automatically." }, "point-chunk-size": { "type": "integer", - "description": "Maximum number of points to send in each request to influx", + "description": "Maximum number of points to send in each request to InfluxDB.", "minimum": 1, "default": 100000 }, - "debug": { - "type": "boolean", - "description": "Debug mode. If true, data will not be pushed to target" - }, "read-extracted-ranges": { "type": "boolean", - "description": "Whether to read start/end points on startup, where possible." + "description": "Whether to read start/end points on startup, where possible. It is recommended that you use [`state-storage`](#state-storage) instead." }, "read-extracted-event-ranges": { "type": "boolean", - "description": "Whether to read start/end points for events on startup, where possible" - }, - "non-finite-replacement": { - "type": ["number", "null"], - "description": "Replace all instances of NaN or Infinity with this floating point number" + "description": "Whether to read start/end points for events on startup, where possible. It is recommended that you use [`state-storage`](#state-storage) instead." } } } \ No newline at end of file diff --git a/schema/logger_config.schema.json b/schema/logger_config.schema.json index 2fcf1a9da..34cc08b3b 100644 --- a/schema/logger_config.schema.json +++ b/schema/logger_config.schema.json @@ -4,7 +4,7 @@ "description": "Configure logging to console or file", "allOf": [ { - "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.16.0/schema/logger_config.schema.json" + "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.19.0/schema/logger_config.schema.json" } ], "unevaluatedProperties": false, @@ -20,11 +20,11 @@ "error", "fatal" ], - "description": "Optionally capture OPC-UA trace at this level or higher." + "description": "Capture OPC UA tracing at this level or above." }, "ua-session-tracing": { "type": "string", - "description": "Trace data sent to and received from the OPC-UA server. WARNING: This produces an enormous amount of logs, only use this when running against a small number of nodes, producing a limited number of datapoints, and make sure it is not turned on in production" + "description": "Log data sent to and received from the OPC-UA server.\n\nWARNING: This produces an enormous amount of logs, only use this when running against a small number of nodes, producing a limited number of datapoints, and make sure it is not turned on in production." } } } \ No newline at end of file diff --git a/schema/metrics_config.schema.json b/schema/metrics_config.schema.json index 76955c90e..b23e70572 100644 --- a/schema/metrics_config.schema.json +++ b/schema/metrics_config.schema.json @@ -17,11 +17,11 @@ "properties": { "server-metrics": { "type": "boolean", - "description": "Map a few relevant static diagnostics contained in `ServerDiagnosticsSummary` in OPC-UA" + "description": "Map a few relevant static diagnostics contained in the `Server/ServerDiagnosticsSummary` node to prometheus metrics." }, "other-metrics": { "type": "array", - "description": "List of nodes to map to metrics", + "description": "List of additional nodes to read as metrics.", "items": { "$ref": "proto_node_id.schema.json" } diff --git a/schema/mqtt_config.schema.json b/schema/mqtt_config.schema.json index 93e5da1ed..38254e0b8 100644 --- a/schema/mqtt_config.schema.json +++ b/schema/mqtt_config.schema.json @@ -2,68 +2,87 @@ "$id": "mqtt_config.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "description": "Push data to MQTT. This is intended to be used with the MQTT-CDF Bridge application.", + "description": "Push data to CDF one-way over [MQTT](https://mqtt.org/). This requires that the [MQTT-CDF Bridge](https://github.com/cognitedata/opcua-extractor-net/releases) application is running somewhere with access to CDF.", "unevaluatedProperties": false, - "required": ["host", "port"], + "required": [ + "host", + "port" + ], "properties": { "host": { "type": "string", - "description": "TCP Broker URL" + "description": "The address of the MQTT broker.", + "examples": [ + "localhost" + ] }, "port": { - "type": ["string", "integer"], - "description": "TCP Broker port" + "type": "integer", + "description": "Port to connect to on the MQTT broker.", + "examples": [ + 1883 + ] }, "username": { "type": "string", - "description": "MQTT broker username" + "description": "The MQTT broker username. Leave empty to connect without authentication." }, "password": { "type": "string", - "description": "MQTT broker password" + "description": "The MQTT broker password. Leave empty to connect without authentication." }, "use-tls": { "type": "boolean", - "description": "`true` to enable TLS to the broker" + "description": "Set this to `true` to enable Transport Level Security when communicating with the broker.", + "default": false + }, + "allow-untrusted-certificates": { + "type": "boolean", + "description": "Set this to `true` to allow untrusted SSL certificates when communicating with the broker. This is a security risk, we recommend using `custom-certificate-authority` instead.", + "default": false + }, + "custom-certificate-authority": { + "type": "string", + "description": "Path to certificate file for a certificate authority the broker SSL certificate will be verified against." }, "client-id": { "type": "string", - "description": "MQTT client id. Should be unique for a given broker", + "description": "MQTT client id. Should be unique for a given broker.", "default": "cognite-opcua-extractor" }, "data-set-id": { "type": "integer", - "description": "Data set to use for new objects. Existing objects will not be updated" + "description": "Data set to use for new assets, relationships, events, and time series. Existing objects will not be updated." }, "asset-topic": { "type": "string", - "description": "Topic to publish assets on", + "description": "Topic to publish assets on.", "default": "cognite/opcua/assets" }, "ts-topic": { "type": "string", - "description": "Topic to publish timeseries on", - "default": "cognite/opcua/assets" + "description": "Topic to publish timeseries on.", + "default": "cognite/opcua/timeseries" }, "event-topic": { "type": "string", - "description": "Topic to publish events on", - "default": "cognite/opcua/assets" + "description": "Topic to publish events on.", + "default": "cognite/opcua/events" }, "datapoint-topic": { "type": "string", - "description": "Topic to publish datapoints on", - "default": "cognite/opcua/assets" + "description": "Topic to publish datapoints on.", + "default": "cognite/opcua/datapoints" }, "raw-topic": { "type": "string", - "description": "Topic to publish raw rows on", - "default": "cognite/opcua/assets" + "description": "Topic to publish raw rows on.", + "default": "cognite/opcua/raw" }, - "relationships-topic": { + "relationship-topic": { "type": "string", - "description": "Topic to publish relationships on", - "default": "cognite/opcua/assets" + "description": "Topic to publish relationships on.", + "default": "cognite/opcua/relationships" }, "local-state": { "type": "string", @@ -71,11 +90,7 @@ }, "invalidate-before": { "type": "integer", - "description": "Timestamp in ms since epoch to invalidate stored mqtt states. On extractor restart, assets/timeseries created before this will be re-created in CDF. They will not be deleted or updated." - }, - "debug": { - "type": "boolean", - "description": "If `true`, data will not be pushed to mqtt" + "description": "Timestamp in ms since epoch to invalidate stored mqtt states. On extractor restart, assets/timeseries created before this will be re-created in CDF. They will not be deleted or updated. Requires the [`state-storage`](#state-storage) to be enabled." }, "skip-metadata": { "type": "boolean", @@ -83,67 +98,78 @@ }, "raw-metadata": { "type": "object", - "description": "Store assets/timeseries metadata and relationships in raw. Assets will not be created at all, timeseries will be created with just `externalId`, `isStep`, and `isString`. Both timeseries and assets will be persisted in their entirity to CDF Raw. Datapoints are not affected, events will be created but without being contextualized to assets. The externalId of the source node is added to metadata if applicable", + "description": "Store assets/timeseries metadata and relationships in raw. Assets will not be created at all, timeseries will be created with just `externalId`, `isStep`, and `isString`. Both timeseries and assets will be persisted in their entirity to CDF Raw. Datapoints are not affected.\nEvents will be created but without being contextualized to assets. The external ID of the source node is added to metadata if applicable", "unevaluatedProperties": false, - "required": ["database"], + "required": [ + "database" + ], "properties": { "database": { "type": "string", - "description": "Raw database" + "description": "Raw database to write metadata to." }, "assets-table": { "type": "string", - "description": "Raw table to use for assets" + "description": "Raw table to use for assets." }, "timeseries-table": { "type": "string", - "description": "Raw table to use for timeseries" + "description": "Raw table to use for timeseries." }, "relationships-table": { "type": "string", - "description": "Raw table to use for relationships" + "description": "Raw table to use for relationships." } } }, "metadata-mapping": { "type": "object", - "description": "Map metadata to asset/timeseries attributes. This lets you assign properties in OPC-UA directly to fields like `description` and `unit` in CDF", - "examples": [{ - "timeseries": { - "EngineeringUnits": "unit", - "EURange": "description" - }, - "assets": { - "Name": "name" + "description": "Define mappings between properties in OPC UA and CDF attributes. For example, it is quite common for variables in OPC UA to have a property named `EngineeringUnits`, which ideally should be mapped to `unit` in CDF. This property lets you do that.", + "examples": [ + { + "timeseries": { + "EngineeringUnits": "unit", + "EURange": "description" + }, + "assets": { + "Name": "name" + } } - }], + ], "unevaluatedProperties": false, "properties": { "assets": { "type": "object", - "description": "Map metadata for assets", + "description": "Map metadata for assets.", "patternProperties": { ".*": { "type": "string", - "enum": ["description", "name", "parentId"] + "description": "Target asset attribute", + "enum": [ + "description", + "name", + "parentId" + ] } } }, "timeseries": { "type": "object", - "description": "Map metadata for timeseries", + "description": "Map metadata for time series.", "patternProperties": { ".*": { "type": "string", - "enum": ["description", "name", "parentId", "unit"] + "description": "Target time series attribute", + "enum": [ + "description", + "name", + "parentId", + "unit" + ] } } } } - }, - "non-finite-replacement": { - "type": ["number", "null"], - "description": "Replace all instances of NaN or Infinity with this floating point number" } } } \ No newline at end of file diff --git a/schema/proto_node_id.schema.json b/schema/proto_node_id.schema.json index f96a20313..67b90a498 100644 --- a/schema/proto_node_id.schema.json +++ b/schema/proto_node_id.schema.json @@ -3,6 +3,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "unevaluatedProperties": false, + "title": "Proto Node ID", "properties": { "namespace-uri": { "type": "string", @@ -13,4 +14,4 @@ "description": "Identifier of the node id, on the form i=123, s=string, etc. See the OPC-UA standard" } } -} \ No newline at end of file +} diff --git a/schema/source_config.schema.json b/schema/source_config.schema.json index c49b8a973..62fb7fb03 100644 --- a/schema/source_config.schema.json +++ b/schema/source_config.schema.json @@ -6,19 +6,22 @@ "properties": { "endpoint-url": { "type": "string", - "description": "URL of OPC-UA server endpoint or discovery server" + "examples": [ + "opc.tcp://some-host:1883" + ], + "description": "The URL of the OPC UA server to connect to.\n\nIn practice, this is the URL of the discovery server, where multiple levels of severity may be provided. The OPC UA extractor attempts to use the highest security possible based on the configuration." }, "alt-endpoint-urls": { "type": "array", - "description": "List of alternate endpoint URLs. Enables server redundancy. Should not contain `endpoint-url`", + "description": "List of alternative endpoint URLs the extractor can attempt when connecting to the server. Use this for non-transparent redundancy. See the [OPC UA standard part 4, section 6.6.2](https://reference.opcfoundation.org/v104/Core/docs/Part4/6.6.2/).\n\nWe recommend setting `force-restart` to `true`. Otherwise, the extractor will reconnect to the same server each time.", "items": { "type": "string", - "description": "URL of OPC-UA server endpoint" + "description": "URL of OPC-UA server endpoint. See `endpoint-url`." } }, "endpoint-details": { "type": "object", - "description": "Details used to override default endpoint behavior", + "description": "Details used to override default endpoint behavior. This is used to make the client connect directly to an OPC UA endpoint, for example if the server is behind NAT (Network Address Translation), circumventing server discovery.", "unevaluatedProperties": false, "properties": { "override-endpoint-url": { @@ -29,7 +32,7 @@ }, "redundancy": { "type": "object", - "description": "Extra configuration options related to redundancy and ServiceLevel monitoring", + "description": "Additional configuration options related to redundant servers. The OPC UA extractor supports _Cold_ redundancy, as described in the [OPC UA standard part 4, section 6.6.2](https://reference.opcfoundation.org/v104/Core/docs/Part4/6.6.2/).", "unevaluatedProperties": false, "properties": { "service-level-threshold": { @@ -37,92 +40,39 @@ "minimum": 0, "maximum": 255, "default": 200, - "description": "Servers above this threshold are considered good" + "description": "Servers above this threshold are considered live. If the server drops below this level, the extractor will switch, provided `monitor-service-level` is set to `true`." }, "reconnect-interval": { "type": "string", - "description": "If using redundancy, the extractor will attempt to find a better server with this interval if service level is below threshold. Format is `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`.", + "description": "If using redundancy, the extractor will attempt to find a better server with this interval if service level is below threshold. Format is as given in [Timestamps and intervals](#timestamps-and-intervals).", "default": "10m" }, "monitor-service-level": { "type": "boolean", - "description": "If true, the extractor will subscribe to changes in ServiceLevel and trigger reconnect once it drops below service-level-threshold, and history restart once it goes back above the threshold. This also makes the server not update extraction state if it is connected to a server with service level below the threshold. If a server does not want the extractor to trust the available data ranges it should set the ServiceLevel below the threshold." + "description": "If `true`, the extractor will subscribe to changes in `ServiceLevel` and attempt to change server once it drops below [`service-level-threshold`](#source.redundancy.service-level-threshold).\n\nThis also prevents the extractor from updating states while service level is below the threshold, letting servers inform the extractor that they are not receiving data from all sources, and history should not be trusted. Once the service level goes back above the threshold, history will be read to fill any gaps." } } }, + "reverse-connect-url": { + "type": "string", + "description": "The local URL used for reverse connect, which means that the server is responsible for initiating connections, not the extractor. This lets the server be behind a firewall, forbidding incoming connections. You must also specify an [`endpoint-url`](#source.endpoint-url), to indicate to the extractor where it should accept connections from." + }, "auto-accept": { "type": "boolean", "default": true, - "description": "True to auto accept untrusted certificates. If this is set to false, server certificates must be trusted by manually moving them to the \"trusted\" certificates folder" - }, - "publishing-interval": { - "type": "integer", - "description": "Interval between OPC-UA subscription publish requests in milliseconds. This is how frequentely the extractor requests updates from the server.", - "minimum": 0, - "default": 500 + "description": "Set to `true` to automatically accept server certificates.\n\nIf this is disabled, received server certificates will be placed in the rejected certificates folder (by default `application_dir/certificates/pki/rejected`), and you can manually move them to te accepted certificates folder (`application_dir/certificates/pki/accepted`). Setting this to `true` makes the extractor move certificates automatically.\n\nA simple solution would be to set this to `true` for the first connection, then change it to `false`.\n\nWarning: This should be disabled if the extractor is running on an untrusted network." }, "username": { "type": "string", - "description": "OPC-UA username" + "description": "OPC UA server username, leave empty to disable username/password authentication." }, "password": { "type": "string", - "description": "OPC-UA password" - }, - "secure": { - "type": "boolean", - "description": "True if the extractor should try to connect to a secure endpoint on the server" - }, - "force-restart": { - "type": "boolean", - "description": "True if the extractor should disconnect from the server instead of using built-in OPC-UA reconnect mechanisms. Some servers handle build in reconnect poorly, in which case this is a useful workaround. If using server redundancy through `alt-endpoint-urls`, this should be set to `true`" - }, - "exit-on-failure": { - "type": "boolean", - "description": "True if the extractor should quit completely when it fails to start, instead of restarting." - }, - "browse-nodes-chunk": { - "type": "integer", - "description": "Number of nodes per browse request. Large numbers are likely to exceed the server's tolerance. Lower numbers greatly increase startup time.", - "default": 1000, - "minimum": 1, - "maximum": 100000 - }, - "browse-chunk": { - "type": "integer", - "description": "Number of maximum requested results per node during browse. The server may decide to return fewer. Setting this lower increases startup times. Setting it to 0 leaves the decision entirely up to the server.", - "minimum": 0, - "maximum": 100000, - "default": 1000 - }, - "attributes-chunk": { - "type": "integer", - "description": "Number of attributes per request. The extractor will read 5-10 attributes per node, so setting this too low will massively increase startup times. Setting it too high will likely exceed server limits.", - "default": 10000, - "minimum": 1, - "maximum": 1000000 - }, - "subscription-chunk": { - "type": "integer", - "description": "Number of monitored items to create per request. Setting this lower will increase startup times. High values may exceed server limits.", - "minimum": 1, - "maximum": 100000, - "default": 1000 - }, - "keep-alive-interval": { - "type": "integer", - "description": "Time between each keep-alive request to the server, in milliseconds. The third failed keep-alive request will time out the extractor and trigger a reconnect. Setting this lower will trigger reconnect logic faster. Setting this too high will make it meaningless.", - "minimum": 100, - "maximum": 60000, - "default": 5000 - }, - "restart-on-reconnect": { - "type": "boolean", - "description": "Restart the extractor on reconnect, browsing the node hierarchy and recreating subscriptions. This may be necessary if the server is expected to change after reconnecting, but it may be too expensive if the connection to the server is lost frequently" + "description": "OPC UA server password." }, "x509-certificate": { "type": "object", - "description": "Configure settings for using an x509-certificate as login credentials. This is separate from the application certificate, and used for servers with automatic systems for authentication.", + "description": "Specifies the configuration for using a signed x509 certificate to connect to the server. Note that this is highly server specific. The extractor uses the provided certificate to sign requests sent to the server. The server must have a mechanism to validate this signature. Typically the certificate must be provided by the server.", "unevaluatedProperties": false, "properties": { "file-name": { @@ -131,87 +81,200 @@ }, "password": { "type": "string", - "description": "Password to local x509-certificate file" + "description": "Password for local x509-certificate file" }, "store": { "type": "string", - "description": "Local certificate store to use. One of `None` (to use a file), `Local` (for LocalMachine) or `User`", + "description": "Local certificate store to use. One of `None` (to use a file), `Local` (for LocalMachine) or `User` for the `User` store.", "default": "None", - "enum": ["None", "Local", "User"] + "enum": [ + "None", + "Local", + "User" + ] }, "cert-name": { "type": "string", - "description": "Name of certificate in store, e.g. CN=my-certificate" + "description": "Name of certificate in store. Required to use `store`", + "examples": [ + "CN=MyCertificate" + ] } - } + }, + "oneOf": [ + { + "required": [ + "file-name" + ] + }, + { + "required": [ + "store", + "cert-name" + ] + } + ] }, - "reverse-connect-url": { - "type": "string", - "description": "Local URL used for reverse-connect. This is the URL the server should connect to. An endpoint-url should also be provided so that the extractor knows where it should accept requests from. The server is responsible for initiating connections" + "secure": { + "type": "boolean", + "default": false, + "description": "Set this to `true` to make the extractor try to connect to an endpoint with security above `None`. If this is enabled, the extractor will try to pick the _most secure_ endpoint, meaning the endpoint with the longest of the most modern cipher types." }, "ignore-certificate-issues": { "type": "boolean", - "description": "Ignore all issues caused by the server certificate. This potentially opens the extractor up to a man-in-the-middle attack, but can be used if the server is noncompliant and on a closed network" + "description": "Ignore all suppressible certificate errors on the server certificate. You can use this setting if you receive errors such as _Certificate use not allowed_.\n\nCAUTION: This is potentially a security risk. Bad certificates can open the extractor to man-in-the-middle attacks or similar. If the server is secured in other ways (it is running locally, over a secure VPN, or similar), it is most likely fairly harmless.\n\nSome errors are not suppressible and must be remedied on the server. Note that enabling this is always a workaround for the server violating the OPC UA standard in some way.", + "default": false }, - "browse-throttling": { - "$ref": "continuation_point_throttling.schema.json", - "description": "Settings for throttling browse operations" + "publishing-interval": { + "type": "integer", + "description": "Sets the interval (in milliseconds) between publish requests to the server, which is when the extractor asks the server for updates to any active subscriptions.\n\nThis limits the maximum frequency of points pushed to CDF, but not the maximum frequency of points on the server. In most cases, this can be set to the same as [`extraction.data-push-delay`](#extraction.data-push-delay). If set to `0` the server chooses the interval to be as low as it supports. Be aware that some servers set this lower limit very low, which may create considerable load on the server.", + "minimum": 0, + "default": 500 + }, + "force-restart": { + "type": "boolean", + "description": "If `true`, the extractor will not attempt to reconnect using the OPC UA reconnect protocol if connection is lost, but instead always create a new connection. Only enable this if reconnect is causing issues with the server. Even if this is disabled, the extractor will generally fall back on regular reconnects if the server produces unexpected errors on reconnect.", + "default": false + }, + "exit-on-failure": { + "type": "boolean", + "description": "If `true`, the OPC UA extractor will be restarted completely on reconnect. Enable this if the server is expected to change dramatically while running, and the extractor cannot keep using state from previous runs.", + "default": false + }, + "keep-alive-interval": { + "type": "integer", + "description": "Specifies the interval in milliseconds between each keep-alive request to the server. The connection times out if a keep-alive request fails twice (`2 * interval + 100ms`). This typically happens if the server is down, or if it is hanging on a heavy operation and doesn't manage to respond to keep alive requests. Set this higher if keep alives often time out without the server being down.", + "minimum": 100, + "maximum": 60000, + "default": 5000 + }, + "restart-on-reconnect": { + "type": "boolean", + "description": "If `true`, the OPC UA extractor will be restarted after reconnecting to the server. This may not be required if the server is the server is expected to not change much, and that it handles reconnects well.", + "default": false }, "node-set-source": { "type": "object", - "description": "Configuration for using NodeSet2 files as sources for the OPC-uA node hierarchy instead of browsing the server. This can be used if the server structure is well known and fixed. Values of nodes are still read from the server", + "description": "Read from NodeSet2 files instead of browsing the OPC UA node hierarchy. This is useful for certain smaller servers, where the full node hierarchy is known before-hand. In general, it can be used to lower the load on the server.", "unevaluatedProperties": false, - "anyOf": [{ "required": ["instance"] }, { "required": ["type"] }], - "required": ["node-sets"], + "anyOf": [ + { + "required": [ + "instance" + ] + }, + { + "required": [ + "type" + ] + } + ], + "required": [ + "node-sets" + ], "properties": { "node-sets": { "type": "array", - "description": "List of nodesets to read. Specified by URL, file name, or both. If no name is specified, the last segment of the URL is used as file name. File name is where downloaded files are saved, and where the extractor looks for existing files.", + "description": "List of nodesets to read. Specified by URL, file name, or both. If no name is specified, the last segment of the URL is used as file name. File name is where downloaded files are saved, and where the extractor looks for existing files.\n\nNote that typically, you will need to define all schemas your server schema depends on. All servers should depend on the base OPC UA schema, so you should always include `https://files.opcfoundation.org/schemas/UA/1.04/Opc.Ua.NodeSet2.xml`", + "examples": [ + [ + { + "file-name": "Server.NodeSet2.xml" + }, + { + "url": "https://files.opcfoundation.org/schemas/UA/1.04/Opc.Ua.NodeSet2.xml" + } + ] + ], "items": { "type": "object", - "description": "Configuration specifying a node set file", + "description": "Configuration specifying a node set file.", "unevaluatedProperties": false, - "anyOf": [{ "required": ["file-name"] }, { "required": ["url"] }], + "anyOf": [ + { + "required": [ + "file-name" + ] + }, + { + "required": [ + "url" + ] + } + ], "properties": { "file-name": { "type": "string", - "description": "Path to nodeset file" + "description": "Path to nodeset file. This is either the place where the downloaded file is saved, or a previously downloaded file." }, "url": { "type": "string", - "description": "URL of publicly available nodeset file" + "description": "URL of publicly available nodeset file." } } } }, "instance": { "type": "boolean", - "description": "Use nodeset files to replace the OPC-UA instance hierarchy, i.e. everything under \"Objects\" in the OPC-A server" + "description": "If `true`, the instance hierarchy is not obtained from the server, but instead read from the NodeSet2 files." }, "types": { "type": "boolean", - "description": "Use nodeset files to replace the OPC-UA type hierarchy" + "description": "If `true`, event types, reference types, object types, and variable types are obtained from NodeSet2 files instead of the server." } } }, + "alt-source-background-browse": { + "type": "boolean", + "description": "If `true`, browses the OPC UA node hierarchy in the background when obtaining nodes from an alternative source, such as CDF Raw or NodeSet2 files." + }, "limit-to-server-config": { "type": "boolean", - "description": "Limit chunking values based on exposed information from the server, if any. This can be set to false if the server info is known to be wrong, but should generally be left on, as exceeding configured server limits will almost certainly cause a crash", + "description": "Uses the `Server/ServerCapabilities` node in the OPC UA server to limit chunk sizes. Set this to `false` only if you know the server reports incorrect limits and you want to set them higher. If the real server limits are exceeded, the extractor will typically crash.", "default": true }, - "alt-source-background-browse": { - "type": "boolean", - "description": "If an alternative source for the node hierarchy is used, like CDF or nodeset files, this can be set to true to also browse the node hierarchy in the background. This is useful to start the extractor quickly but also discover everything in the server" + "browse-nodes-chunk": { + "type": "integer", + "description": "Sets the maximum number of nodes per call to the Browse service. Large numbers are likely to exceed the server's tolerance. Lower numbers greatly increase startup time.", + "default": 1000, + "minimum": 1, + "maximum": 100000 + }, + "browse-chunk": { + "type": "integer", + "description": "Sets the maximum requested results per node for each call to the Browse service. The server may decide to return fewer. Setting this lower increases startup times. Setting it to 0 leaves the decision up to the server.", + "minimum": 0, + "maximum": 100000, + "default": 1000 + }, + "attributes-chunk": { + "type": "integer", + "description": "Specifies the maximum number of attributes to fetch per call to the Read service. If the server fails with `TooManyOperations` during attribute read, it may help to lower this value. This should be set as high as possible for large servers.", + "default": 10000, + "minimum": 1, + "maximum": 1000000 + }, + "subscription-chunk": { + "type": "integer", + "description": "Sets the maximum number of new `MonitoredItems` to create per operation. If the server fails with `TooManyOperations` when creating monitored items, try lowering this value.", + "minimum": 1, + "maximum": 100000, + "default": 1000 + }, + "browse-throttling": { + "$ref": "continuation_point_throttling.schema.json", + "description": "Settings for throttling browse operations." }, "certificate-expiry": { "type": "integer", "minimum": 1, "maximum": 65535, "default": 60, - "description": "Default application certificate expiry in months" + "description": "Specifies the default application certificate expiration time in months. You can also replace the certificate manually by modifying the `opc.ua.net.extractor.Config.xml` configuration file. Note that the default values was changed as of version 2.5.3." }, "retries": { - "$ref": "ua_retries_config.schema.json" + "$ref": "ua_retries_config.schema.json", + "description": "Specify the retry policy for requests to the OPC UA server." } } } \ No newline at end of file diff --git a/schema/state_storage_config.schema.json b/schema/state_storage_config.schema.json index cc9cb30ef..f011631d9 100644 --- a/schema/state_storage_config.schema.json +++ b/schema/state_storage_config.schema.json @@ -2,37 +2,37 @@ "$id": "state_storage_config.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "description": "Configuration for storing the range of extracted datapoints and events to a persistent store", + "description": "A local LiteDB database or a database in CDF RAW that store various persistent information between extractor runs. This is used to replace reading first/last data points from CDF, and also allows storing first/last times for events.\n\nEnabling this is highly recommended, and will be required in a future version of the extractor.", "unevaluatedProperties": false, "allOf": [ { - "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.16.0/schema/state_store_config.schema.json" + "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.19.0/schema/state_store_config.schema.json" } ], "properties": { "interval": { "type": "string", - "description": "Interval between each write to the buffer file, in seconds. 0 or less disables the state store. Alternatively use `N[timeunit]` where `timeunit` is one of `w`, `d`, `h`, `m`, `s`, or `ms`", + "description": "Interval between each write to the buffer file, in seconds. 0 or less disables the state store. Format is as given in [Timestamps and intervals](#timestamps-and-intervals).", "default": "0s" }, "variable-store": { "type": "string", - "description": "Name of raw table or litedb store for variable ranges", + "description": "Name of raw table or litedb collection to store information about extracted OPC UA variables.", "default": "variable_states" }, "event-store": { "type": "string", - "description": "Name of raw table or litedb store for event ranges", + "description": "Name of raw table or litedb collection to store information about extracted events.", "default": "event_states" }, "influx-variable-store": { "type": "string", - "description": "Name of raw table or litedb store for influxdb failure buffer variable ranges", + "description": "Name of raw table or litedb collection to store information about variable ranges in the InfluxDB failure buffer.", "default": "influx_variable_states" }, "influx-event-store": { "type": "string", - "description": "Name of raw table or litedb store for influxdb failure buffer event ranges", + "description": "Name of raw table or litedb collection to store information about events in the InfluxDB failure buffer.", "default": "influx_event_states" } } diff --git a/schema/subscription_config.schema.json b/schema/subscription_config.schema.json index 542a49405..1055ecf61 100644 --- a/schema/subscription_config.schema.json +++ b/schema/subscription_config.schema.json @@ -3,11 +3,11 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "unevaluatedProperties": false, - "description": "Configuration for the subscriptions created on the server.", + "description": "A few options for subscriptions to events and data points. Subscriptions in OPC UA consist of Subscription objects on the server, which contain a list of MonitoredItems. By default, the extractor produces a maximum of four subscriptions:\n * DataChangeListener - handles data point subscriptions.\n * EventListener - handles event subscriptions.\n * AuditListener - which handles audit events.\n * NodeMetrics - which handles subscriptions for use as metrics.\n\nEach of these can contain a number of MonitoredItems.", "properties": { "data-change-filter": { "type": "object", - "description": "Modify the DataChangeFilter used for datapoint subscriptions. See OPC-UA reference part 4 7.17.2 for details. These are just passed to hte server, they have no further effect on extractor behavior. Filters are applied to all noes, but deadband should only affect some, according to the standard.", + "description": "Modify the DataChangeFilter used for datapoint subscriptions. See the OPC UA reference part 4 7.17.2 for details. These are just passed to the server, they have no further effect on extractor behavior. Filters are applied to all nodes, but deadband should only affect some, according to the standard.", "unevaluatedProperties": false, "properties": { "trigger": { @@ -38,24 +38,24 @@ }, "sampling-interval": { "type": "integer", - "description": "Requested sample interval per variable on the server. This is how often the extractor requests the server sample changes to values. The server has no obligation to use this value, or to use any form of sampling at all, but according to the standard this should limit the _lowest_ allowed rate of updates.", + "description": "Requested sample interval per variable on the server. This is how often the extractor requests the server sample changes to values. The server has no obligation to use this value, or to use any form of sampling at all, but according to the standard this should limit the _lowest_ allowed rate of updates. 0 tells the server to sample as fast as possible.", "default": 100, "minimum": 0 }, "queue-length": { "type": "integer", - "description": "Requested length of queue for each variable on the server. This is how many data points the server will buffer. It should in general be set to at least 2 * `publishing-interval` / `sampling-interval`", + "description": "Requested length of queue for each variable on the server. This is how many data points the server will buffer. It should in general be set to at least 2 * `publishing-interval` / `sampling-interval`\n\nThis setting generally sets the maximum rate of points from the server (in milliseconds). On many servers, sampling is an internal operation, but on some, this may access external systems. Setting this very low can increase the load on the server significantly.", "default": 100, "minimum": 0 }, "data-points": { "type": "boolean", - "description": "Enable subscriptions on data-points", + "description": "Enable subscriptions on datapoints.", "default": true }, "events": { "type": "boolean", - "description": "Enable subscriptions on events. Requires `events.enabled` to be set to `true`", + "description": "Enable subscriptions on events. Requires [`events.enabled`](#events) to be set to `true`", "default": true }, "ignore-access-level": { @@ -64,7 +64,7 @@ }, "log-bad-values": { "type": "boolean", - "description": "Log bad subscription datapoints", + "description": "Log bad subscription datapoints.", "default": true }, "keep-alive-count": { @@ -82,12 +82,12 @@ "recreate-stopped-subscriptions": { "type": "boolean", "default": true, - "description": "Recreate subscriptions that have stopped publishing. True by default." + "description": "Recreate subscriptions that have stopped publishing." }, "recreate-subscription-grace-period": { "type": "string", "default": "-1", - "description": "Grace period for recreating stopped subscriptions. If this is negative, default to 8 * publishing-interval. Syntax is `N[timeunit]` where timeunit is `w`, `d`, `h`, `m`, `s`, or `ms`" + "description": "Grace period for recreating stopped subscriptions. If this is negative, default to 8 * publishing-interval. Format is as given in [Timestamps and intervals](#timestamps-and-intervals)." }, "alternative-configs": { "type": "array", @@ -104,7 +104,7 @@ "properties": { "id": { "type": "string", - "descritpion": "Regex match on node external ID" + "descritpion": "Regex on node external ID" }, "data-type": { "type": "string", @@ -121,7 +121,7 @@ }, "data-change-filter": { "type": "object", - "description": "Modify the DataChangeFilter used for datapoint subscriptions. See OPC-UA reference part 4 7.17.2 for details. These are just passed to hte server, they have no further effect on extractor behavior. Filters are applied to all noes, but deadband should only affect some, according to the standard.", + "description": "Modify the DataChangeFilter used for datapoint subscriptions. See the OPC UA reference part 4 7.17.2 for details. These are just passed to the server, they have no further effect on extractor behavior. Filters are applied to all nodes, but deadband should only affect some, according to the standard.", "unevaluatedProperties": false, "properties": { "trigger": { @@ -152,13 +152,13 @@ }, "sampling-interval": { "type": "integer", - "description": "Requested sample interval per variable on the server. This is how often the extractor requests the server sample changes to values. The server has no obligation to use this value, or to use any form of sampling at all, but according to the standard this should limit the _lowest_ allowed rate of updates.", + "description": "Requested sample interval per variable on the server. This is how often the extractor requests the server sample changes to values. The server has no obligation to use this value, or to use any form of sampling at all, but according to the standard this should limit the _lowest_ allowed rate of updates. 0 tells the server to sample as fast as possible.", "default": 100, "minimum": 0 }, "queue-length": { "type": "integer", - "description": "Requested length of queue for each variable on the server. This is how many data points the server will buffer. It should in general be set to at least 2 * `publishing-interval` / `sampling-interval`", + "description": "Requested length of queue for each variable on the server. This is how many data points the server will buffer. It should in general be set to at least 2 * `publishing-interval` / `sampling-interval`\n\nThis setting generally sets the maximum rate of points from the server (in milliseconds). On many servers, sampling is an internal operation, but on some, this may access external systems. Setting this very low can increase the load on the server significantly.", "default": 100, "minimum": 0 } diff --git a/schema/ua_retries_config.schema.json b/schema/ua_retries_config.schema.json index 651af898c..76dc9d4ac 100644 --- a/schema/ua_retries_config.schema.json +++ b/schema/ua_retries_config.schema.json @@ -5,7 +5,7 @@ "description": "Configuration for retries on OPC-UA service calls", "allOf": [ { - "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.16.0/schema/retry_config.schema.json" + "$ref": "https://raw.githubusercontent.com/cognitedata/dotnet-extractor-utils/v1.19.0/schema/retry_config.schema.json" } ], "unevaluatedProperties": false, @@ -13,7 +13,7 @@ "retry-status-codes": { "type": "array", "uniqueItems": true, - "description": "List of additional OPC-UA status codes to retry on. In additional to defaults. Should be integer values from http://www.opcfoundation.org/UA/schemas/StatusCode.csv, or symbolic names as shown in the same csv file.", + "description": "List of additional OPC-UA status codes to retry on. In additional to defaults. Should be integer values from http://www.opcfoundation.org/UA/schemas/StatusCode.csv, or symbolic names as shown in the same .csv file.", "items": { "type": "string" }