From 6dafdf2e5b357c6eec90a004178eaed1eb109c32 Mon Sep 17 00:00:00 2001 From: c-ehrlich Date: Thu, 12 Sep 2024 13:06:56 +0200 Subject: [PATCH 1/6] create very rough first draft of map fields docs --- apl/data-types/map-fields.mdx | 61 +++++++++++++++++++++++++++++++++++ mint.json | 15 +++------ 2 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 apl/data-types/map-fields.mdx diff --git a/apl/data-types/map-fields.mdx b/apl/data-types/map-fields.mdx new file mode 100644 index 00000000..cc796c27 --- /dev/null +++ b/apl/data-types/map-fields.mdx @@ -0,0 +1,61 @@ +--- +title: 'Map fields' +description: 'Learn about the purpose of map fields, and how to ingest and query them.' +sidebarTitle: Map Fields +tags: ['axiom documentation', 'documentation', 'axiom', 'map fields', 'object fields', 'data types'] +--- + +TODO: is this the right location for this page? i could also see it after the "Virtual Fields" page (Understanding Data => Query => Map Fields) + +Map fields are a special type of field that hold a collection of nested key-value pairs within a single field. They are useful for storing and manipulating structured data, such as JSON objects or XML elements. + +## Why + +The purpose of map fields is allow storing additional attributes without adding more fields, particularly in situations where the shape of your data is unpredictable, such as additional attributes added by OpenTelemetry instrumentation. This can be useful to prevent you from reaching the field limit of a dataset, and to improve query performance. + +## `attributes.custom` in Tracing Datasets + +If you are ingesting tracing data using the [OTLP/HTTP endpoint for traces and logs](/send-data/opentelemetry), you may have noticed that some attributes are moved into the `attributes.custom` map field. + +Any instrumentation library can add arbitrary attributes to spans, and in many cases a single piece of auto-instrumentation may add dozens or even hundreds of attributes. For a large system with a wide range of instrumentation, this may add up to thousands of attributes. + +In order to keep the dimensionality of your dataset under control, Axiom preserves all attributes that are described in the [OpenTelemetry Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/general/trace/), and puts everything else into the `attributes.custom` map field. + +## Querying map fields + +To access a field in APL, you might do something like: + +```kusto +['otel-demo-traces'] +| where ['attributes.http.url'] == 'http://opentelemetry-demo-frontendproxy:8080/api/cart' +``` +[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.http.url%27%5d%20%3d%3d%20%27http%3a%2f%2fopentelemetry-demo-frontendproxy%3a8080%2fapi%2fcart%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) + +To access `http.protocol` inside the `attributes.custom` map field, the syntax is `['']['']` instead: + +```kusto +['otel-demo-traces'] +| where ['attributes.custom']['http.protocol'] == 'HTTP/1.1' +``` +[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.custom%27%5d%5b%27http.protocol%27%5d%20%3d%3d%20%27HTTP%2f1.1%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) + +## Ingesting map fields + +TODO: mhr3 suggested not to tell customers about X-Axiom-Object-Fields, need to make a decision about that + +If you would like to manually create a map field in order to prevent unexpected json payload shape from creating new fields in your dataset, you can use the `X-Axiom-Object-Fields` header. See an example request below: + +``` +curl -X 'POST' 'https://api.axiom.co/v1/datasets/$DATASET_NAME/ingest' \ + -H 'Authorization: $API_TOKEN' \ + -H 'Content-Type: application/json' \ + -H 'X-Axiom-Object-Fields: object' \ + -d '[ + { + "normal": { "foo": { "bar": "baz" }, { "biz": "qux" } }, + "object": { "foo": { "bar": "baz" }, { "biz": "qux" } }, + } +]' +``` + +This will create the fields `normal.foo.bar`, `normal.foo.biz`, and `object`. diff --git a/mint.json b/mint.json index 2f6487ec..52d0d764 100644 --- a/mint.json +++ b/mint.json @@ -163,10 +163,7 @@ "pages": [ { "group": "Flow", - "pages": [ - "process-data/introduction", - "process-data/flows" - ] + "pages": ["process-data/introduction", "process-data/flows"] }, { "group": "Destinations", @@ -222,10 +219,7 @@ }, { "group": "Get Started", - "pages": [ - "apl/introduction", - "apl/tutorial" - ] + "pages": ["apl/introduction", "apl/tutorial"] }, { "group": "Functions", @@ -294,6 +288,7 @@ "apl/entities/entity-names", "apl/data-types/null-values", "apl/data-types/scalar-data-types", + "apl/data-types/map-fields", "apl/query-statement/set-statement", "apl/guides/exploring-the-union-operator-in-apl", "apl/reference/special-field-attributes" @@ -326,9 +321,7 @@ }, { "group": "User Endpoints", - "pages": [ - "restapi/endpoints/getCurrentUser" - ] + "pages": ["restapi/endpoints/getCurrentUser"] }, { "group": "Dataset Endpoints", From cf26a742eda574ee94d8af32c39315ecb27f6096 Mon Sep 17 00:00:00 2001 From: c-ehrlich Date: Thu, 12 Sep 2024 13:18:23 +0200 Subject: [PATCH 2/6] remove unnecessary sentence --- apl/data-types/map-fields.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apl/data-types/map-fields.mdx b/apl/data-types/map-fields.mdx index cc796c27..0a068536 100644 --- a/apl/data-types/map-fields.mdx +++ b/apl/data-types/map-fields.mdx @@ -7,7 +7,7 @@ tags: ['axiom documentation', 'documentation', 'axiom', 'map fields', 'object fi TODO: is this the right location for this page? i could also see it after the "Virtual Fields" page (Understanding Data => Query => Map Fields) -Map fields are a special type of field that hold a collection of nested key-value pairs within a single field. They are useful for storing and manipulating structured data, such as JSON objects or XML elements. +Map fields are a special type of field that can hold a collection of nested key-value pairs within a single field. You can think of the content of a map field as a JSON object. ## Why From ed58e1f72c3f981e139c9bbf38a9c826badbbf41 Mon Sep 17 00:00:00 2001 From: c-ehrlich Date: Thu, 12 Sep 2024 13:19:56 +0200 Subject: [PATCH 3/6] show that X-Axiom-Object-Fields can be a list --- apl/data-types/map-fields.mdx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apl/data-types/map-fields.mdx b/apl/data-types/map-fields.mdx index 0a068536..b5038170 100644 --- a/apl/data-types/map-fields.mdx +++ b/apl/data-types/map-fields.mdx @@ -43,19 +43,20 @@ To access `http.protocol` inside the `attributes.custom` map field, the syntax i TODO: mhr3 suggested not to tell customers about X-Axiom-Object-Fields, need to make a decision about that -If you would like to manually create a map field in order to prevent unexpected json payload shape from creating new fields in your dataset, you can use the `X-Axiom-Object-Fields` header. See an example request below: +If you would like to manually create a map field in order to prevent unexpected json payload shape from creating new fields in your dataset, you can use the `X-Axiom-Object-Fields` header. It takes a comma-separated list of field names. See an example request below: ``` curl -X 'POST' 'https://api.axiom.co/v1/datasets/$DATASET_NAME/ingest' \ -H 'Authorization: $API_TOKEN' \ -H 'Content-Type: application/json' \ - -H 'X-Axiom-Object-Fields: object' \ + -H 'X-Axiom-Object-Fields: object,object2' \ -d '[ { "normal": { "foo": { "bar": "baz" }, { "biz": "qux" } }, "object": { "foo": { "bar": "baz" }, { "biz": "qux" } }, + "object2": { "foo": { "bar": "baz" }, { "biz": "qux" } } } ]' ``` -This will create the fields `normal.foo.bar`, `normal.foo.biz`, and `object`. +This will create the fields `normal.foo.bar`, `normal.foo.biz`, `object`, and `object2`. From 0811f80f85569f4ce921805efa33ace7397dce50 Mon Sep 17 00:00:00 2001 From: Mano Toth Date: Wed, 6 Nov 2024 11:59:35 +0100 Subject: [PATCH 4/6] TW review --- apl/data-types/map-fields.mdx | 47 ++++++++--------------------------- mint.json | 16 +++++++++--- 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/apl/data-types/map-fields.mdx b/apl/data-types/map-fields.mdx index b5038170..7fec570a 100644 --- a/apl/data-types/map-fields.mdx +++ b/apl/data-types/map-fields.mdx @@ -1,62 +1,35 @@ --- title: 'Map fields' -description: 'Learn about the purpose of map fields, and how to ingest and query them.' -sidebarTitle: Map Fields +description: 'This page explains what map fields are and how to query them.' tags: ['axiom documentation', 'documentation', 'axiom', 'map fields', 'object fields', 'data types'] --- -TODO: is this the right location for this page? i could also see it after the "Virtual Fields" page (Understanding Data => Query => Map Fields) - Map fields are a special type of field that can hold a collection of nested key-value pairs within a single field. You can think of the content of a map field as a JSON object. -## Why - -The purpose of map fields is allow storing additional attributes without adding more fields, particularly in situations where the shape of your data is unpredictable, such as additional attributes added by OpenTelemetry instrumentation. This can be useful to prevent you from reaching the field limit of a dataset, and to improve query performance. - -## `attributes.custom` in Tracing Datasets +## Benefits of map fields -If you are ingesting tracing data using the [OTLP/HTTP endpoint for traces and logs](/send-data/opentelemetry), you may have noticed that some attributes are moved into the `attributes.custom` map field. +The benefit of map fields is that you can store additional attributes without adding more fields. This is particularly useful when the shape of your data is unpredictable (for example, additional attributes added by an OpenTelemetry instrumentation). Using map fields means that you can avoid reaching the field limit of a dataset and improve query performance. -Any instrumentation library can add arbitrary attributes to spans, and in many cases a single piece of auto-instrumentation may add dozens or even hundreds of attributes. For a large system with a wide range of instrumentation, this may add up to thousands of attributes. +## Custom attributes in tracing datasets -In order to keep the dimensionality of your dataset under control, Axiom preserves all attributes that are described in the [OpenTelemetry Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/general/trace/), and puts everything else into the `attributes.custom` map field. +If you use [OpenTelemetry](/send-data/opentelemetry) to send data to Axiom, you find some attributes in the `attributes.custom` map field. The reason is that instrumentation libraries can add hundreds or even thousands of arbitrary attributes to spans. Storing each custom attribute in a separate field would significantly increase the number of fields in your dataset. To keep the number of fields in your dataset under control, Axiom places all custom attributes in the single `attributes.custom` map field. -## Querying map fields +## Use map fields in queries -To access a field in APL, you might do something like: +The example query below uses the map field `attributes.http.url` to filter results: ```kusto ['otel-demo-traces'] | where ['attributes.http.url'] == 'http://opentelemetry-demo-frontendproxy:8080/api/cart' ``` + [Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.http.url%27%5d%20%3d%3d%20%27http%3a%2f%2fopentelemetry-demo-frontendproxy%3a8080%2fapi%2fcart%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) -To access `http.protocol` inside the `attributes.custom` map field, the syntax is `['']['']` instead: +To access `http.protocol` inside the `attributes.custom` map field, use the syntax `['MAP_FIELD_NAME']['INDEX_TO_MAP_FIELD']` instead: ```kusto ['otel-demo-traces'] | where ['attributes.custom']['http.protocol'] == 'HTTP/1.1' ``` -[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.custom%27%5d%5b%27http.protocol%27%5d%20%3d%3d%20%27HTTP%2f1.1%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) - -## Ingesting map fields - -TODO: mhr3 suggested not to tell customers about X-Axiom-Object-Fields, need to make a decision about that - -If you would like to manually create a map field in order to prevent unexpected json payload shape from creating new fields in your dataset, you can use the `X-Axiom-Object-Fields` header. It takes a comma-separated list of field names. See an example request below: - -``` -curl -X 'POST' 'https://api.axiom.co/v1/datasets/$DATASET_NAME/ingest' \ - -H 'Authorization: $API_TOKEN' \ - -H 'Content-Type: application/json' \ - -H 'X-Axiom-Object-Fields: object,object2' \ - -d '[ - { - "normal": { "foo": { "bar": "baz" }, { "biz": "qux" } }, - "object": { "foo": { "bar": "baz" }, { "biz": "qux" } }, - "object2": { "foo": { "bar": "baz" }, { "biz": "qux" } } - } -]' -``` -This will create the fields `normal.foo.bar`, `normal.foo.biz`, `object`, and `object2`. +[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.custom%27%5d%5b%27http.protocol%27%5d%20%3d%3d%20%27HTTP%2f1.1%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) \ No newline at end of file diff --git a/mint.json b/mint.json index 4be0cc62..6043d3f7 100644 --- a/mint.json +++ b/mint.json @@ -188,7 +188,10 @@ }, { "group": "Monitors", - "pages": ["monitor-data/monitors", "monitor-data/pagerduty"] + "pages": [ + "monitor-data/monitors", + "monitor-data/pagerduty" + ] } ] }, @@ -257,7 +260,10 @@ }, { "group": "Get Started", - "pages": ["apl/introduction", "apl/tutorial"] + "pages": [ + "apl/introduction", + "apl/tutorial" + ] }, { "group": "Functions", @@ -353,9 +359,9 @@ "group": "Reference", "pages": [ "apl/entities/entity-names", + "apl/data-types/map-fields", "apl/data-types/null-values", "apl/data-types/scalar-data-types", - "apl/data-types/map-fields", "apl/query-statement/set-statement", "apl/guides/exploring-the-union-operator-in-apl", "apl/reference/special-field-attributes" @@ -388,7 +394,9 @@ }, { "group": "User Endpoints", - "pages": ["restapi/endpoints/getCurrentUser"] + "pages": [ + "restapi/endpoints/getCurrentUser" + ] }, { "group": "Dataset Endpoints", From b4fc056932e12f4720dc040dc84ae13fe841cae8 Mon Sep 17 00:00:00 2001 From: Mano Toth Date: Mon, 11 Nov 2024 11:22:55 +0100 Subject: [PATCH 5/6] Add teaser about custom fields --- apl/data-types/map-fields.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apl/data-types/map-fields.mdx b/apl/data-types/map-fields.mdx index 7fec570a..e5b386b0 100644 --- a/apl/data-types/map-fields.mdx +++ b/apl/data-types/map-fields.mdx @@ -32,4 +32,8 @@ To access `http.protocol` inside the `attributes.custom` map field, use the synt | where ['attributes.custom']['http.protocol'] == 'HTTP/1.1' ``` -[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.custom%27%5d%5b%27http.protocol%27%5d%20%3d%3d%20%27HTTP%2f1.1%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) \ No newline at end of file +[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.custom%27%5d%5b%27http.protocol%27%5d%20%3d%3d%20%27HTTP%2f1.1%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) + +## Add custom fields + +The possibility to add custom fields is coming in early 2025. To express interest in the feature, [contact Axiom](https://axiom.co/contact). \ No newline at end of file From ff32a05feed234909390e1f5ce42637fab61cc40 Mon Sep 17 00:00:00 2001 From: Mano Toth Date: Wed, 20 Nov 2024 15:32:27 +0100 Subject: [PATCH 6/6] Implement reviews --- apl/data-types/map-fields.mdx | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/apl/data-types/map-fields.mdx b/apl/data-types/map-fields.mdx index e5b386b0..a5227740 100644 --- a/apl/data-types/map-fields.mdx +++ b/apl/data-types/map-fields.mdx @@ -6,9 +6,11 @@ tags: ['axiom documentation', 'documentation', 'axiom', 'map fields', 'object fi Map fields are a special type of field that can hold a collection of nested key-value pairs within a single field. You can think of the content of a map field as a JSON object. -## Benefits of map fields +## Benefits and drawbacks of map fields -The benefit of map fields is that you can store additional attributes without adding more fields. This is particularly useful when the shape of your data is unpredictable (for example, additional attributes added by an OpenTelemetry instrumentation). Using map fields means that you can avoid reaching the field limit of a dataset and improve query performance. +The benefit of map fields is that you can store additional attributes without adding more fields. This is particularly useful when the shape of your data is unpredictable (for example, additional attributes added by OpenTelemetry instrumentation). Using map fields means that you can avoid reaching the field limit of a dataset and improve query performance. + +The drawback of map fields is that querying them uses more query-hours than querying conventional fields. Additionally, you don’t have visibility into map fields from the schema. For example, autocomplete doesn’t know the properties inside the map field. ## Custom attributes in tracing datasets @@ -16,16 +18,7 @@ If you use [OpenTelemetry](/send-data/opentelemetry) to send data to Axiom, you ## Use map fields in queries -The example query below uses the map field `attributes.http.url` to filter results: - -```kusto -['otel-demo-traces'] -| where ['attributes.http.url'] == 'http://opentelemetry-demo-frontendproxy:8080/api/cart' -``` - -[Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.http.url%27%5d%20%3d%3d%20%27http%3a%2f%2fopentelemetry-demo-frontendproxy%3a8080%2fapi%2fcart%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) - -To access `http.protocol` inside the `attributes.custom` map field, use the syntax `['MAP_FIELD_NAME']['INDEX_TO_MAP_FIELD']` instead: +The example query below uses the `http.protocol` property inside the `attributes.custom` map field to filter results: ```kusto ['otel-demo-traces'] @@ -34,6 +27,8 @@ To access `http.protocol` inside the `attributes.custom` map field, use the synt [Run in playground](https://play.axiom.co/axiom-play-qf1k/explorer?initForm=%7b%22apl%22%3a%22%5b%27otel-demo-traces%27%5d%5cn%7c%20where%20%5b%27attributes.custom%27%5d%5b%27http.protocol%27%5d%20%3d%3d%20%27HTTP%2f1.1%27%22%2c%22queryOptions%22%3a%7b%22quickRange%22%3a%2230d%22%7d%7d) +If the entity names have spaces (` `), dots (`.`), or dashes (`-`) as in this example, use the syntax `['MAP_FIELD_NAME']['INDEX_TO_MAP_FIELD']`. For more information, see [Entity names](/apl/entities/entity-names#quote-identifiers). + ## Add custom fields The possibility to add custom fields is coming in early 2025. To express interest in the feature, [contact Axiom](https://axiom.co/contact). \ No newline at end of file