Replace apm-server OTLP endpoint with Otel native processors and Elasticsearch

.env.override

@@ -17,5 +17,4 @@ KAFKA_SERVICE_DOCKERFILE=./src/kafka/Dockerfile.elastic
 # *********************
 COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.16.0-SNAPSHOT
 OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml
-OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml
 ELASTIC_AGENT_OTEL=true

.github/README.md

@@ -27,27 +27,24 @@ Additionally, the OpenTelemetry Contrib collector has also been changed to the [
 - Set up [kubectl](https://kubernetes.io/docs/reference/kubectl/).
 - Set up [Helm](https://helm.sh/).
 
-### Start the Demo
+![Deployment architecture](../kubernetes/elastic-helm/elastic-architecture.png "K8s architecture")
+
+### Start the Demo (Kubernetes deployment)
 1. Setup Elastic Observability on Elastic Cloud.
-1. Create a secret in Kubernetes with the following command.
+2. Create a secret in Kubernetes with the following command.
    ```
-   kubectl create secret generic elastic-secret \
-     --from-literal=elastic_apm_endpoint='YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX' \
-     --from-literal=elastic_apm_secret_token='YOUR_APM_SECRET_TOKEN'
+   kubectl create secret generic elastic-otel-secret \
+     --from-literal=elastic_endpoint='YOUR_ELASTICSEARCH_ENDPOINT' \
+     --from-literal=elastic_api_key='YOUR_ELASTICSEARCH_API_KEY'
    ```
    Don't forget to replace
-   - `YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX`: your Elastic APM endpoint (*without* `https://` prefix) that *must* also include the port (example: `1234567.apm.us-west2.gcp.elastic-cloud.com:443`).
-   - `YOUR_APM_SECRET_TOKEN`: your Elastic APM secret token, include the Bearer or ApiKey but not the "Authorization=" part e.g. Bearer XXXXXX or ApiKey XXXXX below is an example:
-   ```
-   kubectl create secret generic elastic-secret \
-     --from-literal=elastic_apm_endpoint='12345.apm.us-west2.gcp.elastic-cloud.com:443' \
-     --from-literal=elastic_apm_secret_token='Bearer 123456789123456YE2'
-   ```
-1. Execute the following commands to deploy the OpenTelemetry demo to your Kubernetes cluster:
+   - `YOUR_ELASTICSEARCH_ENDPOINT`: your Elasticsearch endpoint (*with* `https://` prefix example: `https://1234567.us-west2.gcp.elastic-cloud.com:443`).
+   - `YOUR_ELASTICSEARCH_API_KEY`: your Elasticsearch API Key
+3. Execute the following commands to deploy the OpenTelemetry demo to your Kubernetes cluster:
    ```
    # clone this repository
    git clone https://github.com/elastic/opentelemetry-demo
-   
+
    # switch to the kubernetes/elastic-helm directory
    cd opentelemetry-demo/kubernetes/elastic-helm
 
@@ -61,24 +58,14 @@ Additionally, the OpenTelemetry Contrib collector has also been changed to the [
    helm install -f deployment.yaml my-otel-demo open-telemetry/opentelemetry-demo
    ```
 
-#### Kubernetes monitoring
+#### Kubernetes monitoring (daemonset)
 
 This demo already enables cluster level metrics collection with `clusterMetrics` and
 Kubernetes events collection with `kubernetesEvents`.
 
 In order to add Node level metrics collection we can run an additional Otel collector Daemonset with the following:
 
-1. Create a secret in Kubernetes with the following command.
-   ```
-   kubectl create secret generic elastic-secret-ds \
-     --from-literal=elastic_endpoint='YOUR_ELASTICSEARCH_ENDPOINT' \
-     --from-literal=elastic_api_key='YOUR_ELASTICSEARCH_API_KEY'
-   ```
-   Don't forget to replace
-   - `YOUR_ELASTICSEARCH_ENDPOINT`: your Elasticsearch endpoint (*with* `https://` prefix example: `https://1234567.us-west2.gcp.elastic-cloud.com:443`).
-   - `YOUR_ELASTICSEARCH_API_KEY`: your Elasticsearch API Key
-
-2. Execute the following command to deploy the OpenTelemetry Collector to your Kubernetes cluster, in the same directory `kubernetes/elastic-helm` in this repository.
+Execute the following command to deploy the OpenTelemetry Collector to your Kubernetes cluster, in the same directory `kubernetes/elastic-helm` in this repository.
 
 ```
 # deploy the Elastic OpenTelemetry collector distribution through helm install

kubernetes/elastic-helm/daemonset.yaml

@@ -23,12 +23,12 @@ extraEnvs:
   - name: ELASTIC_ENDPOINT
     valueFrom:
       secretKeyRef:
-        name: elastic-secret-ds
+        name: elastic-otel-secret
         key: elastic_endpoint
   - name: ELASTIC_API_KEY
     valueFrom:
       secretKeyRef:
-        name: elastic-secret-ds
+        name: elastic-otel-secret
         key: elastic_api_key
   - name: K8S_NODE_NAME
     valueFrom:
@@ -61,7 +61,7 @@ config:
   exporters:
     debug:
       verbosity: basic
-    elasticsearch:
+    elasticsearch/ecs:
       endpoints:
       - ${env:ELASTIC_ENDPOINT}
       api_key: ${env:ELASTIC_API_KEY}
@@ -71,11 +71,22 @@ config:
         enabled: true
       mapping:
         mode: ecs
+    elasticsearch/otel:
+      endpoints:
+      - ${env:ELASTIC_ENDPOINT}
+      api_key: ${env:ELASTIC_API_KEY}
+      logs_dynamic_index:
+        enabled: true
+      metrics_dynamic_index:
+        enabled: true
+      mapping:
+        mode: otel
   processors:
     batch: {}
     elasticinframetrics:
       add_system_metrics: true
       add_k8s_metrics: true
+      drop_original: true
     resourcedetection/eks:
       detectors: [env, eks]
       timeout: 15s
@@ -93,11 +104,6 @@ config:
         - key: service.name
           from_attribute: app.label.component
           action: insert
-    attributes/k8s_logs_dataset:
-      actions:
-        - key: data_stream.dataset
-          value: "kubernetes.container_logs"
-          action: upsert
     attributes/dataset:
       actions:
         - key: event.dataset
@@ -311,12 +317,16 @@ config:
     pipelines:
       logs:
         receivers: [filelog]
-        processors: [batch, k8sattributes, resourcedetection/system, resourcedetection/eks, resourcedetection/gcp, resource/demo, resource/k8s, resource/cloud, attributes/k8s_logs_dataset]
-        exporters: [debug, elasticsearch]
+        processors: [batch, k8sattributes, resourcedetection/system, resourcedetection/eks, resourcedetection/gcp, resource/demo, resource/k8s, resource/cloud]
+        exporters: [debug, elasticsearch/otel]
       metrics:
         receivers: [hostmetrics, kubeletstats]
         processors: [batch, k8sattributes, elasticinframetrics, resourcedetection/system, resource/demo, resourcedetection/eks, resourcedetection/gcp, resource/k8s, resource/cloud, attributes/dataset, resource/process]
-        exporters: [debug, elasticsearch]
+        exporters: [debug, elasticsearch/ecs]
+      metrics/otel:
+        receivers: [kubeletstats]
+        processors: [batch, k8sattributes, resourcedetection/system, resource/demo, resourcedetection/eks, resourcedetection/gcp, resource/k8s, resource/cloud]
+        exporters: [debug, elasticsearch/otel]
       traces: null
     telemetry:
       metrics: