Merge branch 'main' into helm/checksum

grafana · Mar 15, 2024 · d3c2c4f · d3c2c4f
2 parents 2520cc3 + 0a90f05
commit d3c2c4f
Show file tree

Hide file tree

Showing 101 changed files with 1,167 additions and 543 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -23,4 +23,4 @@ jobs:
           metricsWriteAPIKey: ${{secrets.GRAFANA_MISC_STATS_API_KEY}}
           token: ${{secrets.GH_BOT_ACCESS_TOKEN}}
           labelsToAdd: "backport"
-          title: "[{{base}}] {{originalTitle}}"
+          title: "chore: [{{base}}] {{originalTitle}}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -110,6 +110,8 @@
 
 #### LogCLI
 
+* [11852](https://github.com/grafana/loki/pull/11852) **MichelHollands**: feat: update logcli so it tries to load the latest version of the schemaconfig
+
 #### Mixins
 
 * [11087](https://github.com/grafana/loki/pull/11087) **JoaoBraveCoding**: Adds structured metadata panels for ingested data

diff --git a/cmd/loki/main.go b/cmd/loki/main.go
@@ -43,6 +43,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Set the global OTLP config which is needed in per tenant otlp config
+	config.LimitsConfig.SetGlobalOTLPConfig(config.Distributor.OTLPConfig)
 	// This global is set to the config passed into the last call to `NewOverrides`. If we don't
 	// call it atleast once, the defaults are set to an empty struct.
 	// We call it with the flag values so that the config file unmarshalling only overrides the values set in the config.

diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md
@@ -551,6 +551,11 @@ write_failures_logging:
   # logged or not. Default: false.
   # CLI flag: -distributor.write-failures-logging.add-insights-label
   [add_insights_label: <boolean> | default = false]
+
+otlp_config:
+  # List of default otlp resource attributes to be picked as index labels
+  # CLI flag: -distributor.otlp.default_resource_attributes_as_index_labels
+  [default_resource_attributes_as_index_labels: <list of strings> | default = [service.name service.namespace service.instance.id deployment.environment cloud.region cloud.availability_zone k8s.cluster.name k8s.namespace.name k8s.pod.name k8s.container.name container.name k8s.replicaset.name k8s.deployment.name k8s.statefulset.name k8s.daemonset.name k8s.cronjob.name k8s.job.name]]
 ```
 
 ### querier
@@ -1758,6 +1763,12 @@ ring:
   # CLI flag: -index-gateway.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Deprecated: How many index gateway instances are assigned to each tenant.
+  # Use -index-gateway.shard-size instead. The shard size is also a per-tenant
+  # setting.
+  # CLI flag: -replication-factor
+  [replication_factor: <int> | default = 3]
+
   # Instance ID to register in the ring.
   # CLI flag: -index-gateway.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]
@@ -1782,12 +1793,6 @@ ring:
   # Enable using a IPv6 instance address.
   # CLI flag: -index-gateway.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
-
-  # Deprecated: How many index gateway instances are assigned to each tenant.
-  # Use -index-gateway.shard-size instead. The shard size is also a per-tenant
-  # setting.
-  # CLI flag: -replication-factor
-  [replication_factor: <int> | default = 3]
 ```
 
 ### bloom_gateway
@@ -1855,6 +1860,17 @@ ring:
   # CLI flag: -bloom-gateway.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Number of tokens to use in the ring. The bigger the number of tokens, the
+  # more fingerprint ranges the compactor will own, but the smaller these ranges
+  # will be. Bigger number of tokens means that more but smaller requests will
+  # be handled by each gateway.
+  # CLI flag: -bloom-gateway.ring.tokens
+  [num_tokens: <int> | default = 16]
+
+  # Factor for data replication.
+  # CLI flag: -bloom-gateway.ring.replication-factor
+  [replication_factor: <int> | default = 3]
+
   # Instance ID to register in the ring.
   # CLI flag: -bloom-gateway.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]
@@ -1880,17 +1896,6 @@ ring:
   # CLI flag: -bloom-gateway.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-  # Factor for data replication.
-  # CLI flag: -bloom-gateway.replication-factor
-  [replication_factor: <int> | default = 3]
-
-  # Number of tokens to use in the ring. The bigger the number of tokens, the
-  # more fingerprint ranges the compactor will own, but the smaller these ranges
-  # will be. Bigger number of tokens means that more but smaller requests will
-  # be handled by each gateway.
-  # CLI flag: -bloom-gateway.ring.tokens
-  [tokens: <int> | default = 16]
-
 # Flag to enable or disable the bloom gateway component globally.
 # CLI flag: -bloom-gateway.enabled
 [enabled: <boolean> | default = false]
@@ -2649,6 +2654,11 @@ ring:
   # CLI flag: -bloom-compactor.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Number of tokens to use in the ring per compactor. Higher number of tokens
+  # will result in more and smaller files (metas and blocks.)
+  # CLI flag: -bloom-compactor.ring.num-tokens
+  [num_tokens: <int> | default = 10]
+
   # Instance ID to register in the ring.
   # CLI flag: -bloom-compactor.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]
@@ -2674,13 +2684,6 @@ ring:
   # CLI flag: -bloom-compactor.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-  # Number of tokens to use in the ring. The bigger the number of tokens, the
-  # more fingerprint ranges the compactor will own, but the smaller these ranges
-  # will be. Bigger number of tokens will result in more and smaller metas and
-  # blocks.
-  # CLI flag: -bloom-compactor.ring.tokens
-  [tokens: <int> | default = 10]
-
 # Flag to enable or disable the usage of the bloom-compactor component.
 # CLI flag: -bloom-compactor.enabled
 [enabled: <boolean> | default = false]
@@ -3181,21 +3184,12 @@ shard_streams:
 # The shard size defines how many bloom compactors should be used by a tenant
 # when computing blooms. If it's set to 0, shuffle sharding is disabled.
 # CLI flag: -bloom-compactor.shard-size
-[bloom_compactor_shard_size: <int> | default = 1]
-
-# The maximum age of a table before it is compacted. Do not compact tables older
-# than the the configured time. Default to 7 days. 0s means no limit.
-# CLI flag: -bloom-compactor.max-table-age
-[bloom_compactor_max_table_age: <duration> | default = 168h]
+[bloom_compactor_shard_size: <int> | default = 0]
 
 # Whether to compact chunks into bloom filters.
 # CLI flag: -bloom-compactor.enable-compaction
 [bloom_compactor_enable_compaction: <boolean> | default = false]
 
-# The batch size of the chunks the bloom-compactor downloads at once.
-# CLI flag: -bloom-compactor.chunks-batch-size
-[bloom_compactor_chunks_batch_size: <int> | default = 100]
-
 # Length of the n-grams created when computing blooms from log lines.
 # CLI flag: -bloom-compactor.ngram-length
 [bloom_ngram_length: <int> | default = 4]
@@ -3239,7 +3233,8 @@ otlp_config:
   # Configuration for resource attributes to store them as index labels or
   # Structured Metadata or drop them altogether
   resource_attributes:
-    # Configure whether to ignore the default list of resource attributes to be
+    # Configure whether to ignore the default list of resource attributes set in
+    # 'distributor.otlp.default_resource_attributes_as_index_labels' to be
     # stored as index labels and only use the given resource attributes config
     [ignore_defaults: <boolean> | default = false]
 
@@ -3839,6 +3834,14 @@ ring:
   # CLI flag: -common.storage.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Number of tokens to own in the ring.
+  # CLI flag: -common.storage.ring.num-tokens
+  [num_tokens: <int> | default = 128]
+
+  # Factor for data replication.
+  # CLI flag: -common.storage.ring.replication-factor
+  [replication_factor: <int> | default = 3]
+
   # Instance ID to register in the ring.
   # CLI flag: -common.storage.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]

diff --git a/docs/sources/send-data/docker-driver/_index.md b/docs/sources/send-data/docker-driver/_index.md
@@ -75,6 +75,6 @@ docker plugin rm loki
 
 The driver keeps all logs in memory and will drop log entries if Loki is not reachable and if the quantity of `max_retries` has been exceeded. To avoid the dropping of log entries, setting `max_retries` to zero allows unlimited retries; the driver will continue trying forever until Loki is again reachable. Trying forever may have undesired consequences, because the Docker daemon will wait for the Loki driver to process all logs of a container, until the container is removed. Thus, the Docker daemon might wait forever if the container is stuck.
 
-The wait time can be lowered by setting `loki-retries=2`, `loki-max-backoff_800ms`, `loki-timeout=1s` and `keep-file=true`. This way the daemon will be locked only for a short time and the logs will be persisted locally when the Loki client is unable to re-connect.
+The wait time can be lowered by setting `loki-retries=2`, `loki-max-backoff=800ms`, `loki-timeout=1s` and `keep-file=true`. This way the daemon will be locked only for a short time and the logs will be persisted locally when the Loki client is unable to re-connect.
 
-To avoid this issue, use the Promtail [Docker target]({{< relref "../../send-data/promtail/configuration#docker" >}}) or [Docker service discovery]({{< relref "../../send-data/promtail/configuration#docker_sd_config" >}}).
+To avoid this issue, use the Promtail [Docker target]({{< relref "../../send-data/promtail/configuration#docker" >}}) or [Docker service discovery]({{< relref "../../send-data/promtail/configuration#docker_sd_configs" >}}).
diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md
@@ -124,8 +124,9 @@ otlp_config:
   # Configuration for Resource Attributes to store them as index labels or
   # Structured Metadata or drop them altogether
   resource_attributes:
-    # Configure whether to ignore the default list of Resource Attributes to be
-    # stored as Index Labels and only use the given Resource Attributes config
+    # Configure whether to ignore the default list of resource attributes set in
+    # 'distributor.otlp.default_resource_attributes_as_index_labels' to be
+    # stored as index labels and only use the given resource attributes config
     [ignore_defaults: <boolean>]
 
     [attributes_config: <list of attributes_configs>]

diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -1,5 +1,8 @@
 ## Main
 
+- [12164](https://github.com/grafana/loki/pull/12164) **periklis**: Use safe bearer token authentication to scrape operator metrics
+- [12216](https://github.com/grafana/loki/pull/12216) **xperimental**: Fix duplicate operator metrics due to ServiceMonitor selector
+- [12212](https://github.com/grafana/loki/pull/12212) **xperimental**: Keep credentialMode in status when updating schemas
 - [12165](https://github.com/grafana/loki/pull/12165) **JoaoBraveCoding**: Change attribute value used for CCO-based credential mode
 - [12157](https://github.com/grafana/loki/pull/12157) **periklis**: Fix managed auth features annotation for community-openshift bundle
 - [12104](https://github.com/grafana/loki/pull/12104) **periklis**: Upgrade build and runtime dependencies

diff --git a/...penshift/manifests/loki-operator-controller-manager-metrics-reader_v1_serviceaccount.yaml b/...penshift/manifests/loki-operator-controller-manager-metrics-reader_v1_serviceaccount.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-metrics-reader
diff --git a/...nity-openshift/manifests/loki-operator-controller-manager-metrics-service_v1_service.yaml b/...nity-openshift/manifests/loki-operator-controller-manager-metrics-service_v1_service.yaml
@@ -5,6 +5,7 @@ metadata:
     service.beta.openshift.io/serving-cert-secret-name: loki-operator-metrics
   creationTimestamp: null
   labels:
+    app.kubernetes.io/component: metrics
     app.kubernetes.io/instance: loki-operator-v0.5.0
     app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: loki-operator

diff --git a/...mmunity-openshift/manifests/loki-operator-controller-manager-metrics-token_v1_secret.yaml b/...mmunity-openshift/manifests/loki-operator-controller-manager-metrics-token_v1_secret.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    kubernetes.io/service-account.name: loki-operator-controller-manager-metrics-reader
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-metrics-token
+type: kubernetes.io/service-account-token
diff --git a/...ator-controller-manager-read-metrics_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml b/...ator-controller-manager-read-metrics_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml
@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-read-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: loki-operator-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: loki-operator-controller-manager-metrics-reader
+  namespace: kubernetes-operators
diff --git a/...hift/manifests/loki-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml b/...hift/manifests/loki-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml
@@ -11,15 +11,23 @@ metadata:
   name: loki-operator-metrics-monitor
 spec:
   endpoints:
-  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+  - authorization:
+      credentials:
+        key: token
+        name: loki-operator-controller-manager-metrics-token
+      type: bearer
     interval: 30s
     path: /metrics
     scheme: https
     scrapeTimeout: 10s
     targetPort: 8443
     tlsConfig:
-      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      ca:
+        secret:
+          key: service-ca.crt
+          name: loki-operator-controller-manager-metrics-token
       serverName: loki-operator-controller-manager-metrics-service.kubernetes-operators.svc
   selector:
     matchLabels:
+      app.kubernetes.io/component: metrics
       app.kubernetes.io/name: loki-operator
diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2024-03-12T09:52:37Z"
+    createdAt: "2024-03-14T13:25:52Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     features.operators.openshift.io/disconnected: "true"
@@ -1678,7 +1678,7 @@ spec:
           - subjectaccessreviews
           verbs:
           - create
-        serviceAccountName: default
+        serviceAccountName: loki-operator-controller-manager
       deployments:
       - label:
           app.kubernetes.io/instance: loki-operator-v0.5.0
@@ -1779,6 +1779,7 @@ spec:
                 runAsNonRoot: true
                 seccompProfile:
                   type: RuntimeDefault
+              serviceAccountName: loki-operator-controller-manager
               terminationGracePeriodSeconds: 10
               volumes:
               - configMap:
@@ -1812,7 +1813,7 @@ spec:
           verbs:
           - create
           - patch
-        serviceAccountName: default
+        serviceAccountName: loki-operator-controller-manager
     strategy: deployment
   installModes:
   - supported: false

diff --git a/...ommunity/manifests/loki-operator-controller-manager-metrics-reader_v1_serviceaccount.yaml b/...ommunity/manifests/loki-operator-controller-manager-metrics-reader_v1_serviceaccount.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-metrics-reader
diff --git a/...ndle/community/manifests/loki-operator-controller-manager-metrics-service_v1_service.yaml b/...ndle/community/manifests/loki-operator-controller-manager-metrics-service_v1_service.yaml
@@ -3,6 +3,7 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
+    app.kubernetes.io/component: metrics
     app.kubernetes.io/instance: loki-operator-v0.5.0
     app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: loki-operator

diff --git a/...ator-controller-manager-read-metrics_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml b/...ator-controller-manager-read-metrics_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml
@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-read-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: loki-operator-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: loki-operator-controller-manager-metrics-reader
+  namespace: loki-operator
diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2024-03-12T09:52:36Z"
+    createdAt: "2024-03-14T13:25:49Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -1658,7 +1658,7 @@ spec:
           - subjectaccessreviews
           verbs:
           - create
-        serviceAccountName: default
+        serviceAccountName: loki-operator-controller-manager
       deployments:
       - label:
           app.kubernetes.io/instance: loki-operator-v0.5.0
@@ -1748,6 +1748,7 @@ spec:
                 kubernetes.io/os: linux
               securityContext:
                 runAsNonRoot: true
+              serviceAccountName: loki-operator-controller-manager
               terminationGracePeriodSeconds: 10
               volumes:
               - name: webhook-cert
@@ -1780,7 +1781,7 @@ spec:
           verbs:
           - create
           - patch
-        serviceAccountName: default
+        serviceAccountName: loki-operator-controller-manager
     strategy: deployment
   installModes:
   - supported: false