Merge branch 'main' into docker_driver

grafana · Mar 15, 2024 · 933e46d · 933e46d
2 parents 5044a96 + 347fd4d
commit 933e46d
Show file tree

Hide file tree

Showing 143 changed files with 1,535 additions and 808 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -23,4 +23,4 @@ jobs:
           metricsWriteAPIKey: ${{secrets.GRAFANA_MISC_STATS_API_KEY}}
           token: ${{secrets.GH_BOT_ACCESS_TOKEN}}
           labelsToAdd: "backport"
-          title: "[{{base}}] {{originalTitle}}"
+          title: "chore: [{{base}}] {{originalTitle}}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -110,6 +110,8 @@
 
 #### LogCLI
 
+* [11852](https://github.com/grafana/loki/pull/11852) **MichelHollands**: feat: update logcli so it tries to load the latest version of the schemaconfig
+
 #### Mixins
 
 * [11087](https://github.com/grafana/loki/pull/11087) **JoaoBraveCoding**: Adds structured metadata panels for ingested data

diff --git a/cmd/loki/main.go b/cmd/loki/main.go
@@ -43,6 +43,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Set the global OTLP config which is needed in per tenant otlp config
+	config.LimitsConfig.SetGlobalOTLPConfig(config.Distributor.OTLPConfig)
 	// This global is set to the config passed into the last call to `NewOverrides`. If we don't
 	// call it atleast once, the defaults are set to an empty struct.
 	// We call it with the flag values so that the config file unmarshalling only overrides the values set in the config.

diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md
@@ -551,6 +551,11 @@ write_failures_logging:
   # logged or not. Default: false.
   # CLI flag: -distributor.write-failures-logging.add-insights-label
   [add_insights_label: <boolean> | default = false]
+
+otlp_config:
+  # List of default otlp resource attributes to be picked as index labels
+  # CLI flag: -distributor.otlp.default_resource_attributes_as_index_labels
+  [default_resource_attributes_as_index_labels: <list of strings> | default = [service.name service.namespace service.instance.id deployment.environment cloud.region cloud.availability_zone k8s.cluster.name k8s.namespace.name k8s.pod.name k8s.container.name container.name k8s.replicaset.name k8s.deployment.name k8s.statefulset.name k8s.daemonset.name k8s.cronjob.name k8s.job.name]]
 ```
 
 ### querier
@@ -1758,6 +1763,12 @@ ring:
   # CLI flag: -index-gateway.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Deprecated: How many index gateway instances are assigned to each tenant.
+  # Use -index-gateway.shard-size instead. The shard size is also a per-tenant
+  # setting.
+  # CLI flag: -replication-factor
+  [replication_factor: <int> | default = 3]
+
   # Instance ID to register in the ring.
   # CLI flag: -index-gateway.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]
@@ -1782,12 +1793,6 @@ ring:
   # Enable using a IPv6 instance address.
   # CLI flag: -index-gateway.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
-
-  # Deprecated: How many index gateway instances are assigned to each tenant.
-  # Use -index-gateway.shard-size instead. The shard size is also a per-tenant
-  # setting.
-  # CLI flag: -replication-factor
-  [replication_factor: <int> | default = 3]
 ```
 
 ### bloom_gateway
@@ -1855,6 +1860,17 @@ ring:
   # CLI flag: -bloom-gateway.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Number of tokens to use in the ring. The bigger the number of tokens, the
+  # more fingerprint ranges the compactor will own, but the smaller these ranges
+  # will be. Bigger number of tokens means that more but smaller requests will
+  # be handled by each gateway.
+  # CLI flag: -bloom-gateway.ring.tokens
+  [num_tokens: <int> | default = 16]
+
+  # Factor for data replication.
+  # CLI flag: -bloom-gateway.ring.replication-factor
+  [replication_factor: <int> | default = 3]
+
   # Instance ID to register in the ring.
   # CLI flag: -bloom-gateway.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]
@@ -1880,17 +1896,6 @@ ring:
   # CLI flag: -bloom-gateway.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-  # Factor for data replication.
-  # CLI flag: -bloom-gateway.replication-factor
-  [replication_factor: <int> | default = 3]
-
-  # Number of tokens to use in the ring. The bigger the number of tokens, the
-  # more fingerprint ranges the compactor will own, but the smaller these ranges
-  # will be. Bigger number of tokens means that more but smaller requests will
-  # be handled by each gateway.
-  # CLI flag: -bloom-gateway.ring.tokens
-  [tokens: <int> | default = 16]
-
 # Flag to enable or disable the bloom gateway component globally.
 # CLI flag: -bloom-gateway.enabled
 [enabled: <boolean> | default = false]
@@ -1937,6 +1942,10 @@ client:
 # Maximum number of outstanding tasks per tenant.
 # CLI flag: -bloom-gateway.max-outstanding-per-tenant
 [max_outstanding_per_tenant: <int> | default = 1024]
+
+# How many tasks are multiplexed at once.
+# CLI flag: -bloom-gateway.num-multiplex-tasks
+[num_multiplex_tasks: <int> | default = 512]
 ```
 
 ### storage_config
@@ -2645,6 +2654,11 @@ ring:
   # CLI flag: -bloom-compactor.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Number of tokens to use in the ring per compactor. Higher number of tokens
+  # will result in more and smaller files (metas and blocks.)
+  # CLI flag: -bloom-compactor.ring.num-tokens
+  [num_tokens: <int> | default = 10]
+
   # Instance ID to register in the ring.
   # CLI flag: -bloom-compactor.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]
@@ -2670,13 +2684,6 @@ ring:
   # CLI flag: -bloom-compactor.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-  # Number of tokens to use in the ring. The bigger the number of tokens, the
-  # more fingerprint ranges the compactor will own, but the smaller these ranges
-  # will be. Bigger number of tokens will result in more and smaller metas and
-  # blocks.
-  # CLI flag: -bloom-compactor.ring.tokens
-  [tokens: <int> | default = 10]
-
 # Flag to enable or disable the usage of the bloom-compactor component.
 # CLI flag: -bloom-compactor.enabled
 [enabled: <boolean> | default = false]
@@ -3177,21 +3184,12 @@ shard_streams:
 # The shard size defines how many bloom compactors should be used by a tenant
 # when computing blooms. If it's set to 0, shuffle sharding is disabled.
 # CLI flag: -bloom-compactor.shard-size
-[bloom_compactor_shard_size: <int> | default = 1]
-
-# The maximum age of a table before it is compacted. Do not compact tables older
-# than the the configured time. Default to 7 days. 0s means no limit.
-# CLI flag: -bloom-compactor.max-table-age
-[bloom_compactor_max_table_age: <duration> | default = 168h]
+[bloom_compactor_shard_size: <int> | default = 0]
 
 # Whether to compact chunks into bloom filters.
 # CLI flag: -bloom-compactor.enable-compaction
 [bloom_compactor_enable_compaction: <boolean> | default = false]
 
-# The batch size of the chunks the bloom-compactor downloads at once.
-# CLI flag: -bloom-compactor.chunks-batch-size
-[bloom_compactor_chunks_batch_size: <int> | default = 100]
-
 # Length of the n-grams created when computing blooms from log lines.
 # CLI flag: -bloom-compactor.ngram-length
 [bloom_ngram_length: <int> | default = 4]
@@ -3235,7 +3233,8 @@ otlp_config:
   # Configuration for resource attributes to store them as index labels or
   # Structured Metadata or drop them altogether
   resource_attributes:
-    # Configure whether to ignore the default list of resource attributes to be
+    # Configure whether to ignore the default list of resource attributes set in
+    # 'distributor.otlp.default_resource_attributes_as_index_labels' to be
     # stored as index labels and only use the given resource attributes config
     [ignore_defaults: <boolean> | default = false]
 
@@ -3835,6 +3834,14 @@ ring:
   # CLI flag: -common.storage.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # Number of tokens to own in the ring.
+  # CLI flag: -common.storage.ring.num-tokens
+  [num_tokens: <int> | default = 128]
+
+  # Factor for data replication.
+  # CLI flag: -common.storage.ring.replication-factor
+  [replication_factor: <int> | default = 3]
+
   # Instance ID to register in the ring.
   # CLI flag: -common.storage.ring.instance-id
   [instance_id: <string> | default = "<hostname>"]

diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md
@@ -124,8 +124,9 @@ otlp_config:
   # Configuration for Resource Attributes to store them as index labels or
   # Structured Metadata or drop them altogether
   resource_attributes:
-    # Configure whether to ignore the default list of Resource Attributes to be
-    # stored as Index Labels and only use the given Resource Attributes config
+    # Configure whether to ignore the default list of resource attributes set in
+    # 'distributor.otlp.default_resource_attributes_as_index_labels' to be
+    # stored as index labels and only use the given resource attributes config
     [ignore_defaults: <boolean>]
 
     [attributes_config: <list of attributes_configs>]

diff --git a/docs/sources/send-data/promtail/installation.md b/docs/sources/send-data/promtail/installation.md
@@ -44,10 +44,20 @@ And the chart repository can be updated by running:
 helm repo update
 ```
 
+Create the configuration file `values.yaml`. The example below illustrates a connection to the locally deployed loki server:
+```yaml
+config:
+  # publish data to loki
+  clients:
+    - url: http://loki-gateway/loki/api/v1/push
+      tenant_id: 1
+```
+
 Finally, Promtail can be deployed with:
 
 ```bash
-helm upgrade --install promtail grafana/promtail
+# The default helm configuration deploys promtail as a daemonSet (recommended)
+helm upgrade --values values.yaml --install promtail grafana/promtail
 ```
 
 ## Install as Kubernetes daemonSet (recommended)

diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -1,5 +1,10 @@
 ## Main
 
+- [12164](https://github.com/grafana/loki/pull/12164) **periklis**: Use safe bearer token authentication to scrape operator metrics
+- [12216](https://github.com/grafana/loki/pull/12216) **xperimental**: Fix duplicate operator metrics due to ServiceMonitor selector
+- [12212](https://github.com/grafana/loki/pull/12212) **xperimental**: Keep credentialMode in status when updating schemas
+- [12165](https://github.com/grafana/loki/pull/12165) **JoaoBraveCoding**: Change attribute value used for CCO-based credential mode
+- [12157](https://github.com/grafana/loki/pull/12157) **periklis**: Fix managed auth features annotation for community-openshift bundle
 - [12104](https://github.com/grafana/loki/pull/12104) **periklis**: Upgrade build and runtime dependencies
 - [11928](https://github.com/grafana/loki/pull/11928) **periklis**: Fix remote write client timeout config rename
 - [12097](https://github.com/grafana/loki/pull/12097) **btaani**: Fix encoding of blocked query pattern in configuration

diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go
@@ -52,9 +52,9 @@ type OpenShiftFeatureGates struct {
 	// Dashboards enables the loki-mixin dashboards into the OpenShift Console
 	Dashboards bool `json:"dashboards,omitempty"`
 
-	// ManagedAuthEnv is true when OpenShift-functions are enabled and the operator has detected
+	// TokenCCOAuthEnv is true when OpenShift-functions are enabled and the operator has detected
 	// that it is running with some kind of "workload identity" (AWS STS, Azure WIF) enabled.
-	ManagedAuthEnv bool
+	TokenCCOAuthEnv bool
 }
 
 // FeatureGates is the supported set of all operator feature gates.

diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
@@ -1070,12 +1070,10 @@ const (
 	ReasonMissingObjectStorageSecret LokiStackConditionReason = "MissingObjectStorageSecret"
 	// ReasonInvalidObjectStorageSecret when the format of the secret is invalid.
 	ReasonInvalidObjectStorageSecret LokiStackConditionReason = "InvalidObjectStorageSecret"
-	// ReasonMissingCredentialsRequest when the required request for managed auth credentials to object
-	// storage is missing.
-	ReasonMissingCredentialsRequest LokiStackConditionReason = "MissingCredentialsRequest"
-	// ReasonMissingManagedAuthSecret when the required secret for managed auth credentials to object
-	// storage is missing.
-	ReasonMissingManagedAuthSecret LokiStackConditionReason = "MissingManagedAuthenticationSecret"
+	// ReasonMissingTokenCCOAuthSecret when the secret generated by CCO for token authentication is missing.
+	// This is usually a transient error because the secret is not immediately available after creating the
+	// CredentialsRequest, but it can persist if the CCO or its configuration are incorrect.
+	ReasonMissingTokenCCOAuthSecret LokiStackConditionReason = "MissingTokenCCOAuthenticationSecret"
 	// ReasonInvalidObjectStorageSchema when the spec contains an invalid schema(s).
 	ReasonInvalidObjectStorageSchema LokiStackConditionReason = "InvalidObjectStorageSchema"
 	// ReasonMissingObjectStorageCAConfigMap when the required configmap to verify object storage
@@ -1204,7 +1202,7 @@ type LokiStackComponentStatus struct {
 
 // CredentialMode represents the type of authentication used for accessing the object storage.
 //
-// +kubebuilder:validation:Enum=static;token;managed
+// +kubebuilder:validation:Enum=static;token;token-cco
 type CredentialMode string
 
 const (
@@ -1216,11 +1214,11 @@ const (
 	// Instead, they are generated during runtime using a service, which allows for shorter-lived credentials and
 	// much more granular control. This authentication mode is not supported for all object storage types.
 	CredentialModeToken CredentialMode = "token"
-	// CredentialModeManaged represents the usage of short-lived tokens retrieved from a credential source.
-	// This mode is similar to CredentialModeToken,but instead of having a user-configured credential source,
-	// it is configured by the environment, for example the Cloud Credential Operator in OpenShift.
-	// This mode is only supported for certain object storage types in certain runtime environments.
-	CredentialModeManaged CredentialMode = "managed"
+	// CredentialModeTokenCCO represents the usage of short-lived tokens retrieved from a credential source.
+	// This mode is similar to CredentialModeToken, but instead of having a user-configured credential source,
+	// it is configured by the environment and the operator relies on the Cloud Credential Operator to provide
+	// a secret. This mode is only supported for certain object storage types in certain runtime environments.
+	CredentialModeTokenCCO CredentialMode = "token-cco"
 )
 
 // LokiStackStorageStatus defines the observed state of

diff --git a/...penshift/manifests/loki-operator-controller-manager-metrics-reader_v1_serviceaccount.yaml b/...penshift/manifests/loki-operator-controller-manager-metrics-reader_v1_serviceaccount.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-metrics-reader
diff --git a/...nity-openshift/manifests/loki-operator-controller-manager-metrics-service_v1_service.yaml b/...nity-openshift/manifests/loki-operator-controller-manager-metrics-service_v1_service.yaml
@@ -5,6 +5,7 @@ metadata:
     service.beta.openshift.io/serving-cert-secret-name: loki-operator-metrics
   creationTimestamp: null
   labels:
+    app.kubernetes.io/component: metrics
     app.kubernetes.io/instance: loki-operator-v0.5.0
     app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: loki-operator

diff --git a/...mmunity-openshift/manifests/loki-operator-controller-manager-metrics-token_v1_secret.yaml b/...mmunity-openshift/manifests/loki-operator-controller-manager-metrics-token_v1_secret.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    kubernetes.io/service-account.name: loki-operator-controller-manager-metrics-reader
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-metrics-token
+type: kubernetes.io/service-account-token
diff --git a/...ator-controller-manager-read-metrics_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml b/...ator-controller-manager-read-metrics_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml
@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: loki-operator-v0.5.0
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: loki-operator
+    app.kubernetes.io/part-of: loki-operator
+    app.kubernetes.io/version: 0.5.0
+  name: loki-operator-controller-manager-read-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: loki-operator-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: loki-operator-controller-manager-metrics-reader
+  namespace: kubernetes-operators
diff --git a/...hift/manifests/loki-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml b/...hift/manifests/loki-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml
@@ -11,15 +11,23 @@ metadata:
   name: loki-operator-metrics-monitor
 spec:
   endpoints:
-  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+  - authorization:
+      credentials:
+        key: token
+        name: loki-operator-controller-manager-metrics-token
+      type: bearer
     interval: 30s
     path: /metrics
     scheme: https
     scrapeTimeout: 10s
     targetPort: 8443
     tlsConfig:
-      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      ca:
+        secret:
+          key: service-ca.crt
+          name: loki-operator-controller-manager-metrics-token
       serverName: loki-operator-controller-manager-metrics-service.kubernetes-operators.svc
   selector:
     matchLabels:
+      app.kubernetes.io/component: metrics
       app.kubernetes.io/name: loki-operator