set up Grafana Alloy (#7)

README.md

@@ -21,9 +21,10 @@ Clean up the project from the terminal using:
     docker compose down --rmi=all
     docker plugin disable loki
 
-## Enable AWS S3
+## Run in Production
 
 To use AWS S3, you need to create S3 bucket and upload video under `priv/videos/` folder.
+Moreover, you need to set up [Grafana Cloud](https://grafana.com/tutorials/) to view logs and metrics in Grafana Labs managed observability platform. You have to update credentials in `docker/alloy/endpoints.json`.
 
 Boot it up from the terminal using:
 

docker-compose-prod.yml

@@ -10,4 +10,25 @@ services:
       - AWS_REGION=${AWS_REGION}
       - S3_BUCKET=${S3_BUCKET}
     ports:
-      - '5454:5454'
+      - 5454:5454
+    labels:
+      metrics_enabled: true
+      app: 'video-streamer'
+
+  alloy:
+    image: grafana/alloy:v1.4.1
+    user: root # Requred to access the Docker socket in MacOS and Windows
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - './docker/alloy/config.alloy:/etc/alloy/config.alloy'
+      - './docker/alloy/endpoints.json:/etc/alloy/endpoints.json'
+    ports:
+      - 12345:12345
+    command:
+      [
+        'run',
+        '--server.http.listen-addr=0.0.0.0:12345',
+        '--stability.level=public-preview',
+        '/etc/alloy/config.alloy'
+      ]

docker-compose.yml

@@ -6,7 +6,6 @@ services:
         - MIX_ENV=dev
     ports:
       - 5454:5454
-      - 9568:9568
     labels:
       - app=web
       - instance=laptop
@@ -20,7 +19,7 @@ services:
   loki:
     image: grafana/loki:3.0.0
     ports:
-      - '3100:3100'
+      - 3100:3100
     volumes:
       - ./docker/loki/loki-config.yml:/etc/loki/loki-config.yml
       - loki-data-index:/loki-index
@@ -31,7 +30,7 @@ services:
     image: prom/prometheus
     user: root # Requred to access the Docker socket in MacOS and Windows
     ports:
-      - '9090:9090'
+      - 9090:9090
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - prometheus_data:/prometheus
@@ -56,7 +55,7 @@ services:
       - /run.sh
     image: grafana/grafana:11.2.0
     ports:
-      - '3000:3000'
+      - 3000:3000
     volumes:
       - grafana-data:/var/lib/grafana
       - ./docker/grafana/:/etc/grafana/provisioning/

docker/alloy/config.alloy

@@ -0,0 +1,100 @@
+// Configuration file
+local.file "endpoints" {
+	// The endpoints file is used to define the endpoints, credentials and options
+	// for the Alloy export to.
+	filename = "/etc/alloy/endpoints.json"
+}
+
+discovery.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5s"
+}
+
+loki.process "docker" {
+	forward_to = [loki.write.default.receiver]
+
+	stage.docker { }
+}
+
+discovery.relabel "docker" {
+	targets = []
+
+	rule {
+		source_labels = ["__meta_docker_container_label_app"]
+		regex         = ".+"
+		action        = "keep"
+	}
+
+	rule {
+		source_labels = ["__meta_docker_container_label_app"]
+		target_label  = "app"
+		action        = "replace"
+	}
+
+	rule {
+		target_label = "job"
+		replacement  = "docker"
+	}
+}
+
+loki.source.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	targets          = discovery.docker.docker.targets
+	forward_to       = [loki.process.docker.receiver]
+	relabel_rules    = discovery.relabel.docker.rules
+	refresh_interval = "5s"
+}
+
+loki.write "default" {
+	// Output the Loki log to the Loki instance.
+	endpoint {
+		url = json_path(local.file.endpoints.content, ".logs.url")[0]
+
+		// The basic auth credentials for the Loki instance.
+		basic_auth {
+			username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0]
+			password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0]
+		}
+	}
+}
+
+discovery.docker "prometheus" {
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5s"
+}
+
+discovery.relabel "prometheus" {
+	targets = discovery.docker.prometheus.targets
+
+	rule {
+		source_labels = ["__meta_docker_container_label_metrics_enabled"]
+		regex         = ".+"
+		action        = "keep"
+	}
+
+	rule {
+		source_labels = ["__meta_docker_container_label_app"]
+		target_label  = "app"
+		action        = "replace"
+	}
+}
+
+prometheus.scrape "prometheus" {
+	targets         = discovery.relabel.prometheus.output
+	forward_to      = [prometheus.remote_write.default.receiver]
+	job_name        = "prometheus"
+	scrape_interval = "15s"
+}
+
+prometheus.remote_write "default" {
+	// Output the Prometheus metrics to the Prometheus instance.
+	endpoint {
+		url = json_path(local.file.endpoints.content, ".metrics.url")[0]
+
+		// The basic auth credentials for the Prometheus instance.
+		basic_auth {
+			username = json_path(local.file.endpoints.content, ".metrics.basicAuth.username")[0]
+			password = json_path(local.file.endpoints.content, ".metrics.basicAuth.password")[0]
+		}
+	}
+}

docker/alloy/endpoints.json

@@ -0,0 +1,16 @@
+{
+	"metrics": {
+		"url": "https://prometheus-prod-link.grafana.net/api/prom/push",
+		"basicAuth": {
+			"username": "username",
+			"password": "password"
+		}
+	},
+	"logs": {
+		"url": "https://logs-prod-link.grafana.net/loki/api/v1/push",
+		"basicAuth": {
+			"username": "username",
+			"password": "password"
+		}
+	}
+}

lib/application.ex

@@ -1,16 +1,46 @@
 defmodule Video.Application do
   use Application
+  @port Application.compile_env!(:video_streamer, :port)
 
   @impl Application
   def start(_, _) do
+    args = [
+      metrics: metrics(),
+      port: @port
+    ]
+
+    opts = ensure_options(args)
+
     children = [
-      {TelemetryMetricsPrometheus, [metrics: metrics()]},
-      Video.Streamer
+      {TelemetryMetricsPrometheus.Core, opts},
+      {Video.Streamer, opts}
     ]
 
     Supervisor.start_link(children, strategy: :one_for_one)
   end
 
+  defp ensure_options(options) do
+    {port, updated_opts} =
+      Keyword.merge(default_options(), options)
+      |> Keyword.pop(:port)
+
+    Keyword.delete(updated_opts, :plug_cowboy_opts)
+    |> Keyword.update!(:options, fn opts ->
+      Keyword.merge(opts, Keyword.get(options, :plug_cowboy_opts, []))
+      |> Keyword.put(:port, port)
+      |> Keyword.put_new(:ref, Keyword.get(updated_opts, :name))
+    end)
+  end
+
+  defp default_options() do
+    [
+      protocol: :http,
+      name: :prometheus_metrics,
+      options: []
+    ]
+  end
+
+  @spec metrics() :: [Telemetry.Metrics.t()]
   def metrics,
     do: [
       Telemetry.Metrics.counter("video.metrics.watched", tags: [:file])

lib/streamer.ex

@@ -8,22 +8,43 @@ defmodule Video.Streamer do
   plug(Plug.Logger)
   plug(:match)
   plug(:dispatch)
+  plug(Plug.Telemetry, event_prefix: [:prometheus_metrics, :plug])
 
-  @port Application.compile_env!(:video_streamer, :port)
-
-  def child_spec(_arg) do
+  def child_spec(args) do
     Plug.Cowboy.child_spec(
-      scheme: :http,
-      options: [port: @port],
-      plug: __MODULE__
+      scheme: Keyword.get(args, :protocol),
+      options: Keyword.get(args, :options),
+      plug:
+        {__MODULE__,
+         [
+           name: Keyword.get(args, :name)
+         ]}
     )
   end
 
+  get "/metrics" do
+    name = :prometheus_metrics
+
+    metrics = TelemetryMetricsPrometheus.Core.scrape(name)
+
+    conn
+    |> put_private(:prometheus_metrics_name, name)
+    |> put_resp_content_type("text/plain")
+    |> send_resp(200, metrics)
+  end
+
+  @spec default_pre_scrape_handler() :: :ok
+  def default_pre_scrape_handler, do: :ok
+
   get "/video" do
     Metrics.viewed("SampleVideo_1280x720_1mb.mp4")
     stream_video(conn, "SampleVideo_1280x720_1mb.mp4")
   end
 
+  match _ do
+    send_resp(conn, 404, "Not Found")
+  end
+
   @spec stream_video(Plug.Conn.t(), String.t()) :: Plug.Conn.t()
   defp stream_video(conn, filename) do
     # Sanitize the filename to prevent directory traversal attacks

mix.exs

@@ -39,7 +39,7 @@ defmodule Video.MixProject do
       {:ex_aws_s3, "~> 2.5"},
       {:hackney, "~> 1.20"},
       {:sweet_xml, "~> 0.6"},
-      {:telemetry_metrics_prometheus, "~> 1.0"},
+      {:telemetry_metrics_prometheus_core, "~> 1.0"},
       {:telemetry, "~> 1.0"},
       {:mox, "~> 1.2", only: :test},
       {:excoveralls, "~> 0.18", only: :test}

mix.lock

@@ -23,7 +23,6 @@
   "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"},
-  "telemetry_metrics_prometheus": {:hex, :telemetry_metrics_prometheus, "1.1.0", "1cc23e932c1ef9aa3b91db257ead31ea58d53229d407e059b29bb962c1505a13", [:mix], [{:plug_cowboy, "~> 2.1", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}], "hexpm", "d43b3659b3244da44fe0275b717701542365d4519b79d9ce895b9719c1ce4d26"},
   "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
 }