feat: Add OpenTelemetry instrumentation

Cargo.toml

@@ -11,11 +11,18 @@ croner = "2.0.4"
 gix = { version = "0.63.0", features = ["blocking-http-transport-reqwest-rust-tls"] }
 human-errors = "0.1.3"
 openssl-sys = { version = "0.9", features=["vendored"], optional = true }
+opentelemetry = "0.23.0"
+opentelemetry-otlp = "0.16.0"
+opentelemetry_sdk = { version = "0.23.0", features = ["rt-tokio"] }
 parse_link_header = "0.3.3"
 reqwest = { version = "0.12.5", features = ["json"] }
 serde = { version = "1.0.204", features = ["derive", "alloc"] }
 serde_yaml = "0.9.34"
 tokio = { version = "1.38.0", features = ["macros", "rt", "rt-multi-thread"] }
+tonic = { version = "0.11", features = ["tls-webpki-roots"] }
+tracing = "0.1.40"
+tracing-opentelemetry = "0.24.0"
+tracing-subscriber = "0.3.18"
 
 [dev-dependencies]
 tempdir = "0.3.7"

examples/config.yaml

@@ -1,4 +1,5 @@
 schedule: "0 * * * *"
+github: {}
 backups:
   - user: notheotherben
     filters:

src/macros.rs

@@ -0,0 +1,21 @@
+#[cfg(not(debug_assertions))]
+#[macro_export]
+macro_rules! version {
+    () => {
+        env!("CARGO_PKG_VERSION")
+    };
+    ($prefix:expr) => {
+        format!("{}{}", $prefix, env!("CARGO_PKG_VERSION"))
+    };
+}
+
+#[cfg(debug_assertions)]
+#[macro_export]
+macro_rules! version {
+    () => {
+        "0.0.0-dev"
+    };
+    ($prefix:expr) => {
+        format!("{}0.0.0-dev", $prefix)
+    };
+}

src/main.rs

@@ -5,11 +5,15 @@ use tokio::task::JoinSet;
 use std::path::PathBuf;
 use std::sync::atomic::AtomicBool;
 
+#[macro_use]
+mod macros;
+
 mod config;
 mod errors;
 mod policy;
 mod sources;
 mod targets;
+mod telemetry;
 
 /// Backup your GitHub repositories automatically.
 #[derive(Parser, Debug)]
@@ -48,6 +52,8 @@ pub trait BackupEntity {
 }
 
 async fn run(args: Args) -> Result<(), Error> {
+    telemetry::setup();
+
     let config = config::Config::try_from(&args)?;
 
     let github = sources::GitHubSource::from(&config);
@@ -56,7 +62,11 @@ async fn run(args: Args) -> Result<(), Error> {
     let cancel = AtomicBool::new(false);
 
     loop {
+        let _span = tracing::info_span!("backup.all").entered();
+
         for policy in config.backups.iter() {
+            let _span = tracing::info_span!("backup.policy", policy = %policy).entered();
+
             println!("Backing up repositories for: {}", &policy);
             let repos = github.get_repos(policy, &cancel).await?;
 

src/sources/github.rs

@@ -1,6 +1,7 @@
 use std::{sync::Arc, path::PathBuf, sync::atomic::AtomicBool};
 
 use reqwest::{header::LINK, Method, StatusCode, Url};
+use tracing::instrument;
 
 use crate::{
     config::Config, errors, policy::{BackupPolicy, RepoFilter}, BackupEntity, RepositorySource
@@ -16,6 +17,7 @@ pub struct GitHubSource {
 
 #[async_trait::async_trait]
 impl RepositorySource<GitHubRepo> for GitHubSource {
+    #[instrument(skip(self, cancel))]
     async fn get_repos(
         &self,
         policy: &BackupPolicy,

src/targets/fs.rs

@@ -1,6 +1,7 @@
 use std::{path::{Path, PathBuf}, sync::{atomic::AtomicBool, Arc}};
 
 use gix::progress::Discard;
+use tracing::instrument;
 
 use crate::{config::Config, errors, BackupEntity, BackupTarget};
 
@@ -9,7 +10,8 @@ pub struct FileSystemBackupTarget {
     target: Arc<PathBuf>,
 }
 
-impl<T: BackupEntity> BackupTarget<T> for FileSystemBackupTarget {
+impl<T: BackupEntity + std::fmt::Debug> BackupTarget<T> for FileSystemBackupTarget {
+    #[instrument(skip(self, cancel))]
     fn backup(&self, repo: &T, cancel: &AtomicBool) -> Result<String, errors::Error> {
         std::fs::create_dir_all(self.target.as_ref()).map_err(|e| errors::user_with_internal(
             &format!("Unable to create backup directory '{}'", &self.target.display()),
@@ -133,6 +135,7 @@ mod tests {
         assert_eq!(id, id2, "the repository should not have changed between backups");
     }
 
+    #[derive(Debug)]
     struct MockTarget;
 
     impl BackupEntity for MockTarget {

src/telemetry/mod.rs

@@ -0,0 +1,120 @@
+use opentelemetry::global;
+use opentelemetry_otlp::WithExportConfig;
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::Sampler};
+use tracing::Subscriber;
+use tracing_subscriber::{prelude::*, registry::LookupSpan, Layer};
+
+pub fn setup() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::filter::LevelFilter::DEBUG)
+        .with(tracing_subscriber::filter::dynamic_filter_fn(
+            |_metadata, ctx| {
+                !ctx.lookup_current()
+                    // Exclude the rustls session "Connection" events which don't have a parent span
+                    .map(|s| s.parent().is_none() && s.name() == "Connection")
+                    .unwrap_or_default()
+            },
+        ))
+        .with(load_output_layer())
+        .init();
+}
+
+fn load_otlp_headers() -> tonic::metadata::MetadataMap {
+    let mut tracing_metadata = tonic::metadata::MetadataMap::new();
+
+    #[cfg(debug_assertions)]
+    tracing_metadata.insert(
+        "x-honeycomb-team",
+        "X6naTEMkzy10PMiuzJKifF".parse().unwrap(),
+    );
+
+    match std::env::var("OTEL_EXPORTER_OTLP_HEADERS").ok() {
+        Some(headers) if !headers.is_empty() => {
+            for header in headers.split_terminator(',') {
+                if let Some((key, value)) = header.split_once('=') {
+                    let key: &str = Box::leak(key.to_string().into_boxed_str());
+                    let value = value.to_owned();
+                    if let Ok(value) = value.parse() {
+                        tracing_metadata.insert(key, value);
+                    } else {
+                        eprintln!("Could not parse value for header {}.", key);
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+
+    tracing_metadata
+}
+
+fn load_trace_sampler() -> Sampler {
+    fn get_trace_ratio() -> f64 {
+        std::env::var("OTEL_TRACES_SAMPLER_ARG")
+            .ok()
+            .and_then(|ratio| ratio.parse().ok())
+            .unwrap_or(1.0)
+    }
+
+    std::env::var("OTEL_TRACES_SAMPLER")
+        .map(|s| match s.as_str() {
+            "always_on" => opentelemetry_sdk::trace::Sampler::AlwaysOn,
+            "always_off" => opentelemetry_sdk::trace::Sampler::AlwaysOff,
+            "traceidratio" => {
+                opentelemetry_sdk::trace::Sampler::TraceIdRatioBased(get_trace_ratio())
+            }
+            "parentbased_always_on" => opentelemetry_sdk::trace::Sampler::ParentBased(Box::new(
+                opentelemetry_sdk::trace::Sampler::AlwaysOn,
+            )),
+            "parentbased_always_off" => opentelemetry_sdk::trace::Sampler::ParentBased(Box::new(
+                opentelemetry_sdk::trace::Sampler::AlwaysOff,
+            )),
+            "parentbased_traceidratio" => opentelemetry_sdk::trace::Sampler::ParentBased(Box::new(
+                opentelemetry_sdk::trace::Sampler::TraceIdRatioBased(get_trace_ratio()),
+            )),
+            _ => opentelemetry_sdk::trace::Sampler::AlwaysOn,
+        })
+        .unwrap_or(opentelemetry_sdk::trace::Sampler::AlwaysOn)
+}
+
+fn load_output_layer<S>() -> Box<dyn Layer<S> + Send + Sync + 'static>
+where
+    S: Subscriber + Send + Sync,
+    for<'a> S: LookupSpan<'a>,
+{
+    #[cfg(not(debug_assertions))]
+    let tracing_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
+
+    #[cfg(debug_assertions)]
+    let tracing_endpoint = Some("https://api.honeycomb.io:443".to_string());
+
+    if let Some(endpoint) = tracing_endpoint {
+        let metadata = load_otlp_headers();
+        let tracer = opentelemetry_otlp::new_pipeline()
+            .tracing()
+            .with_exporter(
+                opentelemetry_otlp::new_exporter()
+                    .tonic()
+                    .with_endpoint(endpoint)
+                    .with_metadata(metadata),
+            )
+            .with_trace_config(
+                opentelemetry_sdk::trace::config()
+                    .with_resource(opentelemetry_sdk::Resource::new(vec![
+                        opentelemetry::KeyValue::new("service.name", "github-backup"),
+                        opentelemetry::KeyValue::new("service.version", version!("v")),
+                        opentelemetry::KeyValue::new("host.os", std::env::consts::OS),
+                        opentelemetry::KeyValue::new("host.architecture", std::env::consts::ARCH),
+                    ]))
+                    .with_sampler(load_trace_sampler()),
+            )
+            .install_batch(opentelemetry_sdk::runtime::Tokio)
+            .unwrap();
+
+        tracing_opentelemetry::layer().with_tracer(tracer).boxed()
+    } else {
+        tracing_subscriber::fmt::layer().boxed()
+    }
+}