From 70671ff07a7032e3625f37b6a20d1a7202d6a24c Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:25:45 +0200 Subject: [PATCH 01/41] Fix lint errors --- fake_data_generator/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fake_data_generator/src/main.rs b/fake_data_generator/src/main.rs index eca99b5..565c28f 100644 --- a/fake_data_generator/src/main.rs +++ b/fake_data_generator/src/main.rs @@ -154,7 +154,7 @@ async fn create_timeseries( 0, ) .unwrap() - - (period * ts_length.try_into().unwrap()) + - (period * ts_length) - year_skew; let end_time = present_time - year_skew; (period, start_time, end_time) @@ -197,7 +197,7 @@ async fn create_timeseries( print!("\r{}/{}", i, n_timeseries); } - println!(""); + println!(); Ok(timeseries) } @@ -307,7 +307,7 @@ async fn copy_in_data( } print!("\r{}/{}", i, timeseries_vec.len()); } - println!(""); + println!(); writer.finish().await?; tx.commit().await?; From 437b07a7cd6c90f7040320deed86b010c67ad7b7 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:39:17 +0200 Subject: [PATCH 02/41] Move main logic to lib file --- api/src/lib.rs | 143 +++++++++++++++++++++++++++ api/src/main.rs | 142 +-------------------------- ingestion/src/lib.rs | 215 +++++++++++++++++++++++++++++++++++++++++ ingestion/src/main.rs | 218 ++---------------------------------------- 4 files changed, 369 insertions(+), 349 deletions(-) create mode 100644 api/src/lib.rs create mode 100644 ingestion/src/lib.rs diff --git a/api/src/lib.rs b/api/src/lib.rs new file mode 100644 index 0000000..98adf3c --- /dev/null +++ b/api/src/lib.rs @@ -0,0 +1,143 @@ +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + routing::get, + Json, Router, +}; +use bb8_postgres::PostgresConnectionManager; +use chrono::{DateTime, Duration, Utc}; +use latest::{get_latest, LatestElem}; +use serde::{Deserialize, Serialize}; +use timeseries::{ + get_timeseries_data_irregular, get_timeseries_data_regular, get_timeseries_info, Timeseries, +}; +use timeslice::{get_timeslice, Timeslice}; +use tokio_postgres::NoTls; + +mod latest; +mod timeseries; +mod timeslice; +pub(crate) mod util; + +type PgConnectionPool = bb8::Pool>; + +/// Utility function for mapping any error into a `500 Internal Server Error` +/// response. +fn internal_error(err: E) -> (StatusCode, String) { + (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) +} + +#[derive(Debug, Deserialize)] +struct TimeseriesParams { + start_time: Option>, + end_time: Option>, + time_resolution: Option, +} + +#[derive(Debug, Serialize)] +struct TimeseriesResp { + tseries: Vec, +} + +#[derive(Debug, Serialize)] +struct TimesliceResp { + tslices: Vec, +} + +#[derive(Debug, Deserialize)] +struct LatestParams { + latest_max_age: Option>, +} + +#[derive(Debug, Serialize)] +struct LatestResp { + data: Vec, +} + +async fn stations_handler( + State(pool): State, + // TODO: this should probably take element_id instead of param_id and do a conversion + Path((station_id, param_id)): Path<(i32, i32)>, + Query(params): Query, +) -> Result, (StatusCode, String)> { + let conn = pool.get().await.map_err(internal_error)?; + + let header = get_timeseries_info(&conn, station_id, param_id) + .await + .map_err(internal_error)?; + + let start_time = params.start_time.unwrap_or(header.fromtime); + let end_time = params.end_time.unwrap_or(header.totime); + + let ts = if let Some(time_resolution) = params.time_resolution { + Timeseries::Regular( + get_timeseries_data_regular(&conn, header, start_time, end_time, time_resolution) + .await + .map_err(internal_error)?, + ) + } else { + Timeseries::Irregular( + get_timeseries_data_irregular(&conn, header, start_time, end_time) + .await + .map_err(internal_error)?, + ) + }; + + Ok(Json(TimeseriesResp { tseries: vec![ts] })) +} + +async fn timeslice_handler( + State(pool): State, + // TODO: this should probably take element_id instead of param_id and do a conversion + Path((timestamp, param_id)): Path<(DateTime, i32)>, +) -> Result, (StatusCode, String)> { + let conn = pool.get().await.map_err(internal_error)?; + + let slice = get_timeslice(&conn, timestamp, param_id) + .await + .map_err(internal_error)?; + + Ok(Json(TimesliceResp { + tslices: vec![slice], + })) +} + +async fn latest_handler( + State(pool): State, + Query(params): Query, +) -> Result, (StatusCode, String)> { + let conn = pool.get().await.map_err(internal_error)?; + + let latest_max_age = params + .latest_max_age + .unwrap_or_else(|| Utc::now() - Duration::hours(3)); + + let data = get_latest(&conn, latest_max_age) + .await + .map_err(internal_error)?; + + Ok(Json(LatestResp { data })) +} + +pub async fn run(connect_string: &str) { + // set up postgres connection pool + let manager = PostgresConnectionManager::new_from_stringlike(connect_string, NoTls).unwrap(); + let pool = bb8::Pool::builder().build(manager).await.unwrap(); + + // build our application with routes + let app = Router::new() + .route( + "/stations/:station_id/params/:param_id", + get(stations_handler), + ) + .route( + "/timeslices/:timestamp/params/:param_id", + get(timeslice_handler), + ) + .route("/latest", get(latest_handler)) + .with_state(pool); + + // run it with hyper on localhost:3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); +} diff --git a/api/src/main.rs b/api/src/main.rs index 3dc9055..28b4b6a 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -1,124 +1,3 @@ -use axum::{ - extract::{Path, Query, State}, - http::StatusCode, - routing::get, - Json, Router, -}; -use bb8_postgres::PostgresConnectionManager; -use chrono::{DateTime, Duration, Utc}; -use latest::{get_latest, LatestElem}; -use serde::{Deserialize, Serialize}; -use timeseries::{ - get_timeseries_data_irregular, get_timeseries_data_regular, get_timeseries_info, Timeseries, -}; -use timeslice::{get_timeslice, Timeslice}; -use tokio_postgres::NoTls; - -mod latest; -mod timeseries; -mod timeslice; -pub(crate) mod util; - -type PgConnectionPool = bb8::Pool>; - -/// Utility function for mapping any error into a `500 Internal Server Error` -/// response. -fn internal_error(err: E) -> (StatusCode, String) { - (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) -} - -#[derive(Debug, Deserialize)] -struct TimeseriesParams { - start_time: Option>, - end_time: Option>, - time_resolution: Option, -} - -#[derive(Debug, Serialize)] -struct TimeseriesResp { - tseries: Vec, -} - -#[derive(Debug, Serialize)] -struct TimesliceResp { - tslices: Vec, -} - -#[derive(Debug, Deserialize)] -struct LatestParams { - latest_max_age: Option>, -} - -#[derive(Debug, Serialize)] -struct LatestResp { - data: Vec, -} - -async fn stations_handler( - State(pool): State, - // TODO: this should probably take element_id instead of param_id and do a conversion - Path((station_id, param_id)): Path<(i32, i32)>, - Query(params): Query, -) -> Result, (StatusCode, String)> { - let conn = pool.get().await.map_err(internal_error)?; - - let header = get_timeseries_info(&conn, station_id, param_id) - .await - .map_err(internal_error)?; - - let start_time = params.start_time.unwrap_or(header.fromtime); - let end_time = params.end_time.unwrap_or(header.totime); - - let ts = if let Some(time_resolution) = params.time_resolution { - Timeseries::Regular( - get_timeseries_data_regular(&conn, header, start_time, end_time, time_resolution) - .await - .map_err(internal_error)?, - ) - } else { - Timeseries::Irregular( - get_timeseries_data_irregular(&conn, header, start_time, end_time) - .await - .map_err(internal_error)?, - ) - }; - - Ok(Json(TimeseriesResp { tseries: vec![ts] })) -} - -async fn timeslice_handler( - State(pool): State, - // TODO: this should probably take element_id instead of param_id and do a conversion - Path((timestamp, param_id)): Path<(DateTime, i32)>, -) -> Result, (StatusCode, String)> { - let conn = pool.get().await.map_err(internal_error)?; - - let slice = get_timeslice(&conn, timestamp, param_id) - .await - .map_err(internal_error)?; - - Ok(Json(TimesliceResp { - tslices: vec![slice], - })) -} - -async fn latest_handler( - State(pool): State, - Query(params): Query, -) -> Result, (StatusCode, String)> { - let conn = pool.get().await.map_err(internal_error)?; - - let latest_max_age = params - .latest_max_age - .unwrap_or_else(|| Utc::now() - Duration::hours(3)); - - let data = get_latest(&conn, latest_max_age) - .await - .map_err(internal_error)?; - - Ok(Json(LatestResp { data })) -} - #[tokio::main] async fn main() { let args: Vec = std::env::args().collect(); @@ -133,24 +12,5 @@ async fn main() { connect_string.push_str(&args[4]) } - // set up postgres connection pool - let manager = PostgresConnectionManager::new_from_stringlike(connect_string, NoTls).unwrap(); - let pool = bb8::Pool::builder().build(manager).await.unwrap(); - - // build our application with routes - let app = Router::new() - .route( - "/stations/:station_id/params/:param_id", - get(stations_handler), - ) - .route( - "/timeslices/:timestamp/params/:param_id", - get(timeslice_handler), - ) - .route("/latest", get(latest_handler)) - .with_state(pool); - - // run it with hyper on localhost:3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); + lard_api::run(&connect_string).await; } diff --git a/ingestion/src/lib.rs b/ingestion/src/lib.rs new file mode 100644 index 0000000..28f67f2 --- /dev/null +++ b/ingestion/src/lib.rs @@ -0,0 +1,215 @@ +use axum::{ + extract::{FromRef, State}, + response::Json, + routing::post, + Router, +}; +use bb8::PooledConnection; +use bb8_postgres::PostgresConnectionManager; +use chrono::{DateTime, Utc}; +use futures::{stream::FuturesUnordered, StreamExt}; +use serde::Serialize; +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; +use thiserror::Error; +use tokio_postgres::NoTls; + +pub mod permissions; +use permissions::{ParamPermitTable, StationPermitTable}; + +#[derive(Error, Debug)] +pub enum Error { + #[error("postgres returned an error: {0}")] + Database(#[from] tokio_postgres::Error), + #[error("database pool could not return a connection: {0}")] + Pool(#[from] bb8::RunError), + #[error("parse error: {0}")] + Parse(String), + #[error("RwLock was poisoned: {0}")] + Lock(String), + #[error("Could not read environment variable: {0}")] + Env(#[from] std::env::VarError), +} + +pub type PgConnectionPool = bb8::Pool>; + +pub type PooledPgConn<'a> = PooledConnection<'a, PostgresConnectionManager>; + +type ParamConversions = Arc>; + +#[derive(Clone, Debug)] +struct IngestorState { + db_pool: PgConnectionPool, + param_conversions: ParamConversions, // converts param codes to element ids + permit_tables: Arc>, +} + +impl FromRef for PgConnectionPool { + fn from_ref(state: &IngestorState) -> PgConnectionPool { + state.db_pool.clone() // the pool is internally reference counted, so no Arc needed + } +} + +impl FromRef for ParamConversions { + fn from_ref(state: &IngestorState) -> ParamConversions { + state.param_conversions.clone() + } +} + +impl FromRef for Arc> { + fn from_ref(state: &IngestorState) -> Arc> { + state.permit_tables.clone() + } +} + +/// Generic container for a piece of data ready to be inserted into the DB +pub struct Datum { + timeseries_id: i32, + timestamp: DateTime, + value: f32, +} + +pub type Data = Vec; + +pub async fn insert_data(data: Data, conn: &mut PooledPgConn<'_>) -> Result<(), Error> { + // TODO: the conflict resolution on this query is an imperfect solution, and needs improvement + // + // I learned from Søren that obsinn and kvalobs organise updates and deletions by sending new + // messages that overwrite previous messages. The catch is that the new message does not need + // to contain all the params of the old message (or indeed any of them), and any that are left + // out should be deleted. + // + // We either need to scan for and delete matching data for every request obsinn sends us, or + // get obsinn to adopt and use a new endpoint or message format to signify deletion. The latter + // option seems to me the much better solution, and Søren seemed receptive when I spoke to him, + // but we would need to hash out the details of such and endpoint/format with him before we can + // implement it here. + let query = conn + .prepare( + "INSERT INTO public.data (timeseries, obstime, obsvalue) \ + VALUES ($1, $2, $3) \ + ON CONFLICT ON CONSTRAINT unique_data_timeseries_obstime \ + DO UPDATE SET obsvalue = EXCLUDED.obsvalue", + ) + .await?; + + let mut futures = data + .iter() + .map(|datum| async { + conn.execute( + &query, + &[&datum.timeseries_id, &datum.timestamp, &datum.value], + ) + .await + }) + .collect::>(); + + while let Some(res) = futures.next().await { + res?; + } + + Ok(()) +} + +pub mod kldata; +use kldata::{filter_and_label_kldata, parse_kldata}; + +/// Format of response Obsinn expects from this API +#[derive(Debug, Serialize)] +struct KldataResp { + /// Optional message indicating what happened to the data + message: String, + /// Should be the same message_id we received in the request + message_id: usize, + /// Result indicator, 0 means success, anything else means fail. + // Kvalobs uses some specific numbers to denote specific errors with this, I don't much see + // the point, the only information Obsinn can really action on as far as I can tell, is whether + // we failed and whether it can retry + res: u8, // TODO: Should be an enum? + /// Indicates whether Obsinn should try to send the message again + retry: bool, +} + +async fn handle_kldata( + State(pool): State, + State(param_conversions): State, + State(permit_table): State>>, + body: String, +) -> Json { + let result: Result = async { + let mut conn = pool.get().await?; + + let (message_id, obsinn_chunk) = parse_kldata(&body)?; + + let data = + filter_and_label_kldata(obsinn_chunk, &mut conn, param_conversions, permit_table) + .await?; + + insert_data(data, &mut conn).await?; + + Ok(message_id) + } + .await; + + match result { + Ok(message_id) => Json(KldataResp { + message: "".into(), + message_id, + res: 0, + retry: false, + }), + Err(e) => Json(KldataResp { + message: e.to_string(), + message_id: 0, // TODO: some clever way to get the message id still if possible? + res: 1, + retry: !matches!(e, Error::Parse(_)), + }), + } +} + +pub async fn run( + connect_string: &str, + param_conversion_path: &str, + permit_tables: Arc>, +) -> Result<(), Box> { + // set up postgres connection pool + let manager = PostgresConnectionManager::new_from_stringlike(connect_string, NoTls)?; + let db_pool = bb8::Pool::builder().build(manager).await?; + + // set up param conversion map + // TODO: extract to separate function? + let param_conversions = Arc::new( + csv::Reader::from_path(param_conversion_path) + .unwrap() + .into_records() + .map(|record_result| { + record_result.map(|record| { + ( + record.get(1).unwrap().to_owned(), // param code + ( + record.get(2).unwrap().to_owned(), // element id + record.get(0).unwrap().parse::().unwrap(), // param id + ), + ) + }) + }) + .collect::, csv::Error>>()?, + ); + + // build our application with a single route + let app = Router::new() + .route("/kldata", post(handle_kldata)) + .with_state(IngestorState { + db_pool, + param_conversions, + permit_tables, + }); + + // run our app with hyper, listening globally on port 3001 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3001").await?; + axum::serve(listener, app).await?; + + Ok(()) +} diff --git a/ingestion/src/main.rs b/ingestion/src/main.rs index 54ecfb1..6e98ecf 100644 --- a/ingestion/src/main.rs +++ b/ingestion/src/main.rs @@ -1,175 +1,10 @@ -use axum::{ - extract::{FromRef, State}, - response::Json, - routing::post, - Router, -}; -use bb8::PooledConnection; -use bb8_postgres::PostgresConnectionManager; -use chrono::{DateTime, Utc}; -use futures::{stream::FuturesUnordered, StreamExt}; -use serde::Serialize; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; -use thiserror::Error; -use tokio_postgres::NoTls; +use lard_ingestion::permissions::fetch_permits; +use std::sync::{Arc, RwLock}; -pub mod permissions; -use permissions::{fetch_permits, ParamPermitTable, StationPermitTable}; - -#[derive(Error, Debug)] -pub enum Error { - #[error("postgres returned an error: {0}")] - Database(#[from] tokio_postgres::Error), - #[error("database pool could not return a connection: {0}")] - Pool(#[from] bb8::RunError), - #[error("parse error: {0}")] - Parse(String), - #[error("RwLock was poisoned: {0}")] - Lock(String), - #[error("Could not read environment variable: {0}")] - Env(#[from] std::env::VarError), -} - -type PgConnectionPool = bb8::Pool>; - -pub type PooledPgConn<'a> = PooledConnection<'a, PostgresConnectionManager>; - -type ParamConversions = Arc>; - -#[derive(Clone, Debug)] -struct IngestorState { - db_pool: PgConnectionPool, - param_conversions: ParamConversions, // converts param codes to element ids - permit_tables: Arc>, -} - -impl FromRef for PgConnectionPool { - fn from_ref(state: &IngestorState) -> PgConnectionPool { - state.db_pool.clone() // the pool is internally reference counted, so no Arc needed - } -} - -impl FromRef for ParamConversions { - fn from_ref(state: &IngestorState) -> ParamConversions { - state.param_conversions.clone() - } -} - -impl FromRef for Arc> { - fn from_ref(state: &IngestorState) -> Arc> { - state.permit_tables.clone() - } -} - -/// Generic container for a piece of data ready to be inserted into the DB -pub struct Datum { - timeseries_id: i32, - timestamp: DateTime, - value: f32, -} -pub type Data = Vec; - -async fn insert_data(data: Data, conn: &mut PooledPgConn<'_>) -> Result<(), Error> { - // TODO: the conflict resolution on this query is an imperfect solution, and needs improvement - // - // I learned from Søren that obsinn and kvalobs organise updates and deletions by sending new - // messages that overwrite previous messages. The catch is that the new message does not need - // to contain all the params of the old message (or indeed any of them), and any that are left - // out should be deleted. - // - // We either need to scan for and delete matching data for every request obsinn sends us, or - // get obsinn to adopt and use a new endpoint or message format to signify deletion. The latter - // option seems to me the much better solution, and Søren seemed receptive when I spoke to him, - // but we would need to hash out the details of such and endpoint/format with him before we can - // implement it here. - let query = conn - .prepare( - "INSERT INTO public.data (timeseries, obstime, obsvalue) \ - VALUES ($1, $2, $3) \ - ON CONFLICT ON CONSTRAINT unique_data_timeseries_obstime \ - DO UPDATE SET obsvalue = EXCLUDED.obsvalue", - ) - .await?; - - let mut futures = data - .iter() - .map(|datum| async { - conn.execute( - &query, - &[&datum.timeseries_id, &datum.timestamp, &datum.value], - ) - .await - }) - .collect::>(); - - while let Some(res) = futures.next().await { - res?; - } - - Ok(()) -} - -pub mod kldata; -use kldata::{filter_and_label_kldata, parse_kldata}; - -/// Format of response Obsinn expects from this API -#[derive(Debug, Serialize)] -struct KldataResp { - /// Optional message indicating what happened to the data - message: String, - /// Should be the same message_id we received in the request - message_id: usize, - /// Result indicator, 0 means success, anything else means fail. - // Kvalobs uses some specific numbers to denote specific errors with this, I don't much see - // the point, the only information Obsinn can really action on as far as I can tell, is whether - // we failed and whether it can retry - res: u8, // TODO: Should be an enum? - /// Indicates whether Obsinn should try to send the message again - retry: bool, -} - -async fn handle_kldata( - State(pool): State, - State(param_conversions): State, - State(permit_table): State>>, - body: String, -) -> Json { - let result: Result = async { - let mut conn = pool.get().await?; - - let (message_id, obsinn_chunk) = parse_kldata(&body)?; - - let data = - filter_and_label_kldata(obsinn_chunk, &mut conn, param_conversions, permit_table) - .await?; - - insert_data(data, &mut conn).await?; - - Ok(message_id) - } - .await; - - match result { - Ok(message_id) => Json(KldataResp { - message: "".into(), - message_id, - res: 0, - retry: false, - }), - Err(e) => Json(KldataResp { - message: e.to_string(), - message_id: 0, // TODO: some clever way to get the message id still if possible? - res: 1, - retry: !matches!(e, Error::Parse(_)), - }), - } -} +const PARAMCONV: &str = "resources/paramconversions.csv"; #[tokio::main] -pub async fn main() -> Result<(), Box> { +async fn main() -> Result<(), Box> { // TODO: use clap for argument parsing let args: Vec = std::env::args().collect(); @@ -181,35 +16,14 @@ pub async fn main() -> Result<(), Box> { if args.len() > 4 { connect_string.push_str(" password="); connect_string.push_str(&args[4]) - } - - // set up postgres connection pool - let manager = PostgresConnectionManager::new_from_stringlike(connect_string, NoTls)?; - let db_pool = bb8::Pool::builder().build(manager).await?; - - // set up param conversion map - let param_conversions = Arc::new( - csv::Reader::from_path("resources/paramconversions.csv") - .unwrap() - .into_records() - .map(|record_result| { - record_result.map(|record| { - ( - record.get(1).unwrap().to_owned(), // param code - ( - record.get(2).unwrap().to_owned(), // element id - record.get(0).unwrap().parse::().unwrap(), // param id - ), - ) - }) - }) - .collect::, csv::Error>>()?, - ); + }; + // Permit tables handling (needs connection to stinfosys database) + // TODO: we probably need to test this as well, ie move into separate function let permit_tables = Arc::new(RwLock::new(fetch_permits().await?)); let background_permit_tables = permit_tables.clone(); - // background task to refresh permit table every 30 mins + // background task to refresh permit tables every 30 mins tokio::task::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(30 * 60)); @@ -227,18 +41,6 @@ pub async fn main() -> Result<(), Box> { } }); - // build our application with a single route - let app = Router::new() - .route("/kldata", post(handle_kldata)) - .with_state(IngestorState { - db_pool, - param_conversions, - permit_tables, - }); - - // run our app with hyper, listening globally on port 3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3001").await?; - axum::serve(listener, app).await?; - - Ok(()) + // Set up and run our server + database + lard_ingestion::run(&connect_string, PARAMCONV, permit_tables).await } From 9a1d71e507cea8d0a5b07dcb37a1385d474b31c1 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:40:47 +0200 Subject: [PATCH 03/41] Wrap nullable types in Option --- api/src/latest.rs | 2 +- api/src/timeseries.rs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/src/latest.rs b/api/src/latest.rs index b1e5c73..48ccfda 100644 --- a/api/src/latest.rs +++ b/api/src/latest.rs @@ -7,7 +7,7 @@ pub struct LatestElem { value: f32, timestamp: DateTime, station_id: i32, - loc: Location, + loc: Option, } pub async fn get_latest( diff --git a/api/src/timeseries.rs b/api/src/timeseries.rs index 55658c0..5be680e 100644 --- a/api/src/timeseries.rs +++ b/api/src/timeseries.rs @@ -3,7 +3,6 @@ use chrono::{DateTime, Utc}; use serde::Serialize; // TODO: this should be more comprehensive once the schema supports it -// TODO: figure out what should be wrapped in Option here #[derive(Debug, Serialize)] pub struct TimeseriesInfo { pub ts_id: i32, @@ -11,9 +10,9 @@ pub struct TimeseriesInfo { pub totime: DateTime, station_id: i32, param_id: i32, - lvl: i32, - sensor: i32, - location: Location, + lvl: Option, + sensor: Option, + location: Option, } #[derive(Debug, Serialize)] @@ -127,6 +126,8 @@ pub async fn get_timeseries_data_regular( "P1D" => "1 day", _ => "1 minute", // FIXME: this should error instead of falling back to a default }; + + // TODO: this generates nulls till utc.now if end_time is not specified in the database? let query_string = format!("SELECT data.obsvalue, ts_rule.timestamp \ FROM (SELECT data.obsvalue, data.obstime FROM data WHERE data.timeseries = $1) as data RIGHT JOIN generate_series($2::timestamptz, $3::timestamptz, interval '{}') AS ts_rule(timestamp) \ From cd21a12e6c68ef2a101ab76a4477400c93f85194 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:47:48 +0200 Subject: [PATCH 04/41] Fix regex and how we handle its matches --- ingestion/src/kldata.rs | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index b7f4214..76b5dad 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -44,8 +44,12 @@ where )) })?; - // TODO: Replace assert with error? - assert_eq!(key, expected_key); + if key != expected_key { + return Err(Error::Parse(format!( + "wrong key in field: expected '{}', got '{}'", + expected_key, key + ))); + } let res = value .parse::() @@ -75,27 +79,27 @@ fn parse_meta(meta: &str) -> Result<(i32, i32, usize), Error> { fn parse_columns(cols_raw: &str) -> Result, Error> { // this regex is taken from kvkafka's kldata parser // let col_regex = Regex::new(r"([^(),]+)(\([0-9]+,[0-9]+\))?").unwrap(); + // It matches all comma separated fields with pattern of type `name` and `name(x,y)`, + // where `x` and `y` are ints // it is modified below to capture sensor and level separately, while keeping // the block collectively optional - // + // TODO: is it possible to reuse this regex even more? - let col_regex = Regex::new(r"([^(),]+)\(([0-9]+),([0-9]+)\)?").unwrap(); + let col_regex = Regex::new(r"([^(),]+)(\(([0-9]+),([0-9]+)\))?").unwrap(); + // TODO: gracefully handle errors here? Even though this shouldn't really ever panic? col_regex .captures_iter(cols_raw) - .map(|caps| match caps.len() { - 2 => Ok(ObsinnId { - param_code: caps.get(1).unwrap().as_str().to_owned(), - sensor_and_level: None, - }), - 4 => Ok(ObsinnId { + .map(|caps| { + Ok(ObsinnId { param_code: caps.get(1).unwrap().as_str().to_owned(), - sensor_and_level: Some(( - caps.get(2).unwrap().as_str().parse().unwrap(), - caps.get(3).unwrap().as_str().parse().unwrap(), - )), - }), - _ => Err(Error::Parse("malformed entry in kldata column".to_string())), + sensor_and_level: caps.get(2).map(|_| { + ( + caps.get(3).unwrap().as_str().parse().unwrap(), + caps.get(4).unwrap().as_str().parse().unwrap(), + ) + }), + }) }) .collect::, Error>>() } From e3cc4741fae89d5a5e4200ef724b95d17083be58 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:51:03 +0200 Subject: [PATCH 05/41] Use NaiveDateTime to parse Obsinn timestamps --- ingestion/src/kldata.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index 76b5dad..5d73e84 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -2,7 +2,7 @@ use crate::{ permissions::{timeseries_is_open, ParamPermitTable, StationPermitTable}, Datum, Error, PooledPgConn, }; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDateTime, Utc}; use regex::Regex; use std::{ collections::HashMap, @@ -106,28 +106,27 @@ fn parse_columns(cols_raw: &str) -> Result, Error> { fn parse_obs(csv_body: Lines<'_>, columns: &[ObsinnId]) -> Result, Error> { let mut obs = Vec::new(); + let line_is_empty = || Error::Parse("empty row in kldata csv".to_string()); for row in csv_body { let (timestamp, vals) = { let mut vals = row.split(','); - let raw_timestamp = vals - .next() - .ok_or_else(|| Error::Parse("empty row in kldata csv".to_string()))?; + let raw_timestamp = vals.next().ok_or_else(line_is_empty)?; // TODO: timestamp parsing needs to handle milliseconds and truncated timestamps? - let timestamp = DateTime::parse_from_str(raw_timestamp, "%Y%m%d%H%M%S") + let timestamp = NaiveDateTime::parse_from_str(raw_timestamp, "%Y%m%d%H%M%S") .map_err(|e| Error::Parse(e.to_string()))? - .with_timezone(&Utc); + .and_utc(); (timestamp, vals) }; for (i, val) in vals.enumerate() { - let col = columns[i].clone(); // Should we do some smart bounds-checking?? + // Should we do some smart bounds-checking?? + let col = columns[i].clone(); // TODO: parse differently based on param_code? - obs.push(ObsinnObs { timestamp, id: col, From 088bae32fb79cdc8110d66049e8f83e326b746b2 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:52:05 +0200 Subject: [PATCH 06/41] Return an error if there is no data --- ingestion/src/kldata.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index 5d73e84..8f6c448 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -137,6 +137,10 @@ fn parse_obs(csv_body: Lines<'_>, columns: &[ObsinnId]) -> Result } } + if obs.is_empty() { + return Err(line_is_empty()); + } + Ok(obs) } From 4ccaba2bd89536d2e3dd251001371af7e27cf27b Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:53:30 +0200 Subject: [PATCH 07/41] Fix SQL queries in filter_and_label_kldata --- ingestion/src/kldata.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index 8f6c448..fe7504a 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -183,8 +183,8 @@ pub async fn filter_and_label_kldata( WHERE nationalnummer = $1 \ AND type_id = $2 \ AND param_code = $3 \ - AND lvl = $4 \ - AND sensor = $5", + AND (($4::int IS NULL AND lvl IS NULL) OR (lvl = $4)) \ + AND (($5::int IS NULL AND sensor IS NULL) OR (sensor = $5))", ) .await?; @@ -208,6 +208,7 @@ pub async fn filter_and_label_kldata( chunk.type_id, param_id.to_owned(), )? { + // TODO: log that the timeseries is closed? continue; } @@ -236,6 +237,8 @@ pub async fn filter_and_label_kldata( Some(row) => row.get(0), None => { // create new timeseries + // TODO: currently we create a timeseries with null location + // In the future the location column should be moved to the timeseries metadata table let timeseries_id = transaction .query_one( "INSERT INTO public.timeseries (fromtime) VALUES ($1) RETURNING id", @@ -266,7 +269,7 @@ pub async fn filter_and_label_kldata( .execute( "INSERT INTO labels.met \ (timeseries, station_id, param_id, type_id, lvl, sensor) \ - VALUES ($1, $2, $3, $4, $5)", + VALUES ($1, $2, $3, $4, $5, $6)", &[ ×eries_id, &chunk.station_id, From 44fb9637dd55f55cb9e133a6e1de26ee4336e0d0 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:56:09 +0200 Subject: [PATCH 08/41] Add unit tests for kldata --- Cargo.toml | 1 + ingestion/Cargo.toml | 5 +- ingestion/src/kldata.rs | 175 +++++++++++++++++++++++++++++++++++++++- ingestion/src/lib.rs | 15 ++++ 4 files changed, 194 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e8ee076..cd103b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,3 +24,4 @@ serde = { version = "1.0.188", features = ["derive"] } thiserror = "1.0.56" tokio = { version = "1.33.0", features = ["rt-multi-thread", "macros"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4"] } +test-case = "3.3.1" diff --git a/ingestion/Cargo.toml b/ingestion/Cargo.toml index 761e259..ce9e208 100644 --- a/ingestion/Cargo.toml +++ b/ingestion/Cargo.toml @@ -7,7 +7,7 @@ edition.workspace = true axum.workspace = true bb8.workspace = true bb8-postgres.workspace = true -bytes. workspace = true +bytes.workspace = true chrono.workspace = true csv.workspace = true futures.workspace = true @@ -16,3 +16,6 @@ serde.workspace = true thiserror.workspace = true tokio.workspace = true tokio-postgres.workspace = true + +[dev-dependencies] +test-case.workspace = true diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index fe7504a..9047bcd 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -13,6 +13,7 @@ use std::{ /// Represents a set of observations that came in the same message from obsinn, with shared /// station_id and type_id +#[derive(Debug, PartialEq)] pub struct ObsinnChunk { observations: Vec, station_id: i32, // TODO: change name here to nationalnummer? @@ -20,6 +21,7 @@ pub struct ObsinnChunk { } /// Represents a single observation from an obsinn message +#[derive(Debug, PartialEq)] pub struct ObsinnObs { timestamp: DateTime, id: ObsinnId, @@ -27,7 +29,7 @@ pub struct ObsinnObs { } /// Identifier for a single observation within a given obsinn message -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] struct ObsinnId { param_code: String, sensor_and_level: Option<(i32, i32)>, @@ -296,3 +298,174 @@ pub async fn filter_and_label_kldata( Ok(data) } + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + use test_case::test_case; + + use super::*; + + #[test_case( + ("nationalnr=99999", "nationalnr") => Ok(99999); + "parsing nationalnr" + )] + #[test_case( + ("type=508", "type") => Ok(508); + "parsing type" + )] + #[test_case( + ("messageid=23", "messageid") => Ok(23); + "parsing messageid" + )] + #[test_case( + ("unexpected", "messageid") => Err(Error::Parse("unexpected field in kldata header format: unexpected".to_string())); + "unexpected field" + )] + #[test_case( + ("unexpected=10", "messageid") => Err(Error::Parse("wrong key in field: expected 'messageid', got 'unexpected'".to_string())); + "unexpected key" + )] + fn test_parse_meta_field((field, key): (&str, &'static str)) -> Result { + parse_meta_field(field, key) + } + + #[test_case( + "Test message that fails." => Err(Error::Parse("kldata indicator missing or out of order".to_string())); + "missing kldata indicator" + )] + #[test_case( + "kldata/nationalnr=100/type=504/messageid=25" => Ok((100, 504, 25)); + "valid header 1" + )] + #[test_case( + "kldata/nationalnr=99993/type=508/messageid=23" => Ok((99993, 508, 23)); + "valid header 2" + )] + #[test_case( + "kldata/nationalnr=93140/type=501/add" => Err(Error::Parse("unexpected field in kldata header format: add".to_string())); + "unexpected field" + )] + #[test_case( + "kldata/nationalnr=40510/type=501" => Err(Error::Parse("kldata header terminated early".to_string())); + "missing messageid" + )] + fn test_parse_meta(msg: &str) -> Result<(i32, i32, usize), Error> { + parse_meta(msg) + } + + #[test_case( + "KLOBS,QSI_01(0,0)" => Ok(vec![ + ObsinnId{param_code: "KLOBS".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "QSI_01".to_string(), sensor_and_level: Some((0,0))} + ]); + "match 1" + )] + #[test_case( + "param_1,param_2,QSI_01(0,0)" => Ok(vec![ + ObsinnId{param_code: "param_1".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "QSI_01".to_string(), sensor_and_level: Some((0,0))} + ]); + "match 2" + )] + #[test_case( + "param_1(0,0),param_2,param_3(0,0)" => Ok(vec![ + ObsinnId{param_code: "param_1".to_string(), sensor_and_level: Some((0,0))}, + ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "param_3".to_string(), sensor_and_level: Some((0,0))} + ]); + "match 3" + )] + // NOTE: cases not taken into account here + // - "()" => Vec::new() + // - "param(0.1,0)" => vec[param, 0.1, 0] + // - "param(0,0.1)" => vec[param, 0.1, 0] + fn test_parse_columns(cols: &str) -> Result, Error> { + parse_columns(cols) + } + + #[test_case( + "20160201054100,-1.1,0,2.80", + &[ + ObsinnId{param_code: "param_1".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "param_3".to_string(), sensor_and_level: None}, + ] => Ok(vec![ + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 41, 0).unwrap(), + id: ObsinnId{param_code: "param_1".to_string(), sensor_and_level: None}, + value: -1.1 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 41, 0).unwrap(), + id: ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + value: 0.0 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 41, 0).unwrap(), + id: ObsinnId{param_code: "param_3".to_string(), sensor_and_level: None}, + value: 2.8 + }, + ]); + "single line" + )] + #[test_case( + "20160201054100,-1.1,0,2.80\n20160201055100,-1.5,1,2.90", + &[ + ObsinnId{param_code: "param_1".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + ObsinnId{param_code: "param_3".to_string(), sensor_and_level: None}, + ] => Ok(vec![ + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 41, 0).unwrap(), + id: ObsinnId{param_code: "param_1".to_string(), sensor_and_level: None}, + value: -1.1 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 41, 0).unwrap(), + id: ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + value: 0.0 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 41, 0).unwrap(), + id: ObsinnId{param_code: "param_3".to_string(), sensor_and_level: None}, + value: 2.8 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 51, 0).unwrap(), + id: ObsinnId{param_code: "param_1".to_string(), sensor_and_level: None}, + value: -1.5 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 51, 0).unwrap(), + id: ObsinnId{param_code: "param_2".to_string(), sensor_and_level: None}, + value: 1.0 + }, + ObsinnObs{ + timestamp: Utc.with_ymd_and_hms(2016,2, 1, 5, 51, 0).unwrap(), + id: ObsinnId{param_code: "param_3".to_string(), sensor_and_level: None}, + value: 2.9 + }, + ]); + "multiple lines" + )] + fn test_parse_obs(data: &str, cols: &[ObsinnId]) -> Result, Error> { + parse_obs(data.lines(), cols) + } + + // NOTE: just test for basic failures, the happy path should already be captured by the other tests + #[test_case("" => Err(Error::Parse("kldata message contained too few lines".to_string())); + "empty line" + )] + #[test_case("kldata/nationalnr=99993/type=508/messageid=23" => Err(Error::Parse("kldata message contained too few lines".to_string())); + "header only" + )] + #[test_case("kldata/nationalnr=93140/type=501/messageid=23 +DD(0,0),FF(0,0),DG_1(0,0),FG_1(0,0),KLFG_1(0,0),FX_1(0,0)" => Err(Error::Parse("empty row in kldata csv".to_string())); + "missing data" + )] + fn test_parse_kldata(body: &str) -> Result<(usize, ObsinnChunk), Error> { + parse_kldata(body) + } +} diff --git a/ingestion/src/lib.rs b/ingestion/src/lib.rs index 28f67f2..062045d 100644 --- a/ingestion/src/lib.rs +++ b/ingestion/src/lib.rs @@ -33,6 +33,21 @@ pub enum Error { Env(#[from] std::env::VarError), } +impl PartialEq for Error { + fn eq(&self, other: &Self) -> bool { + use Error::*; + + match (self, other) { + (Database(a), Database(b)) => a.to_string() == b.to_string(), + (Pool(a), Pool(b)) => a.to_string() == b.to_string(), + (Parse(a), Parse(b)) => a == b, + (Lock(a), Lock(b)) => a == b, + (Env(a), Env(b)) => a.to_string() == b.to_string(), + _ => false, + } + } +} + pub type PgConnectionPool = bb8::Pool>; pub type PooledPgConn<'a> = PooledConnection<'a, PostgresConnectionManager>; From 1d448a9aefde567d57f1c3ce6ca230c68c32aa77 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:58:03 +0200 Subject: [PATCH 09/41] Add binary to populate a test database --- Cargo.lock | 1212 ++++++++++++++++++++++++------ Cargo.toml | 5 +- lard_tests/Cargo.toml | 23 + lard_tests/src/bin/prepare_db.rs | 339 +++++++++ 4 files changed, 1358 insertions(+), 221 deletions(-) create mode 100644 lard_tests/Cargo.toml create mode 100644 lard_tests/src/bin/prepare_db.rs diff --git a/Cargo.lock b/Cargo.lock index fc35f44..59bf96f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -43,26 +43,32 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core", @@ -84,7 +90,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "tokio", "tower", "tower-layer", @@ -107,7 +113,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower-layer", "tower-service", "tracing", @@ -115,9 +121,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -130,18 +136,23 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bb8" -version = "0.8.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" +checksum = "b10cf871f3ff2ce56432fddc2615ac7acc3aa22ca321f8fea800846fbb32f188" dependencies = [ "async-trait", - "futures-channel", "futures-util", "parking_lot", "tokio", @@ -165,6 +176,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "block-buffer" version = "0.10.4" @@ -176,9 +193,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -188,18 +205,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" [[package]] name = "cfg-if" @@ -209,9 +223,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -219,20 +233,30 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.5", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -279,12 +303,42 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fake_data_generator" version = "0.1.0" @@ -306,10 +360,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] -name = "finl_unicode" -version = "1.2.0" +name = "fastrand" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fnv" @@ -317,20 +371,35 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -343,9 +412,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -353,15 +422,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -370,15 +439,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -387,21 +456,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -427,9 +496,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -438,21 +507,21 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "h2" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http", "indexmap", "slab", @@ -463,9 +532,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heck" @@ -475,9 +544,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hmac" @@ -490,9 +559,9 @@ dependencies = [ [[package]] name = "http" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -511,9 +580,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", @@ -524,9 +593,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" @@ -536,9 +605,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", @@ -550,14 +619,32 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", "tokio", + "tokio-native-tls", + "tower-service", ] [[package]] name = "hyper-util" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -568,14 +655,16 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower", + "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -594,27 +683,163 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +dependencies = [ + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", +] + [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -646,16 +871,33 @@ dependencies = [ "futures", "regex", "serde", + "test-case", "thiserror", "tokio", "tokio-postgres", ] +[[package]] +name = "lard_tests" +version = "0.1.0" +dependencies = [ + "bb8", + "bb8-postgres", + "chrono", + "lard_api", + "lard_ingestion", + "reqwest", + "serde", + "test-case", + "tokio", + "tokio-postgres", +] + [[package]] name = "libc" -version = "0.2.150" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" @@ -663,11 +905,23 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -675,9 +929,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "matchit" @@ -697,9 +951,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" @@ -709,29 +963,46 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", ] [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -749,24 +1020,68 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -774,22 +1089,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.2", "smallvec", - "windows-targets", + "windows-targets 0.52.5", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "phf" @@ -811,18 +1126,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", @@ -831,9 +1146,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -841,6 +1156,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "postgres-derive" version = "0.4.5" @@ -859,7 +1180,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator", @@ -892,18 +1213,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -954,14 +1275,23 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +dependencies = [ + "bitflags 2.5.0", ] [[package]] name = "regex" -version = "1.10.2" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -971,9 +1301,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -982,48 +1312,151 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - -[[package]] -name = "ryu" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" - -[[package]] -name = "scopeguard" -version = "1.2.0" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] -name = "serde" -version = "1.0.192" +name = "reqwest" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ - "serde_derive", -] - + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", @@ -1032,9 +1465,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -1043,9 +1476,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -1091,29 +1524,35 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] @@ -1124,9 +1563,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.48" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -1139,26 +1578,119 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "test-case-core", +] + [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1176,9 +1708,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -1189,20 +1721,30 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.10" @@ -1231,16 +1773,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -1291,6 +1832,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.17.0" @@ -1299,9 +1846,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -1311,30 +1858,80 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + +[[package]] +name = "url" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1342,9 +1939,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -1355,11 +1952,23 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1367,9 +1976,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -1380,15 +1989,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.88" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -1396,21 +2005,22 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "wasm-bindgen", + "redox_syscall 0.4.1", + "wasite", "web-sys", ] [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.5", ] [[package]] @@ -1419,7 +2029,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", ] [[package]] @@ -1428,13 +2047,29 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -1443,38 +2078,175 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index cd103b2..f309fbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,10 @@ members = [ "fake_data_generator", "api", - "ingestion" + "ingestion", + "lard_tests", ] +resolver = "2" [workspace.package] edition = "2021" @@ -24,4 +26,5 @@ serde = { version = "1.0.188", features = ["derive"] } thiserror = "1.0.56" tokio = { version = "1.33.0", features = ["rt-multi-thread", "macros"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4"] } +reqwest = {version = "0.12.4", features = ["json"]} test-case = "3.3.1" diff --git a/lard_tests/Cargo.toml b/lard_tests/Cargo.toml new file mode 100644 index 0000000..0c21132 --- /dev/null +++ b/lard_tests/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "lard_tests" +version = "0.1.0" +edition.workspace = true + +[dependencies] +lard_api = { path = "../api" } +lard_ingestion = { path = "../ingestion" } +chrono.workspace = true +tokio.workspace = true +tokio-postgres.workspace = true +bb8.workspace = true +bb8-postgres.workspace = true +serde.workspace = true + +[dev-dependencies] +reqwest.workspace = true +test-case.workspace = true + +[[bin]] +name = "prepare_db" +test = false +bench = false diff --git a/lard_tests/src/bin/prepare_db.rs b/lard_tests/src/bin/prepare_db.rs new file mode 100644 index 0000000..d41303f --- /dev/null +++ b/lard_tests/src/bin/prepare_db.rs @@ -0,0 +1,339 @@ +// use std::env; +use std::fs; + +use chrono::{DateTime, Duration, DurationRound, TimeDelta, TimeZone, Utc}; +use tokio_postgres::{Client, Error, NoTls}; + +const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; + +struct Param<'a> { + id: i32, + code: &'a str, +} + +// TODO: maybe merge into fake_data_generator, a lot of the code is shared +struct Labels<'a> { + // Assigned automatically + // timeseries: i32, + station_id: i32, + param: Param<'a>, + type_id: i32, + level: Option, + sensor: Option, +} + +struct Location { + lat: f32, + lon: f32, + // hamsl: f32, + // hag: f32, +} + +struct Timeseries { + from: DateTime, + to: DateTime, + period: Duration, + // len: i32, + deactivated: bool, + loc: Location, +} + +impl Timeseries { + fn new( + from: DateTime, + period: Duration, + len: i32, + loc: Location, + deactivated: bool, + ) -> Self { + Timeseries { + from, + to: from + period * len, + period, + // len, + loc, + deactivated, + } + } +} + +struct Case<'a> { + title: &'a str, + ts: Timeseries, + meta: Labels<'a>, +} + +async fn create_single_ts(client: &Client, ts: Timeseries) -> Result { + // Insert timeseries + let id = match ts.deactivated { + true => client + .query_one( + "INSERT INTO public.timeseries (fromtime, totime, loc.lat, loc.lon, deactivated) + VALUES ($1, $2, $3, $4, true) RETURNING id", + &[&ts.from, &ts.to, &ts.loc.lat, &ts.loc.lon], + ) + .await? + .get(0), + false => client + .query_one( + "INSERT INTO public.timeseries (fromtime, loc.lat, loc.lon, deactivated) + VALUES ($1, $2, $3, false) RETURNING id", + &[&ts.from, &ts.loc.lat, &ts.loc.lon], + ) + .await? + .get(0), + }; + + // insert data + let mut value: f32 = 0.0; + let mut time = ts.from; + while time <= ts.to { + client + .execute( + "INSERT INTO public.data (timeseries, obstime, obsvalue) + VALUES ($1, $2, $3)", + &[&id, &time, &value], + ) + .await?; + time += ts.period; + value += 1.0; + } + + Ok(id) +} + +async fn insert_ts_metadata<'a>(client: &Client, id: i32, meta: Labels<'a>) -> Result<(), Error> { + client + .execute( + "INSERT INTO labels.met (timeseries, station_id, param_id, type_id, lvl, sensor) + VALUES($1, $2, $3, $4, $5, $6)", + &[ + &id, + &meta.station_id, + &meta.param.id, + &meta.type_id, + &meta.level, + &meta.sensor, + ], + ) + .await?; + + client + .execute( + "INSERT INTO labels.obsinn (timeseries, nationalnummer, type_id, param_code, lvl, sensor) + VALUES($1, $2, $3, $4, $5, $6)", + &[ + &id, + &meta.station_id, + &meta.type_id, + &meta.param.code, + &meta.level, + &meta.sensor, + ], + ) + .await?; + + Ok(()) +} + +async fn create_timeseries(client: &Client) -> Result<(), Error> { + let cases = vec![ + Case { + title: "Daily, active", + ts: Timeseries::new( + Utc.with_ymd_and_hms(1970, 6, 5, 0, 0, 0).unwrap(), + Duration::days(1), + 19, + Location { + lat: 59.9, + lon: 10.4, + }, + false, + ), + meta: Labels { + station_id: 10000, + param: Param { + id: 103, + code: "EV_24", // sum(water_evaporation_amount) + }, + type_id: 1, // Is there a type_id for daily data? + level: Some(0), + sensor: Some(0), + }, + }, + Case { + title: "Hourly, active", + ts: Timeseries::new( + Utc.with_ymd_and_hms(2012, 2, 14, 0, 0, 0).unwrap(), + Duration::hours(1), + 47, + Location { + lat: 46.0, + lon: -73.0, + }, + false, + ), + meta: Labels { + station_id: 11000, + param: Param { + id: 222, + code: "TGM", // mean(grass_temperature) + }, + type_id: 501, // hourly data + level: Some(0), + sensor: Some(0), + }, + }, + Case { + title: "Minutely, active 1", + ts: Timeseries::new( + Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), + Duration::minutes(1), + 99, + Location { + lat: 65.89, + lon: 13.61, + }, + false, + ), + meta: Labels { + station_id: 12000, + param: Param { + id: 211, + code: "TA", // air_temperature + }, + type_id: 508, // minute data + level: None, + sensor: None, + }, + }, + Case { + title: "Minutely, active 2", + ts: Timeseries::new( + Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), + Duration::minutes(1), + 99, + Location { + lat: 66.0, + lon: 14.0, + }, + false, + ), + meta: Labels { + station_id: 12100, + param: Param { + id: 255, + code: "TWD", // sea_water_temperature + }, + type_id: 508, // minute data + level: Some(0), + sensor: Some(0), + }, + }, + Case { + // use it to test latest endpoint without optional query + title: "3hrs old minute data", + ts: Timeseries::new( + Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() - Duration::minutes(179), + Duration::minutes(1), + 179, + Location { lat: 1.0, lon: 1.0 }, + false, + ), + meta: Labels { + station_id: 20000, + param: Param { + id: 211, + code: "TA", // air_temperature + }, + type_id: 508, // minute data + level: Some(0), + sensor: Some(0), + }, + }, + Case { + // use it to test stations endpoint with optional time resolution (PT1H) + title: "Air temperature over the last 12 hours", + ts: Timeseries::new( + // TODO: check that this adds the correct number of data points every time + Utc::now().duration_trunc(TimeDelta::hours(1)).unwrap() - Duration::hours(11), + Duration::hours(1), + 11, + Location { lat: 2.0, lon: 2.0 }, + false, + ), + meta: Labels { + station_id: 30000, + param: Param { + id: 211, + code: "TA", // air_temperature + }, + type_id: 501, // hourly data + level: Some(0), + sensor: Some(0), + }, + }, + ]; + + for case in cases { + println!("Inserting timeseries: {}", case.title); + let id = create_single_ts(client, case.ts).await?; + insert_ts_metadata(client, id, case.meta).await?; + } + + Ok(()) +} + +async fn cleanup(client: &Client) -> Result<(), Error> { + client + .batch_execute("DROP TABLE IF EXISTS timeseries, data, labels.met, labels.obsinn CASCADE") + .await?; + client.batch_execute("DROP TYPE IF EXISTS location").await?; + + Ok(()) +} + +async fn create_schema(client: &Client) -> Result<(), Error> { + let public_schema = + fs::read_to_string("db/public.sql").expect("Should be able to read SQL file"); + client.batch_execute(public_schema.as_str()).await?; + + let labels_schema = + fs::read_to_string("db/labels.sql").expect("Should be able to read SQL file"); + client.batch_execute(labels_schema.as_str()).await?; + + Ok(()) +} + +async fn create_partitions(client: &Client) -> Result<(), Error> { + // TODO: add multiple partitions? + let partition_string = format!( + "CREATE TABLE data_y{}_to_y{} PARTITION OF public.data FOR VALUES FROM ('{}') TO ('{}')", + "1950", "2100", "1950-01-01 00:00:00+00", "2100-01-01 00:00:00+00", + ); + client.batch_execute(partition_string.as_str()).await?; + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + // let _test_type: String = env::args() + // .next() + // .expect("Provide test type for database setup ('api', 'ingestion', 'e2e')"); + + let (client, connection) = tokio_postgres::connect(CONNECT_STRING, NoTls).await?; + + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("connection error: {}", e); + } + }); + + cleanup(&client).await?; + create_schema(&client).await?; + create_partitions(&client).await?; + create_timeseries(&client).await?; + + Ok(()) +} From 469ba38502c5a49241dbb7ea8db5c819571a565b Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 13:59:42 +0200 Subject: [PATCH 10/41] Add common functions for integration tests --- ingestion/src/permissions.rs | 12 ++++ lard_tests/tests/common/mod.rs | 120 +++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 lard_tests/tests/common/mod.rs diff --git a/ingestion/src/permissions.rs b/ingestion/src/permissions.rs index f5c089f..e09bec3 100644 --- a/ingestion/src/permissions.rs +++ b/ingestion/src/permissions.rs @@ -12,6 +12,18 @@ pub struct ParamPermit { permit_id: i32, } +// Only used in tests +#[doc(hidden)] +impl ParamPermit { + pub fn new(type_id: i32, param_id: i32, permit_id: i32) -> ParamPermit { + ParamPermit { + type_id, + param_id, + permit_id, + } + } +} + type StationId = i32; /// This integer is used like an enum in stinfosys to define who data can be shared with. For /// details on what each number means, refer to the `permit` table in stinfosys. Here we mostly diff --git a/lard_tests/tests/common/mod.rs b/lard_tests/tests/common/mod.rs new file mode 100644 index 0000000..f62d664 --- /dev/null +++ b/lard_tests/tests/common/mod.rs @@ -0,0 +1,120 @@ +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use bb8::Pool; +use bb8_postgres::PostgresConnectionManager; +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use tokio::task::JoinHandle; +use tokio_postgres::NoTls; + +use lard_ingestion::permissions::{ParamPermit, ParamPermitTable, StationPermitTable}; +use lard_ingestion::{PgConnectionPool, PooledPgConn}; + +const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; +pub const PARAMCONV_CSV: &str = "../ingestion/resources/paramconversions.csv"; + +#[derive(Debug, Deserialize)] +pub struct IngestorResponse { + pub message: String, + pub message_id: usize, + pub res: u8, + pub retry: bool, +} + +#[derive(Debug, Deserialize)] +pub struct StationsResponse { + pub tseries: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Tseries { + pub regularity: String, + pub data: Vec, + // header: ... +} + +#[derive(Debug, Deserialize)] +pub struct LatestResponse { + pub data: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Data { + // TODO: Missing param_id here? + pub value: f64, + pub timestamp: DateTime, + pub station_id: i32, + // loc: {lat, lon, hamsl, hag} +} + +#[derive(Debug, Deserialize)] +pub struct TimesliceResponse { + pub tslices: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Tslice { + pub timestamp: DateTime, + pub param_id: i32, + pub data: Vec, +} + +pub async fn init_api_server() -> JoinHandle<()> { + tokio::spawn(lard_api::run(CONNECT_STRING)) +} + +pub async fn init_db_pool() -> Result { + let manager = PostgresConnectionManager::new_from_stringlike(CONNECT_STRING, NoTls)?; + let pool = Pool::builder().build(manager).await?; + Ok(pool) +} + +pub fn mock_permit_tables() -> Arc> { + let param_permit = HashMap::from([ + // station_id -> (type_id, param_id, permit_id) + (1, vec![ParamPermit::new(0, 0, 0)]), + (2, vec![ParamPermit::new(0, 0, 1)]), // open + ]); + + let station_permit = HashMap::from([ + // station_id -> permit_id + (1, 0), + (2, 0), + (3, 0), + (4, 1), // open + // used in e2e tests + (20000, 0), + (11000, 1), + (12000, 1), + (12100, 1), + (13000, 1), + (40000, 1), + ]); + + Arc::new(RwLock::new((param_permit, station_permit))) +} + +pub async fn number_of_data_rows(conn: &PooledPgConn<'_>, ts_id: i32) -> usize { + let rows = conn + .query( + "SELECT * FROM public.data + WHERE timeseries = $1", + &[&ts_id], + ) + .await + .unwrap(); + + rows.len() +} + +pub async fn init_ingestion_server( +) -> JoinHandle>> { + tokio::spawn(lard_ingestion::run( + CONNECT_STRING, + PARAMCONV_CSV, + mock_permit_tables(), + )) +} From d7676861de9026c81df43a37a587611f0c644b0e Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 14:00:16 +0200 Subject: [PATCH 11/41] Add api integration tests --- lard_tests/tests/api.rs | 123 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 lard_tests/tests/api.rs diff --git a/lard_tests/tests/api.rs b/lard_tests/tests/api.rs new file mode 100644 index 0000000..4a2543b --- /dev/null +++ b/lard_tests/tests/api.rs @@ -0,0 +1,123 @@ +use std::future::Future; + +use chrono::{TimeZone, Utc}; + +pub mod common; + +async fn api_test_wrapper>(test: T) { + let server = common::init_api_server().await; + + tokio::select! { + _ = server => panic!("Server task terminated first"), + _ = test => {} + } +} + +// TODO: test getting all tseries without specifying param_id +#[tokio::test] +async fn test_stations_endpoint_irregular() { + api_test_wrapper(async { + let station_id = 10000; + let param_id = 103; + let expected_data_len = 20; + + let url = format!( + "http://localhost:3000/stations/{}/params/{}", + station_id, param_id + ); + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let resp: common::StationsResponse = resp.json().await.unwrap(); + assert_eq!(resp.tseries.len(), 1); + + let ts = &resp.tseries[0]; + assert_eq!(ts.regularity, "Irregular"); + assert_eq!(ts.data.len(), expected_data_len); + }) + .await +} + +#[tokio::test] +async fn test_stations_endpoint_regular() { + api_test_wrapper(async { + let station_id = 30000; + let param_id = 211; + let resolution = "PT1H"; + let expected_data_len = 12; + + let url = format!( + "http://localhost:3000/stations/{}/params/{}?time_resolution={}", + station_id, param_id, resolution + ); + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let resp: common::StationsResponse = resp.json().await.unwrap(); + assert_eq!(resp.tseries.len(), 1); + + let ts = &resp.tseries[0]; + assert_eq!(ts.regularity, "Regular"); + assert_eq!(ts.data.len(), expected_data_len); + }) + .await +} + +// TODO: use `test_case` here too? +#[tokio::test] +async fn test_latest_endpoint() { + api_test_wrapper(async { + let query = ""; + let url = format!("http://localhost:3000/latest{}", query); + let expected_data_len = 2; + + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let json: common::LatestResponse = resp.json().await.unwrap(); + assert_eq!(json.data.len(), expected_data_len); + }) + .await +} + +#[tokio::test] +async fn test_latest_endpoint_with_query() { + api_test_wrapper(async { + let query = "?latest_max_age=2012-02-14T12:00:00Z"; + let url = format!("http://localhost:3000/latest{}", query); + let expected_data_len = 5; + + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let json: common::LatestResponse = resp.json().await.unwrap(); + assert_eq!(json.data.len(), expected_data_len); + }) + .await +} + +#[tokio::test] +async fn test_timeslice_endpoint() { + api_test_wrapper(async { + let time = Utc.with_ymd_and_hms(2023, 5, 5, 00, 30, 00).unwrap(); + let param_id = 3; + let expected_data_len = 0; + + let url = format!( + "http://localhost:3000/timeslices/{}/params/{}", + time, param_id + ); + + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let json: common::TimesliceResponse = resp.json().await.unwrap(); + assert_eq!(json.tslices.len(), 1); + + let slice = &json.tslices[0]; + assert_eq!(slice.param_id, param_id); + assert_eq!(slice.timestamp, time); + assert_eq!(slice.data.len(), expected_data_len); + }) + .await +} From f36a9ecc43e2787d9d7d6398c1553a5300b09c96 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 14:07:41 +0200 Subject: [PATCH 12/41] Add ingestion integration tests --- ingestion/src/lib.rs | 12 +++++ lard_tests/tests/ingestion.rs | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 lard_tests/tests/ingestion.rs diff --git a/ingestion/src/lib.rs b/ingestion/src/lib.rs index 062045d..49385f2 100644 --- a/ingestion/src/lib.rs +++ b/ingestion/src/lib.rs @@ -86,6 +86,18 @@ pub struct Datum { value: f32, } +// Only used in tests +#[doc(hidden)] +impl Datum { + pub fn new(timeseries_id: i32, timestamp: DateTime, value: f32) -> Self { + Datum { + timeseries_id, + timestamp, + value, + } + } +} + pub type Data = Vec; pub async fn insert_data(data: Data, conn: &mut PooledPgConn<'_>) -> Result<(), Error> { diff --git a/lard_tests/tests/ingestion.rs b/lard_tests/tests/ingestion.rs new file mode 100644 index 0000000..dbc1d28 --- /dev/null +++ b/lard_tests/tests/ingestion.rs @@ -0,0 +1,82 @@ +use chrono::{TimeZone, Utc}; +use test_case::test_case; + +use lard_ingestion::{insert_data, permissions::timeseries_is_open, Data, Datum}; + +pub mod common; + +#[tokio::test] +async fn test_insert_data() { + let pool = common::init_db_pool().await.unwrap(); + let mut conn = pool.get().await.unwrap(); + + // Timeseries ID 2 has hourly data + let id: i32 = 2; + let count_before_insertion = common::number_of_data_rows(&conn, id).await; + + let data: Data = (1..10) + .map(|i| { + let timestamp = Utc.with_ymd_and_hms(2012, 2, 16, i, 0, 0).unwrap(); + let value = 49. + i as f32; + Datum::new(id, timestamp, value) + }) + .collect(); + let data_len = data.len(); + + insert_data(data, &mut conn).await.unwrap(); + + let count_after_insertion = common::number_of_data_rows(&conn, id).await; + let rows_inserted = count_after_insertion - count_before_insertion; + + // NOTE: The assert will fail locally if the database hasn't been cleaned up between runs + assert_eq!(rows_inserted, data_len); +} + +#[test_case(0, 0, 0 => false; "stationid not in permit_tables")] +#[test_case(3, 0, 0 => false; "stationid in StationPermitTable, timeseries closed")] +#[test_case(1, 0, 0 => false; "stationid in ParamPermitTable, timeseries closed")] +#[test_case(4, 0, 1 => true; "stationid in StationPermitTable, timeseries open")] +#[test_case(2, 0, 0 => true; "stationid in ParamPermitTable, timeseries open")] +fn test_timeseries_is_open(station_id: i32, type_id: i32, permit_id: i32) -> bool { + let permit_tables = common::mock_permit_tables(); + timeseries_is_open(permit_tables, station_id, type_id, permit_id).unwrap() +} + +#[tokio::test] +async fn test_kldata_endpoint() { + let ingestor = common::init_ingestion_server().await; + + let test = async { + let station_id = 12000; + let obsinn_msg = format!( + "kldata/nationalnr={}/type=508/messageid=23 +TA,TWD(0,0) +20240607134900,25.0,18.2 +20240607135100,25.1,18.3 +20240607135200,25.0,18.3 +20240607135300,24.9,18.1 +20240607135400,25.2,18.2 +20240607135500,25.1,18.2 +", + station_id + ); + + let client = reqwest::Client::new(); + let resp = client + .post("http://localhost:3001/kldata") + .body(obsinn_msg) + .send() + .await + .unwrap(); + + let json: common::IngestorResponse = resp.json().await.unwrap(); + + assert_eq!(json.res, 0); + assert_eq!(json.message_id, 23) + }; + + tokio::select! { + _ = ingestor => panic!("Server task terminated first"), + _ = test => {} + } +} From fcc78475defaa625ee56dd0258d187a2df8ca836 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 14:08:00 +0200 Subject: [PATCH 13/41] Add end-to-end integration tests --- lard_tests/tests/end-to-end.rs | 115 +++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 lard_tests/tests/end-to-end.rs diff --git a/lard_tests/tests/end-to-end.rs b/lard_tests/tests/end-to-end.rs new file mode 100644 index 0000000..17596b0 --- /dev/null +++ b/lard_tests/tests/end-to-end.rs @@ -0,0 +1,115 @@ +use std::future::Future; + +use test_case::test_case; + +pub mod common; + +async fn e2e_test_wrapper>(test: T) { + let api_server = common::init_api_server().await; + let ingestor = common::init_ingestion_server().await; + + tokio::select! { + _ = api_server => panic!("API server task terminated first"), + _ = ingestor => panic!("Ingestor server task terminated first"), + _ = test => {} + } +} + +#[test_case(20000, 0; "to closed timeseries")] +#[test_case(12000, 6; "to open timeseries")] +#[tokio::test] +async fn append_data(station_id: i32, expected_rows: usize) { + e2e_test_wrapper(async move { + let param_id = 211; + let obsinn_msg = format!( + "kldata/nationalnr={}/type=508/messageid=23 +TA +20230505014000,20.0 +20230505014100,20.1 +20230505014200,20.0 +20230505014300,20.2 +20230505014400,20.2 +20230505014500,20.1 +", + station_id + ); + + let api_url = format!( + "http://localhost:3000/stations/{}/params/{}", + station_id, param_id + ); + + let client = reqwest::Client::new(); + let resp = client.get(&api_url).send().await.unwrap(); + let json: common::StationsResponse = resp.json().await.unwrap(); + let count_before_ingestion = json.tseries[0].data.len(); + + let resp = client + .post("http://localhost:3001/kldata") + .body(obsinn_msg) + .send() + .await + .unwrap(); + + let json: common::IngestorResponse = resp.json().await.unwrap(); + assert_eq!(json.res, 0); + + let resp = client.get(&api_url).send().await.unwrap(); + let json: common::StationsResponse = resp.json().await.unwrap(); + let count_after_ingestion = json.tseries[0].data.len(); + + let rows_added = count_after_ingestion - count_before_ingestion; + + // NOTE: The assert might fail locally if the database hasn't been cleaned up between runs + assert_eq!(rows_added, expected_rows); + }) + .await +} + +#[test_case(40000; "new_station")] +#[test_case(11000; "old_station")] +#[tokio::test] +async fn create_timeseries(station_id: i32) { + e2e_test_wrapper(async move { + // 145, AGM, mean(air_gap PT10M) + // 146, AGX, max(air_gap PT10M) + let param_ids = [145, 146]; + let expected_len = 5; + let obsinn_msg = format!( + // 506 == ten minute data + "kldata/nationalnr={}/type=506/messageid=23 +AGM(1,2),AGX(1,2) +20240606000000,11.0,12.0 +20240606001000,12.0,19.0 +20240606002000,10.0,16.0 +20240606003000,12.0,16.0 +20240606004000,11.0,15.0 +", + station_id + ); + + let client = reqwest::Client::new(); + let resp = client + .post("http://localhost:3001/kldata") + .body(obsinn_msg) + .send() + .await + .unwrap(); + let json: common::IngestorResponse = resp.json().await.unwrap(); + assert_eq!(json.res, 0); + + for id in param_ids { + let url = format!( + "http://localhost:3000/stations/{}/params/{}", + station_id, id + ); + + let resp = client.get(&url).send().await.unwrap(); + assert!(resp.status().is_success()); + + let json: common::StationsResponse = resp.json().await.unwrap(); + assert_eq!(json.tseries[0].data.len(), expected_len) + } + }) + .await +} From 95432470404c9008a29309ee46f3780f9c9872c2 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 17 Jun 2024 14:08:17 +0200 Subject: [PATCH 14/41] Add github workflow --- .github/workflows/ci.yml | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a3ab6f4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,85 @@ +name: Continuous integration + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - closed + +jobs: + build-and-test: + name: Build and test + runs-on: ubuntu-latest + + services: + postgres: + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + + steps: + - uses: actions/checkout@v4 + + - name: Cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Run cargo check + run: cargo check + + - name: Formatting check + run: cargo fmt --all -- --check + + - name: Build + run: cargo build --workspace --tests + + - name: Lint + run: cargo clippy --workspace -- -D warnings + + - name: Prepare PostgreSQL + run: target/debug/prepare_db + + - name: Run unit and integration tests + run: cargo test --no-fail-fast -- --nocapture --test-threads=1 + + # TODO: potentially we can split them up in multiple steps + # - name: Prepare PostgreSQL for API tests + # run: target/debug/prepare_db api + + # - name: API integration tests + # run: cargo test --test api -- --nocapture --test-threads=1 + + # - name: Prepare PostgreSQL for ingestion tests + # run: target/debug/prepare_db ingestion + + # - name: Ingestions integration tests + # run: cargo test --test ingestion -- --nocapture --test-threads=1 + + # - name: Prepare PostgreSQL for E2E tests + # run: target/debug/prepare_db end-to-end + + # - name: E2E integration tests + # run: cargo test --test end-to-end -- --nocapture --test-threads=1 From 2ed389370380cf675e4f8c132b492aee84f6a138 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 18 Jun 2024 10:23:47 +0200 Subject: [PATCH 15/41] Make timeseries creation easier to understand --- lard_tests/src/bin/prepare_db.rs | 111 ++++++++++++++----------------- 1 file changed, 50 insertions(+), 61 deletions(-) diff --git a/lard_tests/src/bin/prepare_db.rs b/lard_tests/src/bin/prepare_db.rs index d41303f..74377d5 100644 --- a/lard_tests/src/bin/prepare_db.rs +++ b/lard_tests/src/bin/prepare_db.rs @@ -31,29 +31,15 @@ struct Location { struct Timeseries { from: DateTime, - to: DateTime, period: Duration, - // len: i32, + len: i32, deactivated: bool, loc: Location, } impl Timeseries { - fn new( - from: DateTime, - period: Duration, - len: i32, - loc: Location, - deactivated: bool, - ) -> Self { - Timeseries { - from, - to: from + period * len, - period, - // len, - loc, - deactivated, - } + fn end_time(&self) -> DateTime { + self.from + self.period * self.len } } @@ -64,13 +50,15 @@ struct Case<'a> { } async fn create_single_ts(client: &Client, ts: Timeseries) -> Result { + let end_time = ts.end_time(); + // Insert timeseries let id = match ts.deactivated { true => client .query_one( "INSERT INTO public.timeseries (fromtime, totime, loc.lat, loc.lon, deactivated) VALUES ($1, $2, $3, $4, true) RETURNING id", - &[&ts.from, &ts.to, &ts.loc.lat, &ts.loc.lon], + &[&ts.from, &end_time, &ts.loc.lat, &ts.loc.lon], ) .await? .get(0), @@ -87,7 +75,7 @@ async fn create_single_ts(client: &Client, ts: Timeseries) -> Result // insert data let mut value: f32 = 0.0; let mut time = ts.from; - while time <= ts.to { + while time <= end_time { client .execute( "INSERT INTO public.data (timeseries, obstime, obsvalue) @@ -140,16 +128,16 @@ async fn create_timeseries(client: &Client) -> Result<(), Error> { let cases = vec![ Case { title: "Daily, active", - ts: Timeseries::new( - Utc.with_ymd_and_hms(1970, 6, 5, 0, 0, 0).unwrap(), - Duration::days(1), - 19, - Location { + ts: Timeseries { + from: Utc.with_ymd_and_hms(1970, 6, 5, 0, 0, 0).unwrap(), + period: Duration::days(1), + len: 19, + loc: Location { lat: 59.9, lon: 10.4, }, - false, - ), + deactivated: false, + }, meta: Labels { station_id: 10000, param: Param { @@ -163,16 +151,16 @@ async fn create_timeseries(client: &Client) -> Result<(), Error> { }, Case { title: "Hourly, active", - ts: Timeseries::new( - Utc.with_ymd_and_hms(2012, 2, 14, 0, 0, 0).unwrap(), - Duration::hours(1), - 47, - Location { + ts: Timeseries { + from: Utc.with_ymd_and_hms(2012, 2, 14, 0, 0, 0).unwrap(), + period: Duration::hours(1), + len: 47, + loc: Location { lat: 46.0, lon: -73.0, }, - false, - ), + deactivated: false, + }, meta: Labels { station_id: 11000, param: Param { @@ -186,16 +174,16 @@ async fn create_timeseries(client: &Client) -> Result<(), Error> { }, Case { title: "Minutely, active 1", - ts: Timeseries::new( - Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), - Duration::minutes(1), - 99, - Location { + ts: Timeseries { + from: Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), + period: Duration::minutes(1), + len: 99, + loc: Location { lat: 65.89, lon: 13.61, }, - false, - ), + deactivated: false, + }, meta: Labels { station_id: 12000, param: Param { @@ -209,16 +197,16 @@ async fn create_timeseries(client: &Client) -> Result<(), Error> { }, Case { title: "Minutely, active 2", - ts: Timeseries::new( - Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), - Duration::minutes(1), - 99, - Location { + ts: Timeseries { + from: Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), + period: Duration::minutes(1), + len: 99, + loc: Location { lat: 66.0, lon: 14.0, }, - false, - ), + deactivated: false, + }, meta: Labels { station_id: 12100, param: Param { @@ -233,13 +221,14 @@ async fn create_timeseries(client: &Client) -> Result<(), Error> { Case { // use it to test latest endpoint without optional query title: "3hrs old minute data", - ts: Timeseries::new( - Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() - Duration::minutes(179), - Duration::minutes(1), - 179, - Location { lat: 1.0, lon: 1.0 }, - false, - ), + ts: Timeseries { + from: Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() + - Duration::minutes(179), + period: Duration::minutes(1), + len: 179, + loc: Location { lat: 1.0, lon: 1.0 }, + deactivated: false, + }, meta: Labels { station_id: 20000, param: Param { @@ -254,14 +243,14 @@ async fn create_timeseries(client: &Client) -> Result<(), Error> { Case { // use it to test stations endpoint with optional time resolution (PT1H) title: "Air temperature over the last 12 hours", - ts: Timeseries::new( + ts: Timeseries { // TODO: check that this adds the correct number of data points every time - Utc::now().duration_trunc(TimeDelta::hours(1)).unwrap() - Duration::hours(11), - Duration::hours(1), - 11, - Location { lat: 2.0, lon: 2.0 }, - false, - ), + from: Utc::now().duration_trunc(TimeDelta::hours(1)).unwrap() - Duration::hours(11), + period: Duration::hours(1), + len: 11, + loc: Location { lat: 2.0, lon: 2.0 }, + deactivated: false, + }, meta: Labels { station_id: 30000, param: Param { From 4c8e05b1e1fb6186873133798ce4a12212acf1bf Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 19 Jun 2024 10:56:30 +0200 Subject: [PATCH 16/41] Update TODOs --- api/src/timeseries.rs | 1 - ingestion/src/main.rs | 1 - lard_tests/tests/api.rs | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/timeseries.rs b/api/src/timeseries.rs index 5be680e..cf39caa 100644 --- a/api/src/timeseries.rs +++ b/api/src/timeseries.rs @@ -127,7 +127,6 @@ pub async fn get_timeseries_data_regular( _ => "1 minute", // FIXME: this should error instead of falling back to a default }; - // TODO: this generates nulls till utc.now if end_time is not specified in the database? let query_string = format!("SELECT data.obsvalue, ts_rule.timestamp \ FROM (SELECT data.obsvalue, data.obstime FROM data WHERE data.timeseries = $1) as data RIGHT JOIN generate_series($2::timestamptz, $3::timestamptz, interval '{}') AS ts_rule(timestamp) \ diff --git a/ingestion/src/main.rs b/ingestion/src/main.rs index 6e98ecf..664e3a8 100644 --- a/ingestion/src/main.rs +++ b/ingestion/src/main.rs @@ -19,7 +19,6 @@ async fn main() -> Result<(), Box> { }; // Permit tables handling (needs connection to stinfosys database) - // TODO: we probably need to test this as well, ie move into separate function let permit_tables = Arc::new(RwLock::new(fetch_permits().await?)); let background_permit_tables = permit_tables.clone(); diff --git a/lard_tests/tests/api.rs b/lard_tests/tests/api.rs index 4a2543b..b0f3262 100644 --- a/lard_tests/tests/api.rs +++ b/lard_tests/tests/api.rs @@ -85,6 +85,7 @@ async fn test_latest_endpoint_with_query() { api_test_wrapper(async { let query = "?latest_max_age=2012-02-14T12:00:00Z"; let url = format!("http://localhost:3000/latest{}", query); + // TODO: Right now this test works only if it is run before ingestion/e2e integration tests let expected_data_len = 5; let resp = reqwest::get(url).await.unwrap(); From cdcd54389cdfe93f27f7a36e60780e6cf719a99a Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 19 Jun 2024 10:56:47 +0200 Subject: [PATCH 17/41] Add makefile to run tests locally --- lard_tests/Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lard_tests/Makefile diff --git a/lard_tests/Makefile b/lard_tests/Makefile new file mode 100644 index 0000000..fff0f0d --- /dev/null +++ b/lard_tests/Makefile @@ -0,0 +1,14 @@ +run_tests: + @echo "Starting Postgres docker container..." + docker run --name sql-data-test -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres + @sleep 5 + cargo build --workspace --tests + cd ..; target/debug/prepare_db + cargo test --no-fail-fast -- --nocapture --test-threads=1 + make clean + +clean: + @echo "Stopping Postgres container..." + docker stop sql-data-test + @echo "Removing Postgres container..." + docker rm sql-data-test From 68d4386b59f0e7ffe8cbbe72b04ef8b25b22d7d7 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 21 Jun 2024 16:10:49 +0200 Subject: [PATCH 18/41] prepare_db now only creates the schemas --- lard_tests/src/bin/prepare_db.rs | 316 +------------------------------ 1 file changed, 10 insertions(+), 306 deletions(-) diff --git a/lard_tests/src/bin/prepare_db.rs b/lard_tests/src/bin/prepare_db.rs index 74377d5..e658807 100644 --- a/lard_tests/src/bin/prepare_db.rs +++ b/lard_tests/src/bin/prepare_db.rs @@ -1,288 +1,20 @@ // use std::env; use std::fs; -use chrono::{DateTime, Duration, DurationRound, TimeDelta, TimeZone, Utc}; -use tokio_postgres::{Client, Error, NoTls}; +use tokio_postgres::{Error, NoTls}; const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; -struct Param<'a> { - id: i32, - code: &'a str, -} - -// TODO: maybe merge into fake_data_generator, a lot of the code is shared -struct Labels<'a> { - // Assigned automatically - // timeseries: i32, - station_id: i32, - param: Param<'a>, - type_id: i32, - level: Option, - sensor: Option, -} - -struct Location { - lat: f32, - lon: f32, - // hamsl: f32, - // hag: f32, -} - -struct Timeseries { - from: DateTime, - period: Duration, - len: i32, - deactivated: bool, - loc: Location, -} - -impl Timeseries { - fn end_time(&self) -> DateTime { - self.from + self.period * self.len - } -} - -struct Case<'a> { - title: &'a str, - ts: Timeseries, - meta: Labels<'a>, -} - -async fn create_single_ts(client: &Client, ts: Timeseries) -> Result { - let end_time = ts.end_time(); - - // Insert timeseries - let id = match ts.deactivated { - true => client - .query_one( - "INSERT INTO public.timeseries (fromtime, totime, loc.lat, loc.lon, deactivated) - VALUES ($1, $2, $3, $4, true) RETURNING id", - &[&ts.from, &end_time, &ts.loc.lat, &ts.loc.lon], - ) - .await? - .get(0), - false => client - .query_one( - "INSERT INTO public.timeseries (fromtime, loc.lat, loc.lon, deactivated) - VALUES ($1, $2, $3, false) RETURNING id", - &[&ts.from, &ts.loc.lat, &ts.loc.lon], - ) - .await? - .get(0), - }; - - // insert data - let mut value: f32 = 0.0; - let mut time = ts.from; - while time <= end_time { - client - .execute( - "INSERT INTO public.data (timeseries, obstime, obsvalue) - VALUES ($1, $2, $3)", - &[&id, &time, &value], - ) - .await?; - time += ts.period; - value += 1.0; - } - - Ok(id) -} - -async fn insert_ts_metadata<'a>(client: &Client, id: i32, meta: Labels<'a>) -> Result<(), Error> { - client - .execute( - "INSERT INTO labels.met (timeseries, station_id, param_id, type_id, lvl, sensor) - VALUES($1, $2, $3, $4, $5, $6)", - &[ - &id, - &meta.station_id, - &meta.param.id, - &meta.type_id, - &meta.level, - &meta.sensor, - ], - ) - .await?; - - client - .execute( - "INSERT INTO labels.obsinn (timeseries, nationalnummer, type_id, param_code, lvl, sensor) - VALUES($1, $2, $3, $4, $5, $6)", - &[ - &id, - &meta.station_id, - &meta.type_id, - &meta.param.code, - &meta.level, - &meta.sensor, - ], - ) - .await?; - - Ok(()) -} - -async fn create_timeseries(client: &Client) -> Result<(), Error> { - let cases = vec![ - Case { - title: "Daily, active", - ts: Timeseries { - from: Utc.with_ymd_and_hms(1970, 6, 5, 0, 0, 0).unwrap(), - period: Duration::days(1), - len: 19, - loc: Location { - lat: 59.9, - lon: 10.4, - }, - deactivated: false, - }, - meta: Labels { - station_id: 10000, - param: Param { - id: 103, - code: "EV_24", // sum(water_evaporation_amount) - }, - type_id: 1, // Is there a type_id for daily data? - level: Some(0), - sensor: Some(0), - }, - }, - Case { - title: "Hourly, active", - ts: Timeseries { - from: Utc.with_ymd_and_hms(2012, 2, 14, 0, 0, 0).unwrap(), - period: Duration::hours(1), - len: 47, - loc: Location { - lat: 46.0, - lon: -73.0, - }, - deactivated: false, - }, - meta: Labels { - station_id: 11000, - param: Param { - id: 222, - code: "TGM", // mean(grass_temperature) - }, - type_id: 501, // hourly data - level: Some(0), - sensor: Some(0), - }, - }, - Case { - title: "Minutely, active 1", - ts: Timeseries { - from: Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), - period: Duration::minutes(1), - len: 99, - loc: Location { - lat: 65.89, - lon: 13.61, - }, - deactivated: false, - }, - meta: Labels { - station_id: 12000, - param: Param { - id: 211, - code: "TA", // air_temperature - }, - type_id: 508, // minute data - level: None, - sensor: None, - }, - }, - Case { - title: "Minutely, active 2", - ts: Timeseries { - from: Utc.with_ymd_and_hms(2023, 5, 5, 0, 0, 0).unwrap(), - period: Duration::minutes(1), - len: 99, - loc: Location { - lat: 66.0, - lon: 14.0, - }, - deactivated: false, - }, - meta: Labels { - station_id: 12100, - param: Param { - id: 255, - code: "TWD", // sea_water_temperature - }, - type_id: 508, // minute data - level: Some(0), - sensor: Some(0), - }, - }, - Case { - // use it to test latest endpoint without optional query - title: "3hrs old minute data", - ts: Timeseries { - from: Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() - - Duration::minutes(179), - period: Duration::minutes(1), - len: 179, - loc: Location { lat: 1.0, lon: 1.0 }, - deactivated: false, - }, - meta: Labels { - station_id: 20000, - param: Param { - id: 211, - code: "TA", // air_temperature - }, - type_id: 508, // minute data - level: Some(0), - sensor: Some(0), - }, - }, - Case { - // use it to test stations endpoint with optional time resolution (PT1H) - title: "Air temperature over the last 12 hours", - ts: Timeseries { - // TODO: check that this adds the correct number of data points every time - from: Utc::now().duration_trunc(TimeDelta::hours(1)).unwrap() - Duration::hours(11), - period: Duration::hours(1), - len: 11, - loc: Location { lat: 2.0, lon: 2.0 }, - deactivated: false, - }, - meta: Labels { - station_id: 30000, - param: Param { - id: 211, - code: "TA", // air_temperature - }, - type_id: 501, // hourly data - level: Some(0), - sensor: Some(0), - }, - }, - ]; - - for case in cases { - println!("Inserting timeseries: {}", case.title); - let id = create_single_ts(client, case.ts).await?; - insert_ts_metadata(client, id, case.meta).await?; - } - - Ok(()) -} - -async fn cleanup(client: &Client) -> Result<(), Error> { - client - .batch_execute("DROP TABLE IF EXISTS timeseries, data, labels.met, labels.obsinn CASCADE") - .await?; - client.batch_execute("DROP TYPE IF EXISTS location").await?; +#[tokio::main] +async fn main() -> Result<(), Error> { + let (client, connection) = tokio_postgres::connect(CONNECT_STRING, NoTls).await?; - Ok(()) -} + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("connection error: {}", e); + } + }); -async fn create_schema(client: &Client) -> Result<(), Error> { let public_schema = fs::read_to_string("db/public.sql").expect("Should be able to read SQL file"); client.batch_execute(public_schema.as_str()).await?; @@ -291,38 +23,10 @@ async fn create_schema(client: &Client) -> Result<(), Error> { fs::read_to_string("db/labels.sql").expect("Should be able to read SQL file"); client.batch_execute(labels_schema.as_str()).await?; - Ok(()) -} - -async fn create_partitions(client: &Client) -> Result<(), Error> { // TODO: add multiple partitions? let partition_string = format!( "CREATE TABLE data_y{}_to_y{} PARTITION OF public.data FOR VALUES FROM ('{}') TO ('{}')", "1950", "2100", "1950-01-01 00:00:00+00", "2100-01-01 00:00:00+00", ); - client.batch_execute(partition_string.as_str()).await?; - - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<(), Error> { - // let _test_type: String = env::args() - // .next() - // .expect("Provide test type for database setup ('api', 'ingestion', 'e2e')"); - - let (client, connection) = tokio_postgres::connect(CONNECT_STRING, NoTls).await?; - - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); - - cleanup(&client).await?; - create_schema(&client).await?; - create_partitions(&client).await?; - create_timeseries(&client).await?; - - Ok(()) + client.batch_execute(partition_string.as_str()).await } From d3473b43ee5207e27ffa085bebb06473cf042637 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 21 Jun 2024 16:11:33 +0200 Subject: [PATCH 19/41] Condense everything in single file --- lard_tests/tests/end-to-end.rs | 354 +++++++++++++++++++++++++++------ 1 file changed, 295 insertions(+), 59 deletions(-) diff --git a/lard_tests/tests/end-to-end.rs b/lard_tests/tests/end-to-end.rs index 17596b0..b89bdee 100644 --- a/lard_tests/tests/end-to-end.rs +++ b/lard_tests/tests/end-to-end.rs @@ -1,74 +1,310 @@ use std::future::Future; +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; -use test_case::test_case; +use chrono::{DateTime, Duration, DurationRound, TimeDelta, TimeZone, Utc}; +use serde::Deserialize; +use tokio_postgres::NoTls; -pub mod common; +use lard_ingestion::permissions::{ParamPermit, ParamPermitTable, StationPermitTable}; + +const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; +const PARAMCONV_CSV: &str = "../ingestion/resources/paramconversions.csv"; + +#[derive(Debug, Deserialize)] +pub struct IngestorResponse { + pub message: String, + pub message_id: usize, + pub res: u8, + pub retry: bool, +} + +#[derive(Debug, Deserialize)] +pub struct StationsResponse { + pub tseries: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Tseries { + pub regularity: String, + pub data: Vec, + // header: ... +} + +#[derive(Debug, Deserialize)] +pub struct LatestResponse { + pub data: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct LatestData { + // TODO: Missing param_id here? + pub value: f64, + pub timestamp: DateTime, + pub station_id: i32, + // loc: {lat, lon, hamsl, hag} +} + +#[derive(Debug, Deserialize)] +pub struct TimesliceResponse { + pub tslices: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Tslice { + pub timestamp: DateTime, + pub param_id: i32, + pub data: Vec, +} + +struct Param<'a> { + id: i32, + code: &'a str, + sensor_level: Option<(i32, i32)>, +} + +impl<'a> Param<'a> { + fn new(id: i32, code: &'a str) -> Param { + Param { + id, + code, + sensor_level: None, + } + } +} + +struct TestData<'a> { + station_id: i32, + type_id: i32, + params: Vec>, + start_time: DateTime, + period: Duration, + len: usize, +} + +impl<'a> TestData<'a> { + // Creates a message with the following format: + // ``` + // kldata/nationalnr=99999/type=501/messageid=23 + // param_1,param_2(0,0),... + // 20240101000000,0.0,0.0,... + // 20240101010000,0.0,0.0,... + // ... + // ``` + fn obsinn_message(&self) -> String { + // TODO: assign different values? + let val = 0.0; + let values = vec![val.to_string(); self.params.len()].join(","); + + let mut msg = vec![self.obsinn_header()]; + + let mut time = self.start_time; + while time < self.end_time() { + msg.push(format!("{},{}", time.format("%Y%m%d%H%M%S"), values)); + time += self.period; + } + + msg.join("\n") + } + + fn obsinn_header(&self) -> String { + format!( + "kldata/nationalnr={}/type={}/messageid=23\n{}", + self.station_id, + self.type_id, + self.param_header() + ) + } + + fn param_header(&self) -> String { + self.params + .iter() + .map(|param| match param.sensor_level { + Some((sensor, level)) => format!("{}({},{})", param.code, sensor, level), + None => param.code.to_string(), + }) + .collect::>() + .join(",") + } + + fn end_time(&self) -> DateTime { + self.start_time + self.period * self.len as i32 + } +} + +pub fn mock_permit_tables() -> Arc> { + let param_permit = HashMap::from([ + // station_id -> (type_id, param_id, permit_id) + (1, vec![ParamPermit::new(0, 0, 0)]), + (2, vec![ParamPermit::new(0, 0, 1)]), // open + ]); + + #[rustfmt::skip] + let station_permit = HashMap::from([ + // station_id -> permit_id ... 1 = open, everything else is closed + (20000, 1), (30000, 1), (11000, 1), (12000, 1), + (12100, 1), (13000, 1), (40000, 1), (50000, 1), + ]); + + Arc::new(RwLock::new((param_permit, station_permit))) +} + +pub async fn cleanup() { + let (client, conn) = tokio_postgres::connect(CONNECT_STRING, NoTls) + .await + .unwrap(); + + tokio::spawn(async move { + if let Err(e) = conn.await { + eprintln!("{}", e); + } + }); + + client + .execute( + // TODO: public.timeseries_id_seq? RESTART IDENTITY CASCADE? + "TRUNCATE public.timeseries, labels.met, labels.obsinn CASCADE", + &[], + ) + .await + .unwrap(); +} async fn e2e_test_wrapper>(test: T) { - let api_server = common::init_api_server().await; - let ingestor = common::init_ingestion_server().await; + let api_server = tokio::spawn(lard_api::run(CONNECT_STRING)); + let ingestor = tokio::spawn(lard_ingestion::run( + CONNECT_STRING, + PARAMCONV_CSV, + mock_permit_tables(), + )); tokio::select! { - _ = api_server => panic!("API server task terminated first"), - _ = ingestor => panic!("Ingestor server task terminated first"), - _ = test => {} + _ = api_server => {panic!("API server task terminated first")}, + _ = ingestor => {panic!("Ingestor server task terminated first")}, + _ = test => {cleanup().await} } } -#[test_case(20000, 0; "to closed timeseries")] -#[test_case(12000, 6; "to open timeseries")] -#[tokio::test] -async fn append_data(station_id: i32, expected_rows: usize) { - e2e_test_wrapper(async move { - let param_id = 211; - let obsinn_msg = format!( - "kldata/nationalnr={}/type=508/messageid=23 -TA -20230505014000,20.0 -20230505014100,20.1 -20230505014200,20.0 -20230505014300,20.2 -20230505014400,20.2 -20230505014500,20.1 -", - station_id - ); +async fn ingest_data(client: &reqwest::Client, obsinn_msg: String) -> IngestorResponse { + let resp = client + .post("http://localhost:3001/kldata") + .body(obsinn_msg) + .send() + .await + .unwrap(); - let api_url = format!( - "http://localhost:3000/stations/{}/params/{}", - station_id, param_id - ); + resp.json().await.unwrap() +} + +#[tokio::test] +async fn test_stations_endpoint_irregular() { + e2e_test_wrapper(async { + let ts = TestData { + station_id: 11000, + params: vec![Param::new(222, "TGM")], // mean(grass_temperature) + start_time: Utc.with_ymd_and_hms(2012, 2, 14, 0, 0, 0).unwrap(), + period: Duration::hours(1), + type_id: 501, + len: 48, + }; let client = reqwest::Client::new(); - let resp = client.get(&api_url).send().await.unwrap(); - let json: common::StationsResponse = resp.json().await.unwrap(); - let count_before_ingestion = json.tseries[0].data.len(); + let ingestor_resp = ingest_data(&client, ts.obsinn_message()).await; + assert_eq!(ingestor_resp.res, 0); - let resp = client - .post("http://localhost:3001/kldata") - .body(obsinn_msg) - .send() - .await - .unwrap(); + for param in ts.params { + let url = format!( + "http://localhost:3000/stations/{}/params/{}", + ts.station_id, param.id + ); + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); - let json: common::IngestorResponse = resp.json().await.unwrap(); - assert_eq!(json.res, 0); + let resp: StationsResponse = resp.json().await.unwrap(); + assert_eq!(resp.tseries.len(), 1); - let resp = client.get(&api_url).send().await.unwrap(); - let json: common::StationsResponse = resp.json().await.unwrap(); - let count_after_ingestion = json.tseries[0].data.len(); + let series = &resp.tseries[0]; + assert_eq!(series.regularity, "Irregular"); + assert_eq!(series.data.len(), ts.len); + println!("{:?}", series.data) + } + }) + .await +} + +#[tokio::test] +async fn test_stations_endpoint_regular() { + e2e_test_wrapper(async { + let ts = TestData { + station_id: 30000, + params: vec![Param::new(211, "TA")], + start_time: Utc::now().duration_trunc(TimeDelta::hours(1)).unwrap() + - Duration::hours(11), + period: Duration::hours(1), + type_id: 501, + len: 12, + }; + + let client = reqwest::Client::new(); + let ingestor_resp = ingest_data(&client, ts.obsinn_message()).await; + assert_eq!(ingestor_resp.res, 0); + + let resolution = "PT1H"; + for param in ts.params { + let url = format!( + "http://localhost:3000/stations/{}/params/{}?time_resolution={}", + ts.station_id, param.id, resolution + ); + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); - let rows_added = count_after_ingestion - count_before_ingestion; + let resp: StationsResponse = resp.json().await.unwrap(); + assert_eq!(resp.tseries.len(), 1); - // NOTE: The assert might fail locally if the database hasn't been cleaned up between runs - assert_eq!(rows_added, expected_rows); + let series = &resp.tseries[0]; + println!("{:?}", series); + assert_eq!(series.regularity, "Regular"); + assert_eq!(series.data.len(), ts.len); + } }) .await } -#[test_case(40000; "new_station")] -#[test_case(11000; "old_station")] #[tokio::test] +async fn test_latest_endpoint() { + e2e_test_wrapper(async { + let ts = TestData { + station_id: 20000, + params: vec![Param::new(211, "TA"), Param::new(255, "TGX")], + start_time: Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() + - Duration::minutes(179), + period: Duration::minutes(1), + type_id: 508, + len: 180, + }; + + let client = reqwest::Client::new(); + let ingestor_resp = ingest_data(&client, ts.obsinn_message()).await; + assert_eq!(ingestor_resp.res, 0); + + let url = "http://localhost:3000/latest"; + // let expected_data_len = 180; + + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let json: LatestResponse = resp.json().await.unwrap(); + println!("{:?}", json); + // assert_eq!(json.data.len(), expected_data_len); + }) + .await +} + +// #[test_case(40000; "new_station")] +// #[test_case(11000; "old_station")] +// #[tokio::test] async fn create_timeseries(station_id: i32) { e2e_test_wrapper(async move { // 145, AGM, mean(air_gap PT10M) @@ -76,15 +312,15 @@ async fn create_timeseries(station_id: i32) { let param_ids = [145, 146]; let expected_len = 5; let obsinn_msg = format!( - // 506 == ten minute data - "kldata/nationalnr={}/type=506/messageid=23 -AGM(1,2),AGX(1,2) -20240606000000,11.0,12.0 -20240606001000,12.0,19.0 -20240606002000,10.0,16.0 -20240606003000,12.0,16.0 -20240606004000,11.0,15.0 -", + concat!( + "kldata/nationalnr={}/type=506/messageid=23\n", + "AGM(1,2),AGX(1,2)\n", + "20240606000000,11.0,12.0\n", + "20240606001000,12.0,19.0\n", + "20240606002000,10.0,16.0\n", + "20240606003000,12.0,16.0\n", + "20240606004000,11.0,15.0", + ), station_id ); @@ -95,7 +331,7 @@ AGM(1,2),AGX(1,2) .send() .await .unwrap(); - let json: common::IngestorResponse = resp.json().await.unwrap(); + let json: IngestorResponse = resp.json().await.unwrap(); assert_eq!(json.res, 0); for id in param_ids { @@ -107,7 +343,7 @@ AGM(1,2),AGX(1,2) let resp = client.get(&url).send().await.unwrap(); assert!(resp.status().is_success()); - let json: common::StationsResponse = resp.json().await.unwrap(); + let json: StationsResponse = resp.json().await.unwrap(); assert_eq!(json.tseries[0].data.len(), expected_len) } }) From 487d7477476e33ba322ebab683cd00316ebc7bd6 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 21 Jun 2024 16:13:39 +0200 Subject: [PATCH 20/41] Add stations to mock --- lard_tests/tests/common/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lard_tests/tests/common/mod.rs b/lard_tests/tests/common/mod.rs index f62d664..dd88628 100644 --- a/lard_tests/tests/common/mod.rs +++ b/lard_tests/tests/common/mod.rs @@ -86,7 +86,8 @@ pub fn mock_permit_tables() -> Arc Date: Mon, 24 Jun 2024 16:42:20 +0200 Subject: [PATCH 21/41] Mark location as Option --- api/src/timeslice.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/timeslice.rs b/api/src/timeslice.rs index 3cbeaf0..e2df484 100644 --- a/api/src/timeslice.rs +++ b/api/src/timeslice.rs @@ -6,7 +6,9 @@ use serde::Serialize; pub struct TimesliceElem { value: f32, station_id: i32, - loc: Location, + // TODO: this shouldn't be an Option, but it avoids panics if location is somehow + // not found in the database + loc: Option, } // TODO: consider whether this should be object-of-arrays style From fdf65ce9b39f2b6b2040bf5961dc244ebf5d4c32 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 24 Jun 2024 16:42:33 +0200 Subject: [PATCH 22/41] Add helpful log for tests --- ingestion/src/kldata.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index 9047bcd..79d5272 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -204,13 +204,18 @@ pub async fn filter_and_label_kldata( )) })?; + // TODO: we only need to check inside this loop if station_id is in the + // param_permit_table if !timeseries_is_open( permit_table.clone(), chunk.station_id, chunk.type_id, param_id.to_owned(), )? { - // TODO: log that the timeseries is closed? + // TODO: log that the timeseries is closed? Mostly useful for tests + #[cfg(test)] + eprintln!("station {}: timeseries is closed", chunk.station_id); + continue; } From 2269dc366a4ccf77c304d7d23d37031b3c919390 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 24 Jun 2024 16:45:59 +0200 Subject: [PATCH 23/41] Final merge --- Cargo.lock | 1 + Cargo.toml | 1 - lard_tests/Cargo.toml | 5 +- lard_tests/src/bin/prepare_db.rs | 1 - lard_tests/tests/api.rs | 124 ---------------- lard_tests/tests/common/mod.rs | 121 --------------- lard_tests/tests/end-to-end.rs | 244 ++++++++++++++++++++----------- lard_tests/tests/ingestion.rs | 82 ----------- 8 files changed, 163 insertions(+), 416 deletions(-) delete mode 100644 lard_tests/tests/api.rs delete mode 100644 lard_tests/tests/common/mod.rs delete mode 100644 lard_tests/tests/ingestion.rs diff --git a/Cargo.lock b/Cargo.lock index 59bf96f..b63cbed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -884,6 +884,7 @@ dependencies = [ "bb8", "bb8-postgres", "chrono", + "futures", "lard_api", "lard_ingestion", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index f309fbd..5095624 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,5 +26,4 @@ serde = { version = "1.0.188", features = ["derive"] } thiserror = "1.0.56" tokio = { version = "1.33.0", features = ["rt-multi-thread", "macros"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4"] } -reqwest = {version = "0.12.4", features = ["json"]} test-case = "3.3.1" diff --git a/lard_tests/Cargo.toml b/lard_tests/Cargo.toml index 0c21132..4fefbd1 100644 --- a/lard_tests/Cargo.toml +++ b/lard_tests/Cargo.toml @@ -12,10 +12,9 @@ tokio-postgres.workspace = true bb8.workspace = true bb8-postgres.workspace = true serde.workspace = true - -[dev-dependencies] -reqwest.workspace = true test-case.workspace = true +futures = "0.3.30" +reqwest = {version = "0.12.4", features = ["json"]} [[bin]] name = "prepare_db" diff --git a/lard_tests/src/bin/prepare_db.rs b/lard_tests/src/bin/prepare_db.rs index e658807..2e9a307 100644 --- a/lard_tests/src/bin/prepare_db.rs +++ b/lard_tests/src/bin/prepare_db.rs @@ -1,4 +1,3 @@ -// use std::env; use std::fs; use tokio_postgres::{Error, NoTls}; diff --git a/lard_tests/tests/api.rs b/lard_tests/tests/api.rs deleted file mode 100644 index b0f3262..0000000 --- a/lard_tests/tests/api.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::future::Future; - -use chrono::{TimeZone, Utc}; - -pub mod common; - -async fn api_test_wrapper>(test: T) { - let server = common::init_api_server().await; - - tokio::select! { - _ = server => panic!("Server task terminated first"), - _ = test => {} - } -} - -// TODO: test getting all tseries without specifying param_id -#[tokio::test] -async fn test_stations_endpoint_irregular() { - api_test_wrapper(async { - let station_id = 10000; - let param_id = 103; - let expected_data_len = 20; - - let url = format!( - "http://localhost:3000/stations/{}/params/{}", - station_id, param_id - ); - let resp = reqwest::get(url).await.unwrap(); - assert!(resp.status().is_success()); - - let resp: common::StationsResponse = resp.json().await.unwrap(); - assert_eq!(resp.tseries.len(), 1); - - let ts = &resp.tseries[0]; - assert_eq!(ts.regularity, "Irregular"); - assert_eq!(ts.data.len(), expected_data_len); - }) - .await -} - -#[tokio::test] -async fn test_stations_endpoint_regular() { - api_test_wrapper(async { - let station_id = 30000; - let param_id = 211; - let resolution = "PT1H"; - let expected_data_len = 12; - - let url = format!( - "http://localhost:3000/stations/{}/params/{}?time_resolution={}", - station_id, param_id, resolution - ); - let resp = reqwest::get(url).await.unwrap(); - assert!(resp.status().is_success()); - - let resp: common::StationsResponse = resp.json().await.unwrap(); - assert_eq!(resp.tseries.len(), 1); - - let ts = &resp.tseries[0]; - assert_eq!(ts.regularity, "Regular"); - assert_eq!(ts.data.len(), expected_data_len); - }) - .await -} - -// TODO: use `test_case` here too? -#[tokio::test] -async fn test_latest_endpoint() { - api_test_wrapper(async { - let query = ""; - let url = format!("http://localhost:3000/latest{}", query); - let expected_data_len = 2; - - let resp = reqwest::get(url).await.unwrap(); - assert!(resp.status().is_success()); - - let json: common::LatestResponse = resp.json().await.unwrap(); - assert_eq!(json.data.len(), expected_data_len); - }) - .await -} - -#[tokio::test] -async fn test_latest_endpoint_with_query() { - api_test_wrapper(async { - let query = "?latest_max_age=2012-02-14T12:00:00Z"; - let url = format!("http://localhost:3000/latest{}", query); - // TODO: Right now this test works only if it is run before ingestion/e2e integration tests - let expected_data_len = 5; - - let resp = reqwest::get(url).await.unwrap(); - assert!(resp.status().is_success()); - - let json: common::LatestResponse = resp.json().await.unwrap(); - assert_eq!(json.data.len(), expected_data_len); - }) - .await -} - -#[tokio::test] -async fn test_timeslice_endpoint() { - api_test_wrapper(async { - let time = Utc.with_ymd_and_hms(2023, 5, 5, 00, 30, 00).unwrap(); - let param_id = 3; - let expected_data_len = 0; - - let url = format!( - "http://localhost:3000/timeslices/{}/params/{}", - time, param_id - ); - - let resp = reqwest::get(url).await.unwrap(); - assert!(resp.status().is_success()); - - let json: common::TimesliceResponse = resp.json().await.unwrap(); - assert_eq!(json.tslices.len(), 1); - - let slice = &json.tslices[0]; - assert_eq!(slice.param_id, param_id); - assert_eq!(slice.timestamp, time); - assert_eq!(slice.data.len(), expected_data_len); - }) - .await -} diff --git a/lard_tests/tests/common/mod.rs b/lard_tests/tests/common/mod.rs deleted file mode 100644 index dd88628..0000000 --- a/lard_tests/tests/common/mod.rs +++ /dev/null @@ -1,121 +0,0 @@ -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use bb8::Pool; -use bb8_postgres::PostgresConnectionManager; -use chrono::{DateTime, Utc}; -use serde::Deserialize; -use tokio::task::JoinHandle; -use tokio_postgres::NoTls; - -use lard_ingestion::permissions::{ParamPermit, ParamPermitTable, StationPermitTable}; -use lard_ingestion::{PgConnectionPool, PooledPgConn}; - -const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; -pub const PARAMCONV_CSV: &str = "../ingestion/resources/paramconversions.csv"; - -#[derive(Debug, Deserialize)] -pub struct IngestorResponse { - pub message: String, - pub message_id: usize, - pub res: u8, - pub retry: bool, -} - -#[derive(Debug, Deserialize)] -pub struct StationsResponse { - pub tseries: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct Tseries { - pub regularity: String, - pub data: Vec, - // header: ... -} - -#[derive(Debug, Deserialize)] -pub struct LatestResponse { - pub data: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct Data { - // TODO: Missing param_id here? - pub value: f64, - pub timestamp: DateTime, - pub station_id: i32, - // loc: {lat, lon, hamsl, hag} -} - -#[derive(Debug, Deserialize)] -pub struct TimesliceResponse { - pub tslices: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct Tslice { - pub timestamp: DateTime, - pub param_id: i32, - pub data: Vec, -} - -pub async fn init_api_server() -> JoinHandle<()> { - tokio::spawn(lard_api::run(CONNECT_STRING)) -} - -pub async fn init_db_pool() -> Result { - let manager = PostgresConnectionManager::new_from_stringlike(CONNECT_STRING, NoTls)?; - let pool = Pool::builder().build(manager).await?; - Ok(pool) -} - -pub fn mock_permit_tables() -> Arc> { - let param_permit = HashMap::from([ - // station_id -> (type_id, param_id, permit_id) - (1, vec![ParamPermit::new(0, 0, 0)]), - (2, vec![ParamPermit::new(0, 0, 1)]), // open - ]); - - let station_permit = HashMap::from([ - // station_id -> permit_id - (1, 0), - (2, 0), - (3, 0), - (4, 1), // open - // used in e2e tests - (20000, 1), - (30000, 1), - (11000, 1), - (12000, 1), - (12100, 1), - (13000, 1), - (40000, 1), - ]); - - Arc::new(RwLock::new((param_permit, station_permit))) -} - -pub async fn number_of_data_rows(conn: &PooledPgConn<'_>, ts_id: i32) -> usize { - let rows = conn - .query( - "SELECT * FROM public.data - WHERE timeseries = $1", - &[&ts_id], - ) - .await - .unwrap(); - - rows.len() -} - -pub async fn init_ingestion_server( -) -> JoinHandle>> { - tokio::spawn(lard_ingestion::run( - CONNECT_STRING, - PARAMCONV_CSV, - mock_permit_tables(), - )) -} diff --git a/lard_tests/tests/end-to-end.rs b/lard_tests/tests/end-to-end.rs index b89bdee..93f2bd2 100644 --- a/lard_tests/tests/end-to-end.rs +++ b/lard_tests/tests/end-to-end.rs @@ -1,18 +1,23 @@ -use std::future::Future; +use std::panic::AssertUnwindSafe; use std::{ collections::HashMap, sync::{Arc, RwLock}, }; use chrono::{DateTime, Duration, DurationRound, TimeDelta, TimeZone, Utc}; +use futures::{Future, FutureExt}; use serde::Deserialize; +use test_case::test_case; use tokio_postgres::NoTls; -use lard_ingestion::permissions::{ParamPermit, ParamPermitTable, StationPermitTable}; +use lard_ingestion::permissions::{ + timeseries_is_open, ParamPermit, ParamPermitTable, StationPermitTable, +}; const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; const PARAMCONV_CSV: &str = "../ingestion/resources/paramconversions.csv"; +// TODO: should directly use the structs already defined in the different packages? #[derive(Debug, Deserialize)] pub struct IngestorResponse { pub message: String, @@ -23,11 +28,11 @@ pub struct IngestorResponse { #[derive(Debug, Deserialize)] pub struct StationsResponse { - pub tseries: Vec, + pub tseries: Vec, } #[derive(Debug, Deserialize)] -pub struct Tseries { +pub struct StationElem { pub regularity: String, pub data: Vec, // header: ... @@ -35,11 +40,11 @@ pub struct Tseries { #[derive(Debug, Deserialize)] pub struct LatestResponse { - pub data: Vec, + pub data: Vec, } #[derive(Debug, Deserialize)] -pub struct LatestData { +pub struct LatestElem { // TODO: Missing param_id here? pub value: f64, pub timestamp: DateTime, @@ -56,7 +61,14 @@ pub struct TimesliceResponse { pub struct Tslice { pub timestamp: DateTime, pub param_id: i32, - pub data: Vec, + pub data: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct SliceElem { + pub value: f64, + pub station_id: i32, + // loc: {lat, lon, hamsl, hag} } struct Param<'a> { @@ -78,7 +90,7 @@ impl<'a> Param<'a> { struct TestData<'a> { station_id: i32, type_id: i32, - params: Vec>, + params: &'a [Param<'a>], start_time: DateTime, period: Duration, len: usize, @@ -137,20 +149,32 @@ impl<'a> TestData<'a> { pub fn mock_permit_tables() -> Arc> { let param_permit = HashMap::from([ // station_id -> (type_id, param_id, permit_id) - (1, vec![ParamPermit::new(0, 0, 0)]), - (2, vec![ParamPermit::new(0, 0, 1)]), // open + (10000, vec![ParamPermit::new(0, 0, 0)]), + (10001, vec![ParamPermit::new(0, 0, 1)]), // open ]); - #[rustfmt::skip] let station_permit = HashMap::from([ - // station_id -> permit_id ... 1 = open, everything else is closed - (20000, 1), (30000, 1), (11000, 1), (12000, 1), - (12100, 1), (13000, 1), (40000, 1), (50000, 1), + // station_id -> permit_id + (10000, 1), // potentially overidden by param_permit + (10001, 0), // potentially overidden by param_permit + (20000, 0), + (20001, 1), + (20002, 1), ]); Arc::new(RwLock::new((param_permit, station_permit))) } +#[test_case(0, 0, 0 => false; "stationid not in permit_tables")] +#[test_case(10000, 0, 0 => false; "stationid in ParamPermitTable, timeseries closed")] +#[test_case(10001, 0, 0 => true; "stationid in ParamPermitTable, timeseries open")] +#[test_case(20000, 0, 0 => false; "stationid in StationPermitTable, timeseries closed")] +#[test_case(20001, 0, 1 => true; "stationid in StationPermitTable, timeseries open")] +fn test_timeseries_is_open(station_id: i32, type_id: i32, permit_id: i32) -> bool { + let permit_tables = mock_permit_tables(); + timeseries_is_open(permit_tables, station_id, type_id, permit_id).unwrap() +} + pub async fn cleanup() { let (client, conn) = tokio_postgres::connect(CONNECT_STRING, NoTls) .await @@ -163,10 +187,9 @@ pub async fn cleanup() { }); client - .execute( - // TODO: public.timeseries_id_seq? RESTART IDENTITY CASCADE? + .batch_execute( + // TODO: should clean public.timeseries_id_seq too? RESTART IDENTITY CASCADE? "TRUNCATE public.timeseries, labels.met, labels.obsinn CASCADE", - &[], ) .await .unwrap(); @@ -181,9 +204,13 @@ async fn e2e_test_wrapper>(test: T) { )); tokio::select! { - _ = api_server => {panic!("API server task terminated first")}, - _ = ingestor => {panic!("Ingestor server task terminated first")}, - _ = test => {cleanup().await} + _ = api_server => { panic!("API server task terminated first") }, + _ = ingestor => { panic!("Ingestor server task terminated first") }, + // Clean up database even if test panics, to avoid test poisoning + test_result = AssertUnwindSafe(test).catch_unwind() => { + cleanup().await; + assert!(test_result.is_ok()) + } } } @@ -202,9 +229,9 @@ async fn ingest_data(client: &reqwest::Client, obsinn_msg: String) -> IngestorRe async fn test_stations_endpoint_irregular() { e2e_test_wrapper(async { let ts = TestData { - station_id: 11000, - params: vec![Param::new(222, "TGM")], // mean(grass_temperature) - start_time: Utc.with_ymd_and_hms(2012, 2, 14, 0, 0, 0).unwrap(), + station_id: 20001, + params: &[Param::new(222, "TGM"), Param::new(225, "TGX")], + start_time: Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap(), period: Duration::hours(1), type_id: 501, len: 48, @@ -222,13 +249,12 @@ async fn test_stations_endpoint_irregular() { let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); - let resp: StationsResponse = resp.json().await.unwrap(); - assert_eq!(resp.tseries.len(), 1); + let json: StationsResponse = resp.json().await.unwrap(); + assert_eq!(json.tseries.len(), 1); - let series = &resp.tseries[0]; + let series = &json.tseries[0]; assert_eq!(series.regularity, "Irregular"); assert_eq!(series.data.len(), ts.len); - println!("{:?}", series.data) } }) .await @@ -238,8 +264,8 @@ async fn test_stations_endpoint_irregular() { async fn test_stations_endpoint_regular() { e2e_test_wrapper(async { let ts = TestData { - station_id: 30000, - params: vec![Param::new(211, "TA")], + station_id: 20001, + params: &[Param::new(211, "TA"), Param::new(225, "TGX")], start_time: Utc::now().duration_trunc(TimeDelta::hours(1)).unwrap() - Duration::hours(11), period: Duration::hours(1), @@ -260,11 +286,10 @@ async fn test_stations_endpoint_regular() { let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); - let resp: StationsResponse = resp.json().await.unwrap(); - assert_eq!(resp.tseries.len(), 1); + let json: StationsResponse = resp.json().await.unwrap(); + assert_eq!(json.tseries.len(), 1); - let series = &resp.tseries[0]; - println!("{:?}", series); + let series = &json.tseries[0]; assert_eq!(series.regularity, "Regular"); assert_eq!(series.data.len(), ts.len); } @@ -272,79 +297,130 @@ async fn test_stations_endpoint_regular() { .await } +#[test_case(99999, 211; "missing station")] +#[test_case(20001, 999; "missing param")] #[tokio::test] -async fn test_latest_endpoint() { +async fn test_stations_endpoint_errors(station_id: i32, param_id: i32) { e2e_test_wrapper(async { let ts = TestData { - station_id: 20000, - params: vec![Param::new(211, "TA"), Param::new(255, "TGX")], - start_time: Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() - - Duration::minutes(179), - period: Duration::minutes(1), - type_id: 508, - len: 180, + station_id: 20001, + params: &[Param::new(211, "TA")], + start_time: Utc.with_ymd_and_hms(2024, 1, 1, 00, 00, 00).unwrap(), + period: Duration::hours(1), + type_id: 501, + len: 48, }; let client = reqwest::Client::new(); let ingestor_resp = ingest_data(&client, ts.obsinn_message()).await; assert_eq!(ingestor_resp.res, 0); - let url = "http://localhost:3000/latest"; - // let expected_data_len = 180; + for _ in ts.params { + let url = format!( + "http://localhost:3000/stations/{}/params/{}", + station_id, param_id + ); + let resp = reqwest::get(url).await.unwrap(); + // TODO: resp.status() returns 500, maybe it should return 404? + assert!(!resp.status().is_success()); + } + }) + .await +} + +// We insert 4 timeseries, 2 with new data (UTC::now()) and 2 with old data (2020) +#[test_case("", 2; "without query")] +#[test_case("?latest_max_age=2021-01-01T00:00:00Z", 2; "latest max age 1")] +#[test_case("?latest_max_age=2019-01-01T00:00:00Z", 4; "latest max age 2")] +#[tokio::test] +async fn test_latest_endpoint(query: &str, expected_len: usize) { + e2e_test_wrapper(async { + let test_data = [ + TestData { + station_id: 20001, + params: &[Param::new(211, "TA"), Param::new(225, "TGX")], + start_time: Utc::now().duration_trunc(TimeDelta::minutes(1)).unwrap() + - Duration::hours(3), + period: Duration::minutes(1), + type_id: 508, + len: 180, + }, + TestData { + station_id: 20002, + params: &[Param::new(211, "TA"), Param::new(225, "TGX")], + start_time: Utc.with_ymd_and_hms(2020, 1, 1, 0, 0, 0).unwrap(), + period: Duration::minutes(1), + type_id: 508, + len: 180, + }, + ]; + let client = reqwest::Client::new(); + for ts in test_data { + let ingestor_resp = ingest_data(&client, ts.obsinn_message()).await; + assert_eq!(ingestor_resp.res, 0); + } + + let url = format!("http://localhost:3000/latest{}", query); let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); let json: LatestResponse = resp.json().await.unwrap(); - println!("{:?}", json); - // assert_eq!(json.data.len(), expected_data_len); + assert_eq!(json.data.len(), expected_len); }) .await } -// #[test_case(40000; "new_station")] -// #[test_case(11000; "old_station")] -// #[tokio::test] -async fn create_timeseries(station_id: i32) { - e2e_test_wrapper(async move { - // 145, AGM, mean(air_gap PT10M) - // 146, AGX, max(air_gap PT10M) - let param_ids = [145, 146]; - let expected_len = 5; - let obsinn_msg = format!( - concat!( - "kldata/nationalnr={}/type=506/messageid=23\n", - "AGM(1,2),AGX(1,2)\n", - "20240606000000,11.0,12.0\n", - "20240606001000,12.0,19.0\n", - "20240606002000,10.0,16.0\n", - "20240606003000,12.0,16.0\n", - "20240606004000,11.0,15.0", - ), - station_id - ); +#[tokio::test] +async fn test_timeslice_endpoint() { + e2e_test_wrapper(async { + // TODO: test multiple slices, can it take a sequence of timeslices? + let timestamp = Utc.with_ymd_and_hms(2024, 1, 1, 1, 0, 0).unwrap(); + let param_id = 211; + + let test_data = [ + TestData { + station_id: 20001, + params: &[Param::new(211, "TA")], + start_time: Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap(), + period: Duration::hours(1), + type_id: 501, + len: 2, + }, + TestData { + station_id: 20002, + params: &[Param::new(211, "TA")], + start_time: Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap(), + period: Duration::minutes(1), + type_id: 501, + len: 120, + }, + ]; let client = reqwest::Client::new(); - let resp = client - .post("http://localhost:3001/kldata") - .body(obsinn_msg) - .send() - .await - .unwrap(); - let json: IngestorResponse = resp.json().await.unwrap(); - assert_eq!(json.res, 0); - - for id in param_ids { - let url = format!( - "http://localhost:3000/stations/{}/params/{}", - station_id, id - ); + for ts in &test_data { + let ingestor_resp = ingest_data(&client, ts.obsinn_message()).await; + assert_eq!(ingestor_resp.res, 0); + } - let resp = client.get(&url).send().await.unwrap(); - assert!(resp.status().is_success()); + let url = format!( + "http://localhost:3000/timeslices/{}/params/{}", + timestamp, param_id + ); - let json: StationsResponse = resp.json().await.unwrap(); - assert_eq!(json.tseries[0].data.len(), expected_len) + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); + + let json: TimesliceResponse = resp.json().await.unwrap(); + assert!(json.tslices.len() == 1); + + let slice = &json.tslices[0]; + assert_eq!(slice.param_id, param_id); + assert_eq!(slice.timestamp, timestamp); + assert_eq!(slice.data.len(), test_data.len()); + + for (data, ts) in slice.data.iter().zip(&test_data) { + assert_eq!(data.station_id, ts.station_id); } }) .await diff --git a/lard_tests/tests/ingestion.rs b/lard_tests/tests/ingestion.rs deleted file mode 100644 index dbc1d28..0000000 --- a/lard_tests/tests/ingestion.rs +++ /dev/null @@ -1,82 +0,0 @@ -use chrono::{TimeZone, Utc}; -use test_case::test_case; - -use lard_ingestion::{insert_data, permissions::timeseries_is_open, Data, Datum}; - -pub mod common; - -#[tokio::test] -async fn test_insert_data() { - let pool = common::init_db_pool().await.unwrap(); - let mut conn = pool.get().await.unwrap(); - - // Timeseries ID 2 has hourly data - let id: i32 = 2; - let count_before_insertion = common::number_of_data_rows(&conn, id).await; - - let data: Data = (1..10) - .map(|i| { - let timestamp = Utc.with_ymd_and_hms(2012, 2, 16, i, 0, 0).unwrap(); - let value = 49. + i as f32; - Datum::new(id, timestamp, value) - }) - .collect(); - let data_len = data.len(); - - insert_data(data, &mut conn).await.unwrap(); - - let count_after_insertion = common::number_of_data_rows(&conn, id).await; - let rows_inserted = count_after_insertion - count_before_insertion; - - // NOTE: The assert will fail locally if the database hasn't been cleaned up between runs - assert_eq!(rows_inserted, data_len); -} - -#[test_case(0, 0, 0 => false; "stationid not in permit_tables")] -#[test_case(3, 0, 0 => false; "stationid in StationPermitTable, timeseries closed")] -#[test_case(1, 0, 0 => false; "stationid in ParamPermitTable, timeseries closed")] -#[test_case(4, 0, 1 => true; "stationid in StationPermitTable, timeseries open")] -#[test_case(2, 0, 0 => true; "stationid in ParamPermitTable, timeseries open")] -fn test_timeseries_is_open(station_id: i32, type_id: i32, permit_id: i32) -> bool { - let permit_tables = common::mock_permit_tables(); - timeseries_is_open(permit_tables, station_id, type_id, permit_id).unwrap() -} - -#[tokio::test] -async fn test_kldata_endpoint() { - let ingestor = common::init_ingestion_server().await; - - let test = async { - let station_id = 12000; - let obsinn_msg = format!( - "kldata/nationalnr={}/type=508/messageid=23 -TA,TWD(0,0) -20240607134900,25.0,18.2 -20240607135100,25.1,18.3 -20240607135200,25.0,18.3 -20240607135300,24.9,18.1 -20240607135400,25.2,18.2 -20240607135500,25.1,18.2 -", - station_id - ); - - let client = reqwest::Client::new(); - let resp = client - .post("http://localhost:3001/kldata") - .body(obsinn_msg) - .send() - .await - .unwrap(); - - let json: common::IngestorResponse = resp.json().await.unwrap(); - - assert_eq!(json.res, 0); - assert_eq!(json.message_id, 23) - }; - - tokio::select! { - _ = ingestor => panic!("Server task terminated first"), - _ = test => {} - } -} From 162c0624409a5166158769f348691012e0cf896d Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 24 Jun 2024 16:53:44 +0200 Subject: [PATCH 24/41] Change workspace path --- Cargo.toml | 2 +- {lard_tests => integration_tests}/Cargo.toml | 0 integration_tests/Makefile | 20 +++++++++++++++++++ .../src/bin/prepare_db.rs | 0 .../tests/end-to-end.rs | 0 lard_tests/Makefile | 14 ------------- 6 files changed, 21 insertions(+), 15 deletions(-) rename {lard_tests => integration_tests}/Cargo.toml (100%) create mode 100644 integration_tests/Makefile rename {lard_tests => integration_tests}/src/bin/prepare_db.rs (100%) rename {lard_tests => integration_tests}/tests/end-to-end.rs (100%) delete mode 100644 lard_tests/Makefile diff --git a/Cargo.toml b/Cargo.toml index 5095624..4bff1c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = [ "fake_data_generator", "api", "ingestion", - "lard_tests", + "integration_tests", ] resolver = "2" diff --git a/lard_tests/Cargo.toml b/integration_tests/Cargo.toml similarity index 100% rename from lard_tests/Cargo.toml rename to integration_tests/Cargo.toml diff --git a/integration_tests/Makefile b/integration_tests/Makefile new file mode 100644 index 0000000..31e740c --- /dev/null +++ b/integration_tests/Makefile @@ -0,0 +1,20 @@ +test: setup + cargo test --no-fail-fast -- --nocapture --test-threads=1 + @make clean + +end-to-end: setup + cargo test --test end-to-end --no-fail-fast -- --nocapture --test-threads=1 + @make clean + +setup: + @echo "Starting Postgres docker container..." + docker run --name lard_tests -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres + @sleep 5 + cargo build --workspace --tests + cd ..; target/debug/prepare_db + +clean: + @echo "Stopping Postgres container..." + docker stop lard_tests + @echo "Removing Postgres container..." + docker rm lard_tests diff --git a/lard_tests/src/bin/prepare_db.rs b/integration_tests/src/bin/prepare_db.rs similarity index 100% rename from lard_tests/src/bin/prepare_db.rs rename to integration_tests/src/bin/prepare_db.rs diff --git a/lard_tests/tests/end-to-end.rs b/integration_tests/tests/end-to-end.rs similarity index 100% rename from lard_tests/tests/end-to-end.rs rename to integration_tests/tests/end-to-end.rs diff --git a/lard_tests/Makefile b/lard_tests/Makefile deleted file mode 100644 index fff0f0d..0000000 --- a/lard_tests/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -run_tests: - @echo "Starting Postgres docker container..." - docker run --name sql-data-test -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres - @sleep 5 - cargo build --workspace --tests - cd ..; target/debug/prepare_db - cargo test --no-fail-fast -- --nocapture --test-threads=1 - make clean - -clean: - @echo "Stopping Postgres container..." - docker stop sql-data-test - @echo "Removing Postgres container..." - docker rm sql-data-test From 3ce2accd00e8344fe2a4a7cb4c5f152772315032 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 24 Jun 2024 17:00:05 +0200 Subject: [PATCH 25/41] Remove unused impl block --- ingestion/src/lib.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ingestion/src/lib.rs b/ingestion/src/lib.rs index 49385f2..062045d 100644 --- a/ingestion/src/lib.rs +++ b/ingestion/src/lib.rs @@ -86,18 +86,6 @@ pub struct Datum { value: f32, } -// Only used in tests -#[doc(hidden)] -impl Datum { - pub fn new(timeseries_id: i32, timestamp: DateTime, value: f32) -> Self { - Datum { - timeseries_id, - timestamp, - value, - } - } -} - pub type Data = Vec; pub async fn insert_data(data: Data, conn: &mut PooledPgConn<'_>) -> Result<(), Error> { From 628c795e4d8e50aa30dd0ff9d9d7b12a62b06601 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 24 Jun 2024 17:14:02 +0200 Subject: [PATCH 26/41] cfg(test) does not apply to integration tests, we specify a separate feature to make it work --- ingestion/Cargo.toml | 3 +++ ingestion/src/kldata.rs | 3 +-- ingestion/src/permissions.rs | 3 +-- integration_tests/Cargo.toml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ingestion/Cargo.toml b/ingestion/Cargo.toml index ce9e208..5cbfcec 100644 --- a/ingestion/Cargo.toml +++ b/ingestion/Cargo.toml @@ -3,6 +3,9 @@ name = "lard_ingestion" version = "0.1.0" edition.workspace = true +[features] +integration_tests = [] + [dependencies] axum.workspace = true bb8.workspace = true diff --git a/ingestion/src/kldata.rs b/ingestion/src/kldata.rs index 79d5272..2659d2f 100644 --- a/ingestion/src/kldata.rs +++ b/ingestion/src/kldata.rs @@ -213,9 +213,8 @@ pub async fn filter_and_label_kldata( param_id.to_owned(), )? { // TODO: log that the timeseries is closed? Mostly useful for tests - #[cfg(test)] + #[cfg(feature = "integration_tests")] eprintln!("station {}: timeseries is closed", chunk.station_id); - continue; } diff --git a/ingestion/src/permissions.rs b/ingestion/src/permissions.rs index e09bec3..7489bdd 100644 --- a/ingestion/src/permissions.rs +++ b/ingestion/src/permissions.rs @@ -12,8 +12,7 @@ pub struct ParamPermit { permit_id: i32, } -// Only used in tests -#[doc(hidden)] +#[cfg(feature = "integration_tests")] impl ParamPermit { pub fn new(type_id: i32, param_id: i32, permit_id: i32) -> ParamPermit { ParamPermit { diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 4fefbd1..6b16480 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [dependencies] lard_api = { path = "../api" } -lard_ingestion = { path = "../ingestion" } +lard_ingestion = { path = "../ingestion", features = ["integration_tests"] } chrono.workspace = true tokio.workspace = true tokio-postgres.workspace = true From f559b5829c70e31306f654704489129287a95c34 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 25 Jun 2024 08:50:08 +0200 Subject: [PATCH 27/41] Clean up comments --- .github/workflows/ci.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3ab6f4..8397053 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,22 +64,3 @@ jobs: - name: Run unit and integration tests run: cargo test --no-fail-fast -- --nocapture --test-threads=1 - - # TODO: potentially we can split them up in multiple steps - # - name: Prepare PostgreSQL for API tests - # run: target/debug/prepare_db api - - # - name: API integration tests - # run: cargo test --test api -- --nocapture --test-threads=1 - - # - name: Prepare PostgreSQL for ingestion tests - # run: target/debug/prepare_db ingestion - - # - name: Ingestions integration tests - # run: cargo test --test ingestion -- --nocapture --test-threads=1 - - # - name: Prepare PostgreSQL for E2E tests - # run: target/debug/prepare_db end-to-end - - # - name: E2E integration tests - # run: cargo test --test end-to-end -- --nocapture --test-threads=1 From 34003d1edae1fe088c11b0fc2fde101fdd4f5699 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 25 Jun 2024 09:24:16 +0200 Subject: [PATCH 28/41] Join headers only at the end --- integration_tests/tests/end-to-end.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/integration_tests/tests/end-to-end.rs b/integration_tests/tests/end-to-end.rs index 93f2bd2..3c4882f 100644 --- a/integration_tests/tests/end-to-end.rs +++ b/integration_tests/tests/end-to-end.rs @@ -110,7 +110,7 @@ impl<'a> TestData<'a> { let val = 0.0; let values = vec![val.to_string(); self.params.len()].join(","); - let mut msg = vec![self.obsinn_header()]; + let mut msg = vec![self.obsinn_header(), self.param_header()]; let mut time = self.start_time; while time < self.end_time() { @@ -123,10 +123,8 @@ impl<'a> TestData<'a> { fn obsinn_header(&self) -> String { format!( - "kldata/nationalnr={}/type={}/messageid=23\n{}", - self.station_id, - self.type_id, - self.param_header() + "kldata/nationalnr={}/type={}/messageid=23", + self.station_id, self.type_id, ) } From 1f9afbc8a220a13a835e36e261631b5ee0e0fe0d Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 25 Jun 2024 09:45:09 +0200 Subject: [PATCH 29/41] Get rid of the bin directory --- integration_tests/Cargo.toml | 1 + integration_tests/src/{bin/prepare_db.rs => main.rs} | 0 2 files changed, 1 insertion(+) rename integration_tests/src/{bin/prepare_db.rs => main.rs} (100%) diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 6b16480..074e2b8 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -18,5 +18,6 @@ reqwest = {version = "0.12.4", features = ["json"]} [[bin]] name = "prepare_db" +path = "src/main.rs" test = false bench = false diff --git a/integration_tests/src/bin/prepare_db.rs b/integration_tests/src/main.rs similarity index 100% rename from integration_tests/src/bin/prepare_db.rs rename to integration_tests/src/main.rs From a2e1b89cacbd843c3ed12e7f24b8aa7d1b7ded06 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 25 Jun 2024 13:39:32 +0200 Subject: [PATCH 30/41] Use futures crate already available in the workspace --- integration_tests/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 074e2b8..d948d24 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -13,7 +13,7 @@ bb8.workspace = true bb8-postgres.workspace = true serde.workspace = true test-case.workspace = true -futures = "0.3.30" +futures.workspace = true reqwest = {version = "0.12.4", features = ["json"]} [[bin]] From d8e0a022c3abe0bb99ea0bc1811bbb92c8503b00 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 26 Jun 2024 09:38:25 +0200 Subject: [PATCH 31/41] Update timeslice test --- integration_tests/tests/end-to-end.rs | 47 ++++++++++++++------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/integration_tests/tests/end-to-end.rs b/integration_tests/tests/end-to-end.rs index 3c4882f..0a6d74b 100644 --- a/integration_tests/tests/end-to-end.rs +++ b/integration_tests/tests/end-to-end.rs @@ -71,6 +71,7 @@ pub struct SliceElem { // loc: {lat, lon, hamsl, hag} } +#[derive(Clone)] struct Param<'a> { id: i32, code: &'a str, @@ -112,8 +113,9 @@ impl<'a> TestData<'a> { let mut msg = vec![self.obsinn_header(), self.param_header()]; + let end_time = self.end_time(); let mut time = self.start_time; - while time < self.end_time() { + while time < end_time { msg.push(format!("{},{}", time.format("%Y%m%d%H%M%S"), values)); time += self.period; } @@ -372,25 +374,24 @@ async fn test_latest_endpoint(query: &str, expected_len: usize) { #[tokio::test] async fn test_timeslice_endpoint() { e2e_test_wrapper(async { - // TODO: test multiple slices, can it take a sequence of timeslices? let timestamp = Utc.with_ymd_and_hms(2024, 1, 1, 1, 0, 0).unwrap(); - let param_id = 211; + let params = vec![Param::new(211, "TA")]; let test_data = [ TestData { station_id: 20001, - params: &[Param::new(211, "TA")], - start_time: Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap(), + params: ¶ms.clone(), + start_time: timestamp - Duration::hours(1), period: Duration::hours(1), type_id: 501, len: 2, }, TestData { station_id: 20002, - params: &[Param::new(211, "TA")], - start_time: Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap(), + params: ¶ms.clone(), + start_time: timestamp - Duration::hours(1), period: Duration::minutes(1), - type_id: 501, + type_id: 508, len: 120, }, ]; @@ -401,24 +402,26 @@ async fn test_timeslice_endpoint() { assert_eq!(ingestor_resp.res, 0); } - let url = format!( - "http://localhost:3000/timeslices/{}/params/{}", - timestamp, param_id - ); + for param in ¶ms { + let url = format!( + "http://localhost:3000/timeslices/{}/params/{}", + timestamp, param.id + ); - let resp = reqwest::get(url).await.unwrap(); - assert!(resp.status().is_success()); + let resp = reqwest::get(url).await.unwrap(); + assert!(resp.status().is_success()); - let json: TimesliceResponse = resp.json().await.unwrap(); - assert!(json.tslices.len() == 1); + let json: TimesliceResponse = resp.json().await.unwrap(); + assert!(json.tslices.len() == 1); - let slice = &json.tslices[0]; - assert_eq!(slice.param_id, param_id); - assert_eq!(slice.timestamp, timestamp); - assert_eq!(slice.data.len(), test_data.len()); + let slice = &json.tslices[0]; + assert_eq!(slice.param_id, param.id); + assert_eq!(slice.timestamp, timestamp); + assert_eq!(slice.data.len(), test_data.len()); - for (data, ts) in slice.data.iter().zip(&test_data) { - assert_eq!(data.station_id, ts.station_id); + for (data, ts) in slice.data.iter().zip(&test_data) { + assert_eq!(data.station_id, ts.station_id); + } } }) .await From 90847887a8ec7b7ec524cc38c4568a569d11940c Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 26 Jun 2024 10:31:00 +0200 Subject: [PATCH 32/41] Update file name --- .github/workflows/ci.yml | 2 +- integration_tests/Cargo.toml | 2 +- integration_tests/Makefile | 16 +++++++++------- .../tests/{end-to-end.rs => end_to_end.rs} | 0 4 files changed, 11 insertions(+), 9 deletions(-) rename integration_tests/tests/{end-to-end.rs => end_to_end.rs} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8397053..75dbfe4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: run: cargo clippy --workspace -- -D warnings - name: Prepare PostgreSQL - run: target/debug/prepare_db + run: target/debug/prepare_postgres - name: Run unit and integration tests run: cargo test --no-fail-fast -- --nocapture --test-threads=1 diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index d948d24..699ebcb 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -17,7 +17,7 @@ futures.workspace = true reqwest = {version = "0.12.4", features = ["json"]} [[bin]] -name = "prepare_db" +name = "prepare_postgres" path = "src/main.rs" test = false bench = false diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 31e740c..80bddab 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,17 +1,19 @@ -test: setup +test_all: _test_dummy clean +_test_dummy: setup cargo test --no-fail-fast -- --nocapture --test-threads=1 - @make clean -end-to-end: setup - cargo test --test end-to-end --no-fail-fast -- --nocapture --test-threads=1 - @make clean +e2e := end_to_end +$(e2e): _e2e_dummy clean +_e2e_dummy: setup + cargo test --test $(e2e) --no-fail-fast -- --nocapture --test-threads=1 setup: @echo "Starting Postgres docker container..." docker run --name lard_tests -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres - @sleep 5 + @echo; sleep 5 cargo build --workspace --tests - cd ..; target/debug/prepare_db + @echo; echo "Loading DB schema..."; echo + @cd ..; target/debug/prepare_postgres clean: @echo "Stopping Postgres container..." diff --git a/integration_tests/tests/end-to-end.rs b/integration_tests/tests/end_to_end.rs similarity index 100% rename from integration_tests/tests/end-to-end.rs rename to integration_tests/tests/end_to_end.rs From 6048f58ea03ad19ee7f443b9b8667a311b314516 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 27 Jun 2024 15:23:50 +0200 Subject: [PATCH 33/41] Clean up makefile --- integration_tests/Makefile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 80bddab..4b6d8a3 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,11 +1,10 @@ -test_all: _test_dummy clean -_test_dummy: setup +test_all: _test_all clean +_test_all: setup cargo test --no-fail-fast -- --nocapture --test-threads=1 -e2e := end_to_end -$(e2e): _e2e_dummy clean -_e2e_dummy: setup - cargo test --test $(e2e) --no-fail-fast -- --nocapture --test-threads=1 +end_to_end: _end_to_end clean +_end_to_end: setup + cargo test --test end_to_end --no-fail-fast -- --nocapture --test-threads=1 setup: @echo "Starting Postgres docker container..." From 4b73813bc4025b9cf8a4b6a2fa570c86ecb4ab57 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 28 Jun 2024 15:04:09 +0200 Subject: [PATCH 34/41] Make database clean up conditional for debugging --- integration_tests/Cargo.toml | 3 +++ integration_tests/Makefile | 4 ++++ integration_tests/tests/end_to_end.rs | 6 ++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 699ebcb..6c45d03 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -3,6 +3,9 @@ name = "lard_tests" version = "0.1.0" edition.workspace = true +[features] +debug = [] + [dependencies] lard_api = { path = "../api" } lard_ingestion = { path = "../ingestion", features = ["integration_tests"] } diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 4b6d8a3..a03d5b4 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -6,6 +6,10 @@ end_to_end: _end_to_end clean _end_to_end: setup cargo test --test end_to_end --no-fail-fast -- --nocapture --test-threads=1 +# pass TEST=... make debug_test +debug_test: setup + cargo test "$(TEST)" --features debug --no-fail-fast -- --nocapture --test-threads=1 + setup: @echo "Starting Postgres docker container..." docker run --name lard_tests -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres diff --git a/integration_tests/tests/end_to_end.rs b/integration_tests/tests/end_to_end.rs index 0a6d74b..cafd764 100644 --- a/integration_tests/tests/end_to_end.rs +++ b/integration_tests/tests/end_to_end.rs @@ -204,10 +204,12 @@ async fn e2e_test_wrapper>(test: T) { )); tokio::select! { - _ = api_server => { panic!("API server task terminated first") }, - _ = ingestor => { panic!("Ingestor server task terminated first") }, + _ = api_server => panic!("API server task terminated first"), + _ = ingestor => panic!("Ingestor server task terminated first"), // Clean up database even if test panics, to avoid test poisoning test_result = AssertUnwindSafe(test).catch_unwind() => { + // For debugging a specific test, it might be useful to avoid cleaning up + #[cfg(not(feature = "debug"))] cleanup().await; assert!(test_result.is_ok()) } From a2e3b0402835f3e950ad849153841c13bb4f350a Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 1 Jul 2024 10:52:19 +0200 Subject: [PATCH 35/41] Small fixes --- integration_tests/Makefile | 4 +++- integration_tests/tests/end_to_end.rs | 28 +++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/integration_tests/Makefile b/integration_tests/Makefile index a03d5b4..720b185 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -6,7 +6,9 @@ end_to_end: _end_to_end clean _end_to_end: setup cargo test --test end_to_end --no-fail-fast -- --nocapture --test-threads=1 -# pass TEST=... make debug_test +# With the `debug` feature, the database is not cleaned up after running the test, +# so it can be inspected with psql. Run with: +# TEST= make debug_test debug_test: setup cargo test "$(TEST)" --features debug --no-fail-fast -- --nocapture --test-threads=1 diff --git a/integration_tests/tests/end_to_end.rs b/integration_tests/tests/end_to_end.rs index cafd764..c6e8670 100644 --- a/integration_tests/tests/end_to_end.rs +++ b/integration_tests/tests/end_to_end.rs @@ -155,8 +155,8 @@ pub fn mock_permit_tables() -> Arc permit_id - (10000, 1), // potentially overidden by param_permit - (10001, 0), // potentially overidden by param_permit + (10000, 1), // overridden by param_permit + (10001, 0), // overridden by param_permit (20000, 0), (20001, 1), (20002, 1), @@ -175,17 +175,7 @@ fn test_timeseries_is_open(station_id: i32, type_id: i32, permit_id: i32) -> boo timeseries_is_open(permit_tables, station_id, type_id, permit_id).unwrap() } -pub async fn cleanup() { - let (client, conn) = tokio_postgres::connect(CONNECT_STRING, NoTls) - .await - .unwrap(); - - tokio::spawn(async move { - if let Err(e) = conn.await { - eprintln!("{}", e); - } - }); - +pub async fn cleanup(client: &tokio_postgres::Client) { client .batch_execute( // TODO: should clean public.timeseries_id_seq too? RESTART IDENTITY CASCADE? @@ -203,6 +193,16 @@ async fn e2e_test_wrapper>(test: T) { mock_permit_tables(), )); + let (client, conn) = tokio_postgres::connect(CONNECT_STRING, NoTls) + .await + .unwrap(); + + tokio::spawn(async move { + if let Err(e) = conn.await { + eprintln!("{}", e); + } + }); + tokio::select! { _ = api_server => panic!("API server task terminated first"), _ = ingestor => panic!("Ingestor server task terminated first"), @@ -210,7 +210,7 @@ async fn e2e_test_wrapper>(test: T) { test_result = AssertUnwindSafe(test).catch_unwind() => { // For debugging a specific test, it might be useful to avoid cleaning up #[cfg(not(feature = "debug"))] - cleanup().await; + cleanup(&client).await; assert!(test_result.is_ok()) } } From a4468c6b4d4ff8a5ea156a242d008949524d97d3 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 1 Jul 2024 11:04:00 +0200 Subject: [PATCH 36/41] Clean up main.rs --- integration_tests/src/main.rs | 44 ++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/integration_tests/src/main.rs b/integration_tests/src/main.rs index 2e9a307..caffc4d 100644 --- a/integration_tests/src/main.rs +++ b/integration_tests/src/main.rs @@ -4,28 +4,40 @@ use tokio_postgres::{Error, NoTls}; const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; +async fn insert_schema(client: &tokio_postgres::Client, filename: &str) -> Result<(), Error> { + let schema = fs::read_to_string(filename).expect("Should be able to read SQL file"); + client.batch_execute(schema.as_str()).await +} + +async fn create_data_partition(client: &tokio_postgres::Client) -> Result<(), Error> { + // TODO: add multiple partitions? + let partition_string = format!( + "CREATE TABLE data_y{}_to_y{} PARTITION OF public.data FOR VALUES FROM ('{}') TO ('{}')", + "1950", "2100", "1950-01-01 00:00:00+00", "2100-01-01 00:00:00+00", + ); + client.batch_execute(partition_string.as_str()).await +} + #[tokio::main] -async fn main() -> Result<(), Error> { - let (client, connection) = tokio_postgres::connect(CONNECT_STRING, NoTls).await?; +async fn main() { + let (client, connection) = tokio_postgres::connect(CONNECT_STRING, NoTls) + .await + .expect("Should be able to connect to database"); tokio::spawn(async move { if let Err(e) = connection.await { - eprintln!("connection error: {}", e); + eprintln!("connection error: {e}"); } }); - let public_schema = - fs::read_to_string("db/public.sql").expect("Should be able to read SQL file"); - client.batch_execute(public_schema.as_str()).await?; - - let labels_schema = - fs::read_to_string("db/labels.sql").expect("Should be able to read SQL file"); - client.batch_execute(labels_schema.as_str()).await?; + let schemas = ["db/public.sql", "db/labels.sql"]; + for schema in schemas { + if let Err(e) = insert_schema(&client, schema).await { + eprintln!("Error: {e} ({schema})") + } + } - // TODO: add multiple partitions? - let partition_string = format!( - "CREATE TABLE data_y{}_to_y{} PARTITION OF public.data FOR VALUES FROM ('{}') TO ('{}')", - "1950", "2100", "1950-01-01 00:00:00+00", "2100-01-01 00:00:00+00", - ); - client.batch_execute(partition_string.as_str()).await + if let Err(e) = create_data_partition(&client).await { + eprintln!("connection error: {e}"); + } } From 8f49c0c0ec980430ccfd309a1844e0c4bbd29237 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 1 Jul 2024 15:02:33 +0200 Subject: [PATCH 37/41] Reuse structs that were already implemented --- api/src/latest.rs | 12 ++-- api/src/lib.rs | 25 ++++---- api/src/timeseries.rs | 34 +++++------ api/src/timeslice.rs | 18 +++--- api/src/util.rs | 4 +- ingestion/src/lib.rs | 14 ++--- integration_tests/tests/end_to_end.rs | 85 ++++++--------------------- 7 files changed, 71 insertions(+), 121 deletions(-) diff --git a/api/src/latest.rs b/api/src/latest.rs index 48ccfda..a438987 100644 --- a/api/src/latest.rs +++ b/api/src/latest.rs @@ -1,13 +1,13 @@ use crate::util::{Location, PooledPgConn}; use chrono::{DateTime, Utc}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct LatestElem { - value: f32, - timestamp: DateTime, - station_id: i32, - loc: Option, + pub value: f32, + pub timestamp: DateTime, + pub station_id: i32, + pub loc: Option, } pub async fn get_latest( diff --git a/api/src/lib.rs b/api/src/lib.rs index 98adf3c..e1120a2 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -14,9 +14,9 @@ use timeseries::{ use timeslice::{get_timeslice, Timeslice}; use tokio_postgres::NoTls; -mod latest; -mod timeseries; -mod timeslice; +pub mod latest; +pub mod timeseries; +pub mod timeslice; pub(crate) mod util; type PgConnectionPool = bb8::Pool>; @@ -34,14 +34,14 @@ struct TimeseriesParams { time_resolution: Option, } -#[derive(Debug, Serialize)] -struct TimeseriesResp { - tseries: Vec, +#[derive(Debug, Serialize, Deserialize)] +pub struct TimeseriesResp { + pub tseries: Vec, } -#[derive(Debug, Serialize)] -struct TimesliceResp { - tslices: Vec, +#[derive(Debug, Serialize, Deserialize)] +pub struct TimesliceResp { + pub tslices: Vec, } #[derive(Debug, Deserialize)] @@ -49,9 +49,9 @@ struct LatestParams { latest_max_age: Option>, } -#[derive(Debug, Serialize)] -struct LatestResp { - data: Vec, +#[derive(Debug, Serialize, Deserialize)] +pub struct LatestResp { + pub data: Vec, } async fn stations_handler( @@ -118,7 +118,6 @@ async fn latest_handler( Ok(Json(LatestResp { data })) } - pub async fn run(connect_string: &str) { // set up postgres connection pool let manager = PostgresConnectionManager::new_from_stringlike(connect_string, NoTls).unwrap(); diff --git a/api/src/timeseries.rs b/api/src/timeseries.rs index cf39caa..6bd4d3f 100644 --- a/api/src/timeseries.rs +++ b/api/src/timeseries.rs @@ -1,36 +1,36 @@ use crate::util::{Location, PooledPgConn}; use chrono::{DateTime, Utc}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; // TODO: this should be more comprehensive once the schema supports it -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct TimeseriesInfo { pub ts_id: i32, pub fromtime: DateTime, pub totime: DateTime, - station_id: i32, - param_id: i32, - lvl: Option, - sensor: Option, - location: Option, + pub station_id: i32, + pub param_id: i32, + pub lvl: Option, + pub sensor: Option, + pub location: Option, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct TimeseriesIrregular { - header: TimeseriesInfo, - data: Vec, - timestamps: Vec>, + pub header: TimeseriesInfo, + pub data: Vec, + pub timestamps: Vec>, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct TimeseriesRegular { - header: TimeseriesInfo, - data: Vec>, - start_time: DateTime, - time_resolution: String, + pub header: TimeseriesInfo, + pub data: Vec>, + pub start_time: DateTime, + pub time_resolution: String, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(tag = "regularity")] pub enum Timeseries { Regular(TimeseriesRegular), diff --git a/api/src/timeslice.rs b/api/src/timeslice.rs index e2df484..1524c8c 100644 --- a/api/src/timeslice.rs +++ b/api/src/timeslice.rs @@ -1,22 +1,22 @@ use crate::util::{Location, PooledPgConn}; use chrono::{DateTime, Utc}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct TimesliceElem { - value: f32, - station_id: i32, + pub value: f32, + pub station_id: i32, // TODO: this shouldn't be an Option, but it avoids panics if location is somehow // not found in the database - loc: Option, + pub loc: Option, } // TODO: consider whether this should be object-of-arrays style -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct Timeslice { - timestamp: DateTime, - param_id: i32, - data: Vec, + pub timestamp: DateTime, + pub param_id: i32, + pub data: Vec, } pub async fn get_timeslice( diff --git a/api/src/util.rs b/api/src/util.rs index 3413f1a..f82739c 100644 --- a/api/src/util.rs +++ b/api/src/util.rs @@ -1,11 +1,11 @@ use bb8::PooledConnection; use bb8_postgres::PostgresConnectionManager; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use tokio_postgres::{types::FromSql, NoTls}; pub type PooledPgConn<'a> = PooledConnection<'a, PostgresConnectionManager>; -#[derive(Debug, Serialize, FromSql)] +#[derive(Debug, Serialize, Deserialize, FromSql)] #[postgres(name = "location")] pub struct Location { lat: Option, diff --git a/ingestion/src/lib.rs b/ingestion/src/lib.rs index 062045d..9a54440 100644 --- a/ingestion/src/lib.rs +++ b/ingestion/src/lib.rs @@ -8,7 +8,7 @@ use bb8::PooledConnection; use bb8_postgres::PostgresConnectionManager; use chrono::{DateTime, Utc}; use futures::{stream::FuturesUnordered, StreamExt}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -132,19 +132,19 @@ pub mod kldata; use kldata::{filter_and_label_kldata, parse_kldata}; /// Format of response Obsinn expects from this API -#[derive(Debug, Serialize)] -struct KldataResp { +#[derive(Debug, Serialize, Deserialize)] +pub struct KldataResp { /// Optional message indicating what happened to the data - message: String, + pub message: String, /// Should be the same message_id we received in the request - message_id: usize, + pub message_id: usize, /// Result indicator, 0 means success, anything else means fail. // Kvalobs uses some specific numbers to denote specific errors with this, I don't much see // the point, the only information Obsinn can really action on as far as I can tell, is whether // we failed and whether it can retry - res: u8, // TODO: Should be an enum? + pub res: u8, // TODO: Should be an enum? /// Indicates whether Obsinn should try to send the message again - retry: bool, + pub retry: bool, } async fn handle_kldata( diff --git a/integration_tests/tests/end_to_end.rs b/integration_tests/tests/end_to_end.rs index c6e8670..eeac499 100644 --- a/integration_tests/tests/end_to_end.rs +++ b/integration_tests/tests/end_to_end.rs @@ -6,7 +6,9 @@ use std::{ use chrono::{DateTime, Duration, DurationRound, TimeDelta, TimeZone, Utc}; use futures::{Future, FutureExt}; -use serde::Deserialize; +use lard_api::timeseries::Timeseries; +use lard_api::{LatestResp, TimeseriesResp, TimesliceResp}; +use lard_ingestion::KldataResp; use test_case::test_case; use tokio_postgres::NoTls; @@ -17,60 +19,6 @@ use lard_ingestion::permissions::{ const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; const PARAMCONV_CSV: &str = "../ingestion/resources/paramconversions.csv"; -// TODO: should directly use the structs already defined in the different packages? -#[derive(Debug, Deserialize)] -pub struct IngestorResponse { - pub message: String, - pub message_id: usize, - pub res: u8, - pub retry: bool, -} - -#[derive(Debug, Deserialize)] -pub struct StationsResponse { - pub tseries: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct StationElem { - pub regularity: String, - pub data: Vec, - // header: ... -} - -#[derive(Debug, Deserialize)] -pub struct LatestResponse { - pub data: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct LatestElem { - // TODO: Missing param_id here? - pub value: f64, - pub timestamp: DateTime, - pub station_id: i32, - // loc: {lat, lon, hamsl, hag} -} - -#[derive(Debug, Deserialize)] -pub struct TimesliceResponse { - pub tslices: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct Tslice { - pub timestamp: DateTime, - pub param_id: i32, - pub data: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct SliceElem { - pub value: f64, - pub station_id: i32, - // loc: {lat, lon, hamsl, hag} -} - #[derive(Clone)] struct Param<'a> { id: i32, @@ -146,7 +94,7 @@ impl<'a> TestData<'a> { } } -pub fn mock_permit_tables() -> Arc> { +fn mock_permit_tables() -> Arc> { let param_permit = HashMap::from([ // station_id -> (type_id, param_id, permit_id) (10000, vec![ParamPermit::new(0, 0, 0)]), @@ -175,7 +123,7 @@ fn test_timeseries_is_open(station_id: i32, type_id: i32, permit_id: i32) -> boo timeseries_is_open(permit_tables, station_id, type_id, permit_id).unwrap() } -pub async fn cleanup(client: &tokio_postgres::Client) { +async fn cleanup(client: &tokio_postgres::Client) { client .batch_execute( // TODO: should clean public.timeseries_id_seq too? RESTART IDENTITY CASCADE? @@ -208,7 +156,7 @@ async fn e2e_test_wrapper>(test: T) { _ = ingestor => panic!("Ingestor server task terminated first"), // Clean up database even if test panics, to avoid test poisoning test_result = AssertUnwindSafe(test).catch_unwind() => { - // For debugging a specific test, it might be useful to avoid cleaning up + // For debugging a specific test, it might be useful to skip cleaning up #[cfg(not(feature = "debug"))] cleanup(&client).await; assert!(test_result.is_ok()) @@ -216,7 +164,7 @@ async fn e2e_test_wrapper>(test: T) { } } -async fn ingest_data(client: &reqwest::Client, obsinn_msg: String) -> IngestorResponse { +async fn ingest_data(client: &reqwest::Client, obsinn_msg: String) -> KldataResp { let resp = client .post("http://localhost:3001/kldata") .body(obsinn_msg) @@ -251,11 +199,13 @@ async fn test_stations_endpoint_irregular() { let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); - let json: StationsResponse = resp.json().await.unwrap(); + let json: TimeseriesResp = resp.json().await.unwrap(); assert_eq!(json.tseries.len(), 1); - let series = &json.tseries[0]; - assert_eq!(series.regularity, "Irregular"); + let Timeseries::Irregular(series) = &json.tseries[0] else { + panic!("Expected irrregular timeseries") + }; + assert_eq!(series.data.len(), ts.len); } }) @@ -288,11 +238,12 @@ async fn test_stations_endpoint_regular() { let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); - let json: StationsResponse = resp.json().await.unwrap(); + let json: TimeseriesResp = resp.json().await.unwrap(); assert_eq!(json.tseries.len(), 1); - let series = &json.tseries[0]; - assert_eq!(series.regularity, "Regular"); + let Timeseries::Regular(series) = &json.tseries[0] else { + panic!("Expected regular timeseries") + }; assert_eq!(series.data.len(), ts.len); } }) @@ -367,7 +318,7 @@ async fn test_latest_endpoint(query: &str, expected_len: usize) { let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); - let json: LatestResponse = resp.json().await.unwrap(); + let json: LatestResp = resp.json().await.unwrap(); assert_eq!(json.data.len(), expected_len); }) .await @@ -413,7 +364,7 @@ async fn test_timeslice_endpoint() { let resp = reqwest::get(url).await.unwrap(); assert!(resp.status().is_success()); - let json: TimesliceResponse = resp.json().await.unwrap(); + let json: TimesliceResp = resp.json().await.unwrap(); assert!(json.tslices.len() == 1); let slice = &json.tslices[0]; From f0e4a6df68357c160eca4ed2ba13b47d59a6c7e7 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 1 Jul 2024 17:04:43 +0200 Subject: [PATCH 38/41] Add README.md for integration tests --- integration_tests/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 integration_tests/README.md diff --git a/integration_tests/README.md b/integration_tests/README.md new file mode 100644 index 0000000..8d71b60 --- /dev/null +++ b/integration_tests/README.md @@ -0,0 +1,25 @@ +# Integration tests + +## End-to-end + +1. Each test is defined inside a wrapper function (`e2e_test_wrapper`) that spawns separate tasks for the ingestor, the API server, + and a Postgres client used to clean up the database after the test is completed. + +1. Each test defines a `TestData` struct (or an array of structs) that contains the metadata + required to build a timeseries. The data is formatted into an Obsinn message, + which is sent via a POST request to the ingestor. + +1. Finally, the data is retrived and checked by sending a GET request to one of the API endpoints. + +If you have Docker installed, you can test locally by using the provided `Makefile`: + +```terminal +# Run all integration tests +make end_to_end + +# Debug a specific test (does not clean up the DB) +TEST=my_test_name make debug_test + +# If any error occurs, you might need to delete the DB container manually +make clean +``` From ea7a91a2dfc3697e4189ad7b12ffd571f2395948 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 2 Jul 2024 12:57:06 +0200 Subject: [PATCH 39/41] Minor changes --- integration_tests/README.md | 6 +++--- integration_tests/src/main.rs | 8 ++------ integration_tests/tests/end_to_end.rs | 6 +++--- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/integration_tests/README.md b/integration_tests/README.md index 8d71b60..c30f4b3 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -3,7 +3,7 @@ ## End-to-end 1. Each test is defined inside a wrapper function (`e2e_test_wrapper`) that spawns separate tasks for the ingestor, the API server, - and a Postgres client used to clean up the database after the test is completed. + and a Postgres client that cleans up the database after the test is completed. 1. Each test defines a `TestData` struct (or an array of structs) that contains the metadata required to build a timeseries. The data is formatted into an Obsinn message, @@ -11,7 +11,7 @@ 1. Finally, the data is retrived and checked by sending a GET request to one of the API endpoints. -If you have Docker installed, you can test locally by using the provided `Makefile`: +If you have Docker installed, you can run the tests locally using the provided `Makefile`: ```terminal # Run all integration tests @@ -20,6 +20,6 @@ make end_to_end # Debug a specific test (does not clean up the DB) TEST=my_test_name make debug_test -# If any error occurs, you might need to delete the DB container manually +# If any error occurs, you might need to reset the DB container manually make clean ``` diff --git a/integration_tests/src/main.rs b/integration_tests/src/main.rs index caffc4d..4514f22 100644 --- a/integration_tests/src/main.rs +++ b/integration_tests/src/main.rs @@ -32,12 +32,8 @@ async fn main() { let schemas = ["db/public.sql", "db/labels.sql"]; for schema in schemas { - if let Err(e) = insert_schema(&client, schema).await { - eprintln!("Error: {e} ({schema})") - } + insert_schema(&client, schema).await.unwrap(); } - if let Err(e) = create_data_partition(&client).await { - eprintln!("connection error: {e}"); - } + create_data_partition(&client).await.unwrap(); } diff --git a/integration_tests/tests/end_to_end.rs b/integration_tests/tests/end_to_end.rs index eeac499..409a1d7 100644 --- a/integration_tests/tests/end_to_end.rs +++ b/integration_tests/tests/end_to_end.rs @@ -6,15 +6,15 @@ use std::{ use chrono::{DateTime, Duration, DurationRound, TimeDelta, TimeZone, Utc}; use futures::{Future, FutureExt}; -use lard_api::timeseries::Timeseries; -use lard_api::{LatestResp, TimeseriesResp, TimesliceResp}; -use lard_ingestion::KldataResp; use test_case::test_case; use tokio_postgres::NoTls; +use lard_api::timeseries::Timeseries; +use lard_api::{LatestResp, TimeseriesResp, TimesliceResp}; use lard_ingestion::permissions::{ timeseries_is_open, ParamPermit, ParamPermitTable, StationPermitTable, }; +use lard_ingestion::KldataResp; const CONNECT_STRING: &str = "host=localhost user=postgres dbname=postgres password=postgres"; const PARAMCONV_CSV: &str = "../ingestion/resources/paramconversions.csv"; From 11a656d83d78e4958f7e042cd82b35cb2c91e6c4 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 2 Jul 2024 14:07:14 +0200 Subject: [PATCH 40/41] Formatting --- integration_tests/README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/integration_tests/README.md b/integration_tests/README.md index c30f4b3..95cb69a 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -2,16 +2,19 @@ ## End-to-end -1. Each test is defined inside a wrapper function (`e2e_test_wrapper`) that spawns separate tasks for the ingestor, the API server, - and a Postgres client that cleans up the database after the test is completed. +1. Each test is defined inside a wrapper function (`e2e_test_wrapper`) that + spawns separate tasks for the ingestor, the API server, and a Postgres + client that cleans up the database after the test is completed. -1. Each test defines a `TestData` struct (or an array of structs) that contains the metadata - required to build a timeseries. The data is formatted into an Obsinn message, - which is sent via a POST request to the ingestor. +1. Each test defines a `TestData` struct (or an array of structs) that contains + the metadata required to build a timeseries. The data is formatted into an + Obsinn message, which is sent via a POST request to the ingestor. -1. Finally, the data is retrived and checked by sending a GET request to one of the API endpoints. +1. Finally, the data is retrived and checked by sending a GET request to one of + the API endpoints. -If you have Docker installed, you can run the tests locally using the provided `Makefile`: +If you have Docker installed, you can run the tests locally using the provided +`Makefile`: ```terminal # Run all integration tests From bfba2b382db922d81c8419c386c88c21ada56a85 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 3 Jul 2024 09:09:36 +0200 Subject: [PATCH 41/41] Revert struct fields privacy changes --- api/src/latest.rs | 8 ++++---- api/src/timeseries.rs | 20 ++++++++++---------- api/src/timeslice.rs | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api/src/latest.rs b/api/src/latest.rs index a438987..cb8bdf1 100644 --- a/api/src/latest.rs +++ b/api/src/latest.rs @@ -4,10 +4,10 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] pub struct LatestElem { - pub value: f32, - pub timestamp: DateTime, - pub station_id: i32, - pub loc: Option, + value: f32, + timestamp: DateTime, + station_id: i32, + loc: Option, } pub async fn get_latest( diff --git a/api/src/timeseries.rs b/api/src/timeseries.rs index 6bd4d3f..d491a3d 100644 --- a/api/src/timeseries.rs +++ b/api/src/timeseries.rs @@ -8,26 +8,26 @@ pub struct TimeseriesInfo { pub ts_id: i32, pub fromtime: DateTime, pub totime: DateTime, - pub station_id: i32, - pub param_id: i32, - pub lvl: Option, - pub sensor: Option, - pub location: Option, + station_id: i32, + param_id: i32, + lvl: Option, + sensor: Option, + location: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct TimeseriesIrregular { - pub header: TimeseriesInfo, pub data: Vec, - pub timestamps: Vec>, + header: TimeseriesInfo, + timestamps: Vec>, } #[derive(Debug, Serialize, Deserialize)] pub struct TimeseriesRegular { - pub header: TimeseriesInfo, pub data: Vec>, - pub start_time: DateTime, - pub time_resolution: String, + header: TimeseriesInfo, + start_time: DateTime, + time_resolution: String, } #[derive(Debug, Serialize, Deserialize)] diff --git a/api/src/timeslice.rs b/api/src/timeslice.rs index 1524c8c..0f9a263 100644 --- a/api/src/timeslice.rs +++ b/api/src/timeslice.rs @@ -4,11 +4,11 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] pub struct TimesliceElem { - pub value: f32, pub station_id: i32, + value: f32, // TODO: this shouldn't be an Option, but it avoids panics if location is somehow // not found in the database - pub loc: Option, + loc: Option, } // TODO: consider whether this should be object-of-arrays style