From 764a9221cc51665c46db436fcf52aa27307f2f61 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 10:19:11 -0500 Subject: [PATCH 01/16] fixed documentation issue --- icann-rdap-client/src/registered_redactions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icann-rdap-client/src/registered_redactions.rs b/icann-rdap-client/src/registered_redactions.rs index 33a4787..45e9c96 100644 --- a/icann-rdap-client/src/registered_redactions.rs +++ b/icann-rdap-client/src/registered_redactions.rs @@ -77,7 +77,7 @@ pub fn is_redaction_registered( } /// This function takes a set of [RedactedName]s instead of just one, -/// and runs them through [is_redacted_name]. +/// and runs them through [is_redaction_registered]. pub fn are_redactions_registered( rdap_response: &RdapResponse, redaction_types: &[&RedactedName], From d6cb4b5143b62dd2b4b69e45d1a36ef64f94924e Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 10:51:44 -0500 Subject: [PATCH 02/16] moving bootstrapping and iana_request into iana --- icann-rdap-cli/src/bin/rdap-test/error.rs | 2 +- icann-rdap-cli/src/bin/rdap/bootstrap.rs | 8 +++++--- icann-rdap-cli/src/bin/rdap/error.rs | 2 +- icann-rdap-cli/src/dirs/fcbs.rs | 8 ++++---- icann-rdap-cli/src/rt/exec.rs | 6 ++---- icann-rdap-client/src/{query => iana}/bootstrap.rs | 8 +++----- icann-rdap-client/src/{ => iana}/iana_request.rs | 0 icann-rdap-client/src/iana/mod.rs | 4 ++++ icann-rdap-client/src/lib.rs | 6 +++--- icann-rdap-client/src/query/mod.rs | 1 - icann-rdap-client/src/query/request.rs | 14 +++++++------- icann-rdap-srv/src/bootstrap.rs | 2 +- icann-rdap-srv/src/error.rs | 2 +- 13 files changed, 32 insertions(+), 31 deletions(-) rename icann-rdap-client/src/{query => iana}/bootstrap.rs (98%) rename icann-rdap-client/src/{ => iana}/iana_request.rs (100%) create mode 100644 icann-rdap-client/src/iana/mod.rs diff --git a/icann-rdap-cli/src/bin/rdap-test/error.rs b/icann-rdap-cli/src/bin/rdap-test/error.rs index 9b56095..126fc15 100644 --- a/icann-rdap-cli/src/bin/rdap-test/error.rs +++ b/icann-rdap-cli/src/bin/rdap-test/error.rs @@ -1,7 +1,7 @@ use std::process::{ExitCode, Termination}; use icann_rdap_cli::rt::exec::TestExecutionError; -use icann_rdap_client::iana_request::IanaResponseError; +use icann_rdap_client::iana::iana_request::IanaResponseError; use icann_rdap_client::RdapClientError; use thiserror::Error; diff --git a/icann-rdap-cli/src/bin/rdap/bootstrap.rs b/icann-rdap-cli/src/bin/rdap/bootstrap.rs index 2ae6eb2..f52e0de 100644 --- a/icann-rdap-cli/src/bin/rdap/bootstrap.rs +++ b/icann-rdap-cli/src/bin/rdap/bootstrap.rs @@ -1,8 +1,10 @@ use crate::error::RdapCliError; use icann_rdap_cli::dirs::fcbs::FileCacheBootstrapStore; -use icann_rdap_client::query::{ - bootstrap::{fetch_bootstrap, qtype_to_bootstrap_url, BootstrapStore, PreferredUrl}, - qtype::QueryType, +use icann_rdap_client::iana::bootstrap::BootstrapStore; +use icann_rdap_client::iana::bootstrap::PreferredUrl; +use icann_rdap_client::{ + iana::bootstrap::{fetch_bootstrap, qtype_to_bootstrap_url}, + query::qtype::QueryType, }; use icann_rdap_common::iana::IanaRegistryType; use reqwest::Client; diff --git a/icann-rdap-cli/src/bin/rdap/error.rs b/icann-rdap-cli/src/bin/rdap/error.rs index 5b450c5..3e3209a 100644 --- a/icann-rdap-cli/src/bin/rdap/error.rs +++ b/icann-rdap-cli/src/bin/rdap/error.rs @@ -1,6 +1,6 @@ use std::process::{ExitCode, Termination}; -use icann_rdap_client::iana_request::IanaResponseError; +use icann_rdap_client::iana::iana_request::IanaResponseError; use icann_rdap_client::RdapClientError; use minus::MinusError; use thiserror::Error; diff --git a/icann-rdap-cli/src/dirs/fcbs.rs b/icann-rdap-cli/src/dirs/fcbs.rs index 5ba0076..cf76eb3 100644 --- a/icann-rdap-cli/src/dirs/fcbs.rs +++ b/icann-rdap-cli/src/dirs/fcbs.rs @@ -4,7 +4,7 @@ use std::{ path::PathBuf, }; -use icann_rdap_client::query::bootstrap::{BootstrapStore, RegistryHasNotExpired}; +use icann_rdap_client::iana::bootstrap::{BootstrapStore, RegistryHasNotExpired}; use icann_rdap_common::{ httpdata::HttpData, iana::{BootstrapRegistry, IanaRegistry, IanaRegistryType}, @@ -94,9 +94,9 @@ where #[cfg(test)] #[allow(non_snake_case)] mod test { - use icann_rdap_client::query::{ - bootstrap::{BootstrapStore, PreferredUrl}, - qtype::QueryType, + use icann_rdap_client::{ + iana::bootstrap::{BootstrapStore, PreferredUrl}, + query::qtype::QueryType, }; use icann_rdap_common::{ httpdata::HttpData, diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 099d92d..e32608d 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -6,11 +6,9 @@ use hickory_client::client::{AsyncClient, ClientConnection, ClientHandle}; use hickory_client::rr::{DNSClass, Name, RecordType}; use hickory_client::udp::UdpClientConnection; use icann_rdap_client::client::create_client_with_addr; +use icann_rdap_client::iana::bootstrap::{qtype_to_bootstrap_url, BootstrapStore}; use icann_rdap_client::{create_client, rdap_url_request, ClientConfig}; -use icann_rdap_client::{ - query::bootstrap::{qtype_to_bootstrap_url, BootstrapStore}, - QueryType, RdapClientError, -}; +use icann_rdap_client::{QueryType, RdapClientError}; use icann_rdap_common::response::get_related_links; use icann_rdap_common::response::types::ExtensionId; use reqwest::header::HeaderValue; diff --git a/icann-rdap-client/src/query/bootstrap.rs b/icann-rdap-client/src/iana/bootstrap.rs similarity index 98% rename from icann-rdap-client/src/query/bootstrap.rs rename to icann-rdap-client/src/iana/bootstrap.rs index 8d12b68..f6067a5 100644 --- a/icann-rdap-client/src/query/bootstrap.rs +++ b/icann-rdap-client/src/iana/bootstrap.rs @@ -11,9 +11,7 @@ use icann_rdap_common::{ }; use reqwest::Client; -use crate::{iana_request::iana_request, RdapClientError}; - -use super::qtype::QueryType; +use crate::{iana::iana_request::iana_request, QueryType, RdapClientError}; const SECONDS_IN_WEEK: i64 = 604800; @@ -154,7 +152,7 @@ impl PreferredUrl for Vec { /// /// This implementation of [BootstrapStore] keeps registries in memory. Every new instance starts with /// no registries in memory. They are added and maintained over time by calls to [MemoryBootstrapStore::put_bootstrap_registry()] by the -/// machinery of [crate::query::request::rdap_bootstrapped_request()] and [crate::query::bootstrap::qtype_to_bootstrap_url()]. +/// machinery of [crate::query::request::rdap_bootstrapped_request()] and [crate::iana::bootstrap::qtype_to_bootstrap_url()]. /// /// Ideally, this should be kept in the same scope as [reqwest::Client]. pub struct MemoryBootstrapStore { @@ -360,7 +358,7 @@ mod test { iana::{IanaRegistry, IanaRegistryType}, }; - use crate::query::{bootstrap::PreferredUrl, qtype::QueryType}; + use crate::{iana::bootstrap::PreferredUrl, QueryType}; use super::{BootstrapStore, MemoryBootstrapStore}; diff --git a/icann-rdap-client/src/iana_request.rs b/icann-rdap-client/src/iana/iana_request.rs similarity index 100% rename from icann-rdap-client/src/iana_request.rs rename to icann-rdap-client/src/iana/iana_request.rs diff --git a/icann-rdap-client/src/iana/mod.rs b/icann-rdap-client/src/iana/mod.rs new file mode 100644 index 0000000..647fe71 --- /dev/null +++ b/icann-rdap-client/src/iana/mod.rs @@ -0,0 +1,4 @@ +//! IANA and RDAP Bootstrapping + +pub mod bootstrap; +pub mod iana_request; diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index 335ef93..ff520ce 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -3,7 +3,7 @@ #![doc = include_str!("../README.md")] use std::{fmt::Display, sync::PoisonError}; -use iana_request::IanaResponseError; +use iana::iana_request::IanaResponseError; use icann_rdap_common::{ dns_types::DomainNameError, httpdata::HttpData, iana::BootstrapRegistryError, response::RdapResponseError, @@ -12,7 +12,7 @@ use thiserror::Error; pub mod client; pub mod gtld; -pub mod iana_request; +pub mod iana; pub mod md; pub mod query; pub mod registered_redactions; @@ -23,7 +23,7 @@ pub use crate::client::create_client; #[doc(inline)] pub use crate::client::ClientConfig; #[doc(inline)] -pub use crate::query::bootstrap::MemoryBootstrapStore; +pub use crate::iana::bootstrap::MemoryBootstrapStore; #[doc(inline)] pub use crate::query::qtype::QueryType; #[doc(inline)] diff --git a/icann-rdap-client/src/query/mod.rs b/icann-rdap-client/src/query/mod.rs index 53d4367..9efbb3e 100644 --- a/icann-rdap-client/src/query/mod.rs +++ b/icann-rdap-client/src/query/mod.rs @@ -1,5 +1,4 @@ //! Code for issuing RDAP queries. -pub mod bootstrap; pub mod qtype; pub mod request; diff --git a/icann-rdap-client/src/query/request.rs b/icann-rdap-client/src/query/request.rs index d2ddba0..7d501cc 100644 --- a/icann-rdap-client/src/query/request.rs +++ b/icann-rdap-client/src/query/request.rs @@ -11,13 +11,13 @@ use reqwest::{ use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::RdapClientError; - -use super::{ - bootstrap::{qtype_to_bootstrap_url, BootstrapStore}, - qtype::QueryType, +use crate::{ + iana::bootstrap::{qtype_to_bootstrap_url, BootstrapStore}, + RdapClientError, }; +use super::qtype::QueryType; + /// Makes an RDAP request with a full RDAP URL. /// /// This function takes the following parameters: @@ -179,7 +179,7 @@ pub async fn rdap_request( /// /// The [BootstrapStore] is responsible for holding IANA RDAP bootstrap registries. /// It will be populated with IANA registries as needed. Ideally, the calling code -/// would be kept it in the same scope as `client`. When using the [crate::query::bootstrap::MemoryBootstrapStore], +/// would be kept it in the same scope as `client`. When using the [crate::iana::bootstrap::MemoryBootstrapStore], /// creating a new store for each request will result it fetching the appropriate IANA /// registry with each request which is most likely not the desired behavior. /// @@ -188,7 +188,7 @@ pub async fn rdap_request( /// use icann_rdap_client::client::create_client; /// use icann_rdap_client::query::request::rdap_bootstrapped_request; /// use icann_rdap_client::query::qtype::QueryType; -/// use icann_rdap_client::query::bootstrap::MemoryBootstrapStore; +/// use icann_rdap_client::iana::bootstrap::MemoryBootstrapStore; /// use icann_rdap_client::RdapClientError; /// use std::str::FromStr; /// use tokio::main; diff --git a/icann-rdap-srv/src/bootstrap.rs b/icann-rdap-srv/src/bootstrap.rs index b9a37c1..2b2c013 100644 --- a/icann-rdap-srv/src/bootstrap.rs +++ b/icann-rdap-srv/src/bootstrap.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, time::Duration}; -use icann_rdap_client::{create_client, iana_request::iana_request, ClientConfig}; +use icann_rdap_client::{create_client, iana::iana_request::iana_request, ClientConfig}; use icann_rdap_common::{ httpdata::HttpData, iana::{IanaRegistry, IanaRegistryType}, diff --git a/icann-rdap-srv/src/error.rs b/icann-rdap-srv/src/error.rs index bdcad43..3d37380 100644 --- a/icann-rdap-srv/src/error.rs +++ b/icann-rdap-srv/src/error.rs @@ -6,7 +6,7 @@ use axum::{ }; use envmnt::errors::EnvmntError; use http::StatusCode; -use icann_rdap_client::{iana_request::IanaResponseError, RdapClientError}; +use icann_rdap_client::{iana::iana_request::IanaResponseError, RdapClientError}; use icann_rdap_common::response::{types::Common, RdapResponse, RdapResponseError}; use ipnet::PrefixLenError; use thiserror::Error; From fbb1c1162ae73b3502d48dc982e1df8cf9b7bfa2 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 12:55:57 -0500 Subject: [PATCH 03/16] moving rr and registered_redactions under rdap --- icann-rdap-cli/src/bin/rdap/bootstrap.rs | 2 +- icann-rdap-cli/src/bin/rdap/main.rs | 2 +- icann-rdap-cli/src/bin/rdap/query.rs | 4 ++-- icann-rdap-cli/src/bin/rdap/request.rs | 2 +- icann-rdap-cli/src/dirs/fcbs.rs | 2 +- icann-rdap-cli/src/rt/results.rs | 2 +- icann-rdap-cli/tests/integration/cache.rs | 2 +- icann-rdap-cli/tests/integration/source.rs | 2 +- icann-rdap-client/src/iana/bootstrap.rs | 2 +- icann-rdap-client/src/lib.rs | 12 +++++------- icann-rdap-client/src/md/domain.rs | 2 +- icann-rdap-client/src/md/entity.rs | 2 +- icann-rdap-client/src/md/mod.rs | 2 +- icann-rdap-client/src/md/table.rs | 2 +- icann-rdap-client/src/query/mod.rs | 4 ---- icann-rdap-client/src/rdap/mod.rs | 6 ++++++ icann-rdap-client/src/{query => rdap}/qtype.rs | 0 .../src/{ => rdap}/registered_redactions.rs | 0 icann-rdap-client/src/{query => rdap}/request.rs | 10 +++++----- icann-rdap-client/src/{ => rdap}/rr.rs | 2 +- icann-rdap-srv/src/bin/rdap-srv-data.rs | 2 +- icann-rdap-srv/tests/integration/srv/bootstrap.rs | 2 +- icann-rdap-srv/tests/integration/srv/domain.rs | 2 +- icann-rdap-srv/tests/integration/srv/redirect.rs | 2 +- icann-rdap-srv/tests/integration/srv/srvhelp.rs | 2 +- 25 files changed, 36 insertions(+), 36 deletions(-) delete mode 100644 icann-rdap-client/src/query/mod.rs create mode 100644 icann-rdap-client/src/rdap/mod.rs rename icann-rdap-client/src/{query => rdap}/qtype.rs (100%) rename icann-rdap-client/src/{ => rdap}/registered_redactions.rs (100%) rename icann-rdap-client/src/{query => rdap}/request.rs (96%) rename icann-rdap-client/src/{ => rdap}/rr.rs (97%) diff --git a/icann-rdap-cli/src/bin/rdap/bootstrap.rs b/icann-rdap-cli/src/bin/rdap/bootstrap.rs index f52e0de..fa97fae 100644 --- a/icann-rdap-cli/src/bin/rdap/bootstrap.rs +++ b/icann-rdap-cli/src/bin/rdap/bootstrap.rs @@ -4,7 +4,7 @@ use icann_rdap_client::iana::bootstrap::BootstrapStore; use icann_rdap_client::iana::bootstrap::PreferredUrl; use icann_rdap_client::{ iana::bootstrap::{fetch_bootstrap, qtype_to_bootstrap_url}, - query::qtype::QueryType, + rdap::qtype::QueryType, }; use icann_rdap_common::iana::IanaRegistryType; use reqwest::Client; diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index ba7381b..5cfc753 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -21,7 +21,7 @@ use write::FmtWrite; use write::PagerWrite; use clap::{ArgGroup, Parser, ValueEnum}; -use icann_rdap_client::query::qtype::QueryType; +use icann_rdap_client::rdap::qtype::QueryType; use icann_rdap_common::VERSION; use query::OutputType; use reqwest::Client; diff --git a/icann-rdap-cli/src/bin/rdap/query.rs b/icann-rdap-cli/src/bin/rdap/query.rs index 57db92b..e04be1c 100644 --- a/icann-rdap-cli/src/bin/rdap/query.rs +++ b/icann-rdap-cli/src/bin/rdap/query.rs @@ -11,8 +11,8 @@ use tracing::info; use icann_rdap_client::{ gtld::{GtldParams, ToGtldWhois}, md::{redacted::replace_redacted_items, MdOptions, MdParams, ToMd}, - query::{qtype::QueryType, request::ResponseData}, - rr::{RequestData, RequestResponse, RequestResponses, SourceType}, + rdap::rr::{RequestData, RequestResponse, RequestResponses, SourceType}, + rdap::{qtype::QueryType, request::ResponseData}, }; use reqwest::Client; use termimad::{crossterm::style::Color::*, Alignment, MadSkin}; diff --git a/icann-rdap-cli/src/bin/rdap/request.rs b/icann-rdap-cli/src/bin/rdap/request.rs index 693768d..50f6b77 100644 --- a/icann-rdap-cli/src/bin/rdap/request.rs +++ b/icann-rdap-cli/src/bin/rdap/request.rs @@ -3,7 +3,7 @@ use std::{ io::{BufRead, BufReader}, }; -use icann_rdap_client::query::{ +use icann_rdap_client::rdap::{ qtype::QueryType, request::{rdap_url_request, ResponseData}, }; diff --git a/icann-rdap-cli/src/dirs/fcbs.rs b/icann-rdap-cli/src/dirs/fcbs.rs index cf76eb3..550e2ba 100644 --- a/icann-rdap-cli/src/dirs/fcbs.rs +++ b/icann-rdap-cli/src/dirs/fcbs.rs @@ -96,7 +96,7 @@ where mod test { use icann_rdap_client::{ iana::bootstrap::{BootstrapStore, PreferredUrl}, - query::qtype::QueryType, + rdap::qtype::QueryType, }; use icann_rdap_common::{ httpdata::HttpData, diff --git a/icann-rdap-cli/src/rt/results.rs b/icann-rdap-cli/src/rt/results.rs index fba66dc..4403db7 100644 --- a/icann-rdap-cli/src/rt/results.rs +++ b/icann-rdap-cli/src/rt/results.rs @@ -4,7 +4,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use chrono::{DateTime, Utc}; use icann_rdap_client::{ md::{string::StringUtil, table::MultiPartTable, MdOptions}, - query::request::ResponseData, + rdap::request::ResponseData, RdapClientError, }; use icann_rdap_common::{ diff --git a/icann-rdap-cli/tests/integration/cache.rs b/icann-rdap-cli/tests/integration/cache.rs index 5e2a8eb..f7d8c6a 100644 --- a/icann-rdap-cli/tests/integration/cache.rs +++ b/icann-rdap-cli/tests/integration/cache.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::rr::RequestResponseOwned; +use icann_rdap_client::rdap::rr::RequestResponseOwned; use icann_rdap_common::response::{domain::Domain, entity::Entity, RdapResponse}; use icann_rdap_srv::storage::StoreOps; diff --git a/icann-rdap-cli/tests/integration/source.rs b/icann-rdap-cli/tests/integration/source.rs index dbf5677..f9466fe 100644 --- a/icann-rdap-cli/tests/integration/source.rs +++ b/icann-rdap-cli/tests/integration/source.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::rr::{RequestResponseOwned, SourceType}; +use icann_rdap_client::rdap::rr::{RequestResponseOwned, SourceType}; use icann_rdap_common::response::network::Network; use icann_rdap_srv::storage::StoreOps; use rstest::rstest; diff --git a/icann-rdap-client/src/iana/bootstrap.rs b/icann-rdap-client/src/iana/bootstrap.rs index f6067a5..ad85cba 100644 --- a/icann-rdap-client/src/iana/bootstrap.rs +++ b/icann-rdap-client/src/iana/bootstrap.rs @@ -152,7 +152,7 @@ impl PreferredUrl for Vec { /// /// This implementation of [BootstrapStore] keeps registries in memory. Every new instance starts with /// no registries in memory. They are added and maintained over time by calls to [MemoryBootstrapStore::put_bootstrap_registry()] by the -/// machinery of [crate::query::request::rdap_bootstrapped_request()] and [crate::iana::bootstrap::qtype_to_bootstrap_url()]. +/// machinery of [crate::rdap::request::rdap_bootstrapped_request()] and [crate::iana::bootstrap::qtype_to_bootstrap_url()]. /// /// Ideally, this should be kept in the same scope as [reqwest::Client]. pub struct MemoryBootstrapStore { diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index ff520ce..4966c0c 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -14,9 +14,7 @@ pub mod client; pub mod gtld; pub mod iana; pub mod md; -pub mod query; -pub mod registered_redactions; -pub mod rr; +pub mod rdap; #[doc(inline)] pub use crate::client::create_client; @@ -25,13 +23,13 @@ pub use crate::client::ClientConfig; #[doc(inline)] pub use crate::iana::bootstrap::MemoryBootstrapStore; #[doc(inline)] -pub use crate::query::qtype::QueryType; +pub use crate::rdap::qtype::QueryType; #[doc(inline)] -pub use crate::query::request::rdap_bootstrapped_request; +pub use crate::rdap::request::rdap_bootstrapped_request; #[doc(inline)] -pub use crate::query::request::rdap_request; +pub use crate::rdap::request::rdap_request; #[doc(inline)] -pub use crate::query::request::rdap_url_request; +pub use crate::rdap::request::rdap_url_request; /// Error returned by RDAP client functions and methods. #[derive(Error, Debug)] diff --git a/icann-rdap-client/src/md/domain.rs b/icann-rdap-client/src/md/domain.rs index c03801d..c4105f8 100644 --- a/icann-rdap-client/src/md/domain.rs +++ b/icann-rdap-client/src/md/domain.rs @@ -5,7 +5,7 @@ use icann_rdap_common::response::domain::{Domain, SecureDns, Variant}; use icann_rdap_common::check::{CheckParams, GetChecks, GetSubChecks}; -use crate::registered_redactions::{self, text_or_registered_redaction}; +use crate::rdap::registered_redactions::{self, text_or_registered_redaction}; use super::redacted::REDACTED_TEXT; use super::types::{events_to_table, links_to_table, public_ids_to_table}; diff --git a/icann-rdap-client/src/md/entity.rs b/icann-rdap-client/src/md/entity.rs index d2f90fe..0cf77bc 100644 --- a/icann-rdap-client/src/md/entity.rs +++ b/icann-rdap-client/src/md/entity.rs @@ -5,7 +5,7 @@ use icann_rdap_common::response::entity::{Entity, EntityRole}; use icann_rdap_common::check::{CheckParams, GetChecks, GetSubChecks}; -use crate::registered_redactions::{ +use crate::rdap::registered_redactions::{ are_redactions_registered_for_roles, is_redaction_registered_for_role, text_or_registered_redaction_for_role, RedactedName, }; diff --git a/icann-rdap-client/src/md/mod.rs b/icann-rdap-client/src/md/mod.rs index 8cb6266..8a49c42 100644 --- a/icann-rdap-client/src/md/mod.rs +++ b/icann-rdap-client/src/md/mod.rs @@ -1,6 +1,6 @@ //! Converts RDAP to Markdown. -use crate::rr::RequestData; +use crate::rdap::rr::RequestData; use buildstructor::Builder; use icann_rdap_common::{check::CheckParams, httpdata::HttpData, response::RdapResponse}; use std::{any::TypeId, char}; diff --git a/icann-rdap-client/src/md/table.rs b/icann-rdap-client/src/md/table.rs index a5b55a0..85d47cb 100644 --- a/icann-rdap-client/src/md/table.rs +++ b/icann-rdap-client/src/md/table.rs @@ -251,7 +251,7 @@ mod tests { use crate::{ md::ToMd, - rr::{RequestData, SourceType}, + rdap::rr::{RequestData, SourceType}, }; use super::MultiPartTable; diff --git a/icann-rdap-client/src/query/mod.rs b/icann-rdap-client/src/query/mod.rs deleted file mode 100644 index 9efbb3e..0000000 --- a/icann-rdap-client/src/query/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Code for issuing RDAP queries. - -pub mod qtype; -pub mod request; diff --git a/icann-rdap-client/src/rdap/mod.rs b/icann-rdap-client/src/rdap/mod.rs new file mode 100644 index 0000000..5e41bdc --- /dev/null +++ b/icann-rdap-client/src/rdap/mod.rs @@ -0,0 +1,6 @@ +//! Code for managing RDAP queries. + +pub mod qtype; +pub mod registered_redactions; +pub mod request; +pub mod rr; diff --git a/icann-rdap-client/src/query/qtype.rs b/icann-rdap-client/src/rdap/qtype.rs similarity index 100% rename from icann-rdap-client/src/query/qtype.rs rename to icann-rdap-client/src/rdap/qtype.rs diff --git a/icann-rdap-client/src/registered_redactions.rs b/icann-rdap-client/src/rdap/registered_redactions.rs similarity index 100% rename from icann-rdap-client/src/registered_redactions.rs rename to icann-rdap-client/src/rdap/registered_redactions.rs diff --git a/icann-rdap-client/src/query/request.rs b/icann-rdap-client/src/rdap/request.rs similarity index 96% rename from icann-rdap-client/src/query/request.rs rename to icann-rdap-client/src/rdap/request.rs index 7d501cc..3e33f8a 100644 --- a/icann-rdap-client/src/query/request.rs +++ b/icann-rdap-client/src/rdap/request.rs @@ -27,7 +27,7 @@ use super::qtype::QueryType; /// ```no_run /// use icann_rdap_client::client::ClientConfig; /// use icann_rdap_client::client::create_client; -/// use icann_rdap_client::query::request::rdap_url_request; +/// use icann_rdap_client::rdap::request::rdap_url_request; /// use icann_rdap_client::RdapClientError; /// use std::str::FromStr; /// use tokio::main; @@ -131,8 +131,8 @@ pub async fn rdap_url_request(url: &str, client: &Client) -> Result Date: Sat, 4 Jan 2025 14:12:39 -0500 Subject: [PATCH 04/16] uplevels the IANA API and puts it in one place. --- icann-rdap-cli/src/bin/rdap-test/error.rs | 2 +- icann-rdap-cli/src/bin/rdap/bootstrap.rs | 6 +++--- icann-rdap-cli/src/bin/rdap/error.rs | 2 +- icann-rdap-cli/src/dirs/fcbs.rs | 4 ++-- icann-rdap-cli/src/rt/exec.rs | 2 +- icann-rdap-client/README.md | 3 ++- icann-rdap-client/src/iana/bootstrap.rs | 4 ++++ icann-rdap-client/src/iana/iana_request.rs | 3 +++ icann-rdap-client/src/iana/mod.rs | 9 +++++++-- icann-rdap-client/src/lib.rs | 2 -- icann-rdap-client/src/rdap/request.rs | 2 +- icann-rdap-srv/src/bootstrap.rs | 2 +- icann-rdap-srv/src/error.rs | 2 +- 13 files changed, 27 insertions(+), 16 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/error.rs b/icann-rdap-cli/src/bin/rdap-test/error.rs index 126fc15..176b14a 100644 --- a/icann-rdap-cli/src/bin/rdap-test/error.rs +++ b/icann-rdap-cli/src/bin/rdap-test/error.rs @@ -1,7 +1,7 @@ use std::process::{ExitCode, Termination}; use icann_rdap_cli::rt::exec::TestExecutionError; -use icann_rdap_client::iana::iana_request::IanaResponseError; +use icann_rdap_client::iana::IanaResponseError; use icann_rdap_client::RdapClientError; use thiserror::Error; diff --git a/icann-rdap-cli/src/bin/rdap/bootstrap.rs b/icann-rdap-cli/src/bin/rdap/bootstrap.rs index fa97fae..f1a58c2 100644 --- a/icann-rdap-cli/src/bin/rdap/bootstrap.rs +++ b/icann-rdap-cli/src/bin/rdap/bootstrap.rs @@ -1,9 +1,9 @@ use crate::error::RdapCliError; use icann_rdap_cli::dirs::fcbs::FileCacheBootstrapStore; -use icann_rdap_client::iana::bootstrap::BootstrapStore; -use icann_rdap_client::iana::bootstrap::PreferredUrl; +use icann_rdap_client::iana::BootstrapStore; +use icann_rdap_client::iana::PreferredUrl; use icann_rdap_client::{ - iana::bootstrap::{fetch_bootstrap, qtype_to_bootstrap_url}, + iana::{fetch_bootstrap, qtype_to_bootstrap_url}, rdap::qtype::QueryType, }; use icann_rdap_common::iana::IanaRegistryType; diff --git a/icann-rdap-cli/src/bin/rdap/error.rs b/icann-rdap-cli/src/bin/rdap/error.rs index 3e3209a..371eb1a 100644 --- a/icann-rdap-cli/src/bin/rdap/error.rs +++ b/icann-rdap-cli/src/bin/rdap/error.rs @@ -1,6 +1,6 @@ use std::process::{ExitCode, Termination}; -use icann_rdap_client::iana::iana_request::IanaResponseError; +use icann_rdap_client::iana::IanaResponseError; use icann_rdap_client::RdapClientError; use minus::MinusError; use thiserror::Error; diff --git a/icann-rdap-cli/src/dirs/fcbs.rs b/icann-rdap-cli/src/dirs/fcbs.rs index 550e2ba..d1f9726 100644 --- a/icann-rdap-cli/src/dirs/fcbs.rs +++ b/icann-rdap-cli/src/dirs/fcbs.rs @@ -4,7 +4,7 @@ use std::{ path::PathBuf, }; -use icann_rdap_client::iana::bootstrap::{BootstrapStore, RegistryHasNotExpired}; +use icann_rdap_client::iana::{BootstrapStore, RegistryHasNotExpired}; use icann_rdap_common::{ httpdata::HttpData, iana::{BootstrapRegistry, IanaRegistry, IanaRegistryType}, @@ -95,7 +95,7 @@ where #[allow(non_snake_case)] mod test { use icann_rdap_client::{ - iana::bootstrap::{BootstrapStore, PreferredUrl}, + iana::{BootstrapStore, PreferredUrl}, rdap::qtype::QueryType, }; use icann_rdap_common::{ diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index e32608d..200cff9 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -6,7 +6,7 @@ use hickory_client::client::{AsyncClient, ClientConnection, ClientHandle}; use hickory_client::rr::{DNSClass, Name, RecordType}; use hickory_client::udp::UdpClientConnection; use icann_rdap_client::client::create_client_with_addr; -use icann_rdap_client::iana::bootstrap::{qtype_to_bootstrap_url, BootstrapStore}; +use icann_rdap_client::iana::{qtype_to_bootstrap_url, BootstrapStore}; use icann_rdap_client::{create_client, rdap_url_request, ClientConfig}; use icann_rdap_client::{QueryType, RdapClientError}; use icann_rdap_common::response::get_related_links; diff --git a/icann-rdap-client/README.md b/icann-rdap-client/README.md index ccb0086..9de1f36 100644 --- a/icann-rdap-client/README.md +++ b/icann-rdap-client/README.md @@ -15,7 +15,7 @@ Add the library to your Cargo.toml: `cargo add icann-rdap-client` Also, add the commons library: `cargo add icann-rdap-common`. -Both icann-rdap-common and icann-rdap-client can be compiled for WASM targets. +Both [icann_rdap_common] and this crate can be compiled for WASM targets. Usage ----- @@ -26,6 +26,7 @@ query using the IANA RDAP bootstrap files. To make a query using bootstrapping: ```rust,no_run use icann_rdap_client::*; +use icann_rdap_client::iana::MemoryBootstrapStore; use std::str::FromStr; use tokio::main; diff --git a/icann-rdap-client/src/iana/bootstrap.rs b/icann-rdap-client/src/iana/bootstrap.rs index ad85cba..16452e3 100644 --- a/icann-rdap-client/src/iana/bootstrap.rs +++ b/icann-rdap-client/src/iana/bootstrap.rs @@ -138,6 +138,7 @@ pub trait BootstrapStore: Send + Sync { fn get_tag_urls(&self, tag: &str) -> Result, RdapClientError>; } +/// A trait to find the preferred URL from a bootstrap service. pub trait PreferredUrl { fn preferred_url(self) -> Result; } @@ -267,6 +268,7 @@ impl BootstrapStore for MemoryBootstrapStore { } } +/// Trait to determine if a bootstrap registry is past its expiration (i.e. needs to be rechecked). pub trait RegistryHasNotExpired { fn registry_has_not_expired(&self) -> bool; } @@ -281,6 +283,7 @@ impl RegistryHasNotExpired for Option<(IanaRegistry, HttpData)> { } } +/// Given a [QueryType], it will get the bootstrap URL. pub async fn qtype_to_bootstrap_url( client: &Client, store: &dyn BootstrapStore, @@ -333,6 +336,7 @@ where } } +/// Fetches a bootstrap registry for a [BootstrapStore]. pub async fn fetch_bootstrap( reg_type: &IanaRegistryType, client: &Client, diff --git a/icann-rdap-client/src/iana/iana_request.rs b/icann-rdap-client/src/iana/iana_request.rs index b582127..2038013 100644 --- a/icann-rdap-client/src/iana/iana_request.rs +++ b/icann-rdap-client/src/iana/iana_request.rs @@ -14,6 +14,7 @@ use reqwest::{ use serde::{Deserialize, Serialize}; use thiserror::Error; +/// Response from getting an IANA registry. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct IanaResponse { pub registry: IanaRegistry, @@ -21,6 +22,7 @@ pub struct IanaResponse { pub http_data: HttpData, } +/// Errors from issuing a request to get an IANA registry. #[derive(Debug, Error)] pub enum IanaResponseError { #[error(transparent)] @@ -29,6 +31,7 @@ pub enum IanaResponseError { SerdeJson(#[from] serde_json::Error), } +/// Issues the HTTP request to get an IANA registry. pub async fn iana_request( registry_type: IanaRegistryType, client: &Client, diff --git a/icann-rdap-client/src/iana/mod.rs b/icann-rdap-client/src/iana/mod.rs index 647fe71..11dbe15 100644 --- a/icann-rdap-client/src/iana/mod.rs +++ b/icann-rdap-client/src/iana/mod.rs @@ -1,4 +1,9 @@ //! IANA and RDAP Bootstrapping -pub mod bootstrap; -pub mod iana_request; +#[doc(inline)] +pub use bootstrap::*; +#[doc(inline)] +pub use iana_request::*; + +pub(crate) mod bootstrap; +pub(crate) mod iana_request; diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index 4966c0c..0b0eba6 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -21,8 +21,6 @@ pub use crate::client::create_client; #[doc(inline)] pub use crate::client::ClientConfig; #[doc(inline)] -pub use crate::iana::bootstrap::MemoryBootstrapStore; -#[doc(inline)] pub use crate::rdap::qtype::QueryType; #[doc(inline)] pub use crate::rdap::request::rdap_bootstrapped_request; diff --git a/icann-rdap-client/src/rdap/request.rs b/icann-rdap-client/src/rdap/request.rs index 3e33f8a..4b4f50b 100644 --- a/icann-rdap-client/src/rdap/request.rs +++ b/icann-rdap-client/src/rdap/request.rs @@ -188,7 +188,7 @@ pub async fn rdap_request( /// use icann_rdap_client::client::create_client; /// use icann_rdap_client::rdap::request::rdap_bootstrapped_request; /// use icann_rdap_client::rdap::qtype::QueryType; -/// use icann_rdap_client::iana::bootstrap::MemoryBootstrapStore; +/// use icann_rdap_client::iana::MemoryBootstrapStore; /// use icann_rdap_client::RdapClientError; /// use std::str::FromStr; /// use tokio::main; diff --git a/icann-rdap-srv/src/bootstrap.rs b/icann-rdap-srv/src/bootstrap.rs index 2b2c013..3c1cf6e 100644 --- a/icann-rdap-srv/src/bootstrap.rs +++ b/icann-rdap-srv/src/bootstrap.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, time::Duration}; -use icann_rdap_client::{create_client, iana::iana_request::iana_request, ClientConfig}; +use icann_rdap_client::{create_client, iana::iana_request, ClientConfig}; use icann_rdap_common::{ httpdata::HttpData, iana::{IanaRegistry, IanaRegistryType}, diff --git a/icann-rdap-srv/src/error.rs b/icann-rdap-srv/src/error.rs index 3d37380..f12eb8e 100644 --- a/icann-rdap-srv/src/error.rs +++ b/icann-rdap-srv/src/error.rs @@ -6,7 +6,7 @@ use axum::{ }; use envmnt::errors::EnvmntError; use http::StatusCode; -use icann_rdap_client::{iana::iana_request::IanaResponseError, RdapClientError}; +use icann_rdap_client::{iana::IanaResponseError, RdapClientError}; use icann_rdap_common::response::{types::Common, RdapResponse, RdapResponseError}; use ipnet::PrefixLenError; use thiserror::Error; From 3ee150a815fe0141be78d35fda197d9b13ae4074 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 14:35:29 -0500 Subject: [PATCH 05/16] upleveling and centralizing the rdap request API --- icann-rdap-cli/src/bin/rdap-test/main.rs | 2 +- icann-rdap-cli/src/bin/rdap/bootstrap.rs | 2 +- icann-rdap-cli/src/bin/rdap/main.rs | 2 +- icann-rdap-cli/src/bin/rdap/query.rs | 4 ++-- icann-rdap-cli/src/bin/rdap/request.rs | 3 +-- icann-rdap-cli/src/dirs/fcbs.rs | 2 +- icann-rdap-cli/src/rt/exec.rs | 4 ++-- icann-rdap-cli/src/rt/results.rs | 2 +- icann-rdap-cli/tests/integration/cache.rs | 2 +- icann-rdap-cli/tests/integration/source.rs | 2 +- icann-rdap-client/README.md | 4 +++- icann-rdap-client/src/iana/bootstrap.rs | 4 ++-- icann-rdap-client/src/lib.rs | 8 -------- icann-rdap-client/src/rdap/mod.rs | 17 +++++++++++++---- .../src/rdap/registered_redactions.rs | 1 + icann-rdap-client/src/rdap/request.rs | 11 ++++++----- icann-rdap-client/src/rdap/rr.rs | 1 + icann-rdap-srv/src/bin/rdap-srv-data.rs | 2 +- .../tests/integration/srv/bootstrap.rs | 2 +- icann-rdap-srv/tests/integration/srv/domain.rs | 2 +- .../tests/integration/srv/redirect.rs | 2 +- icann-rdap-srv/tests/integration/srv/srvhelp.rs | 2 +- 22 files changed, 43 insertions(+), 38 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index 6e7b774..2e60413 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -13,7 +13,7 @@ use icann_rdap_cli::rt::results::RunOutcome; use icann_rdap_cli::rt::results::TestResults; use icann_rdap_client::client::ClientConfig; use icann_rdap_client::md::MdOptions; -use icann_rdap_client::QueryType; +use icann_rdap_client::rdap::QueryType; use icann_rdap_common::check::traverse_checks; use icann_rdap_common::check::CheckClass; use termimad::crossterm::style::Color::*; diff --git a/icann-rdap-cli/src/bin/rdap/bootstrap.rs b/icann-rdap-cli/src/bin/rdap/bootstrap.rs index f1a58c2..2e08a36 100644 --- a/icann-rdap-cli/src/bin/rdap/bootstrap.rs +++ b/icann-rdap-cli/src/bin/rdap/bootstrap.rs @@ -4,7 +4,7 @@ use icann_rdap_client::iana::BootstrapStore; use icann_rdap_client::iana::PreferredUrl; use icann_rdap_client::{ iana::{fetch_bootstrap, qtype_to_bootstrap_url}, - rdap::qtype::QueryType, + rdap::QueryType, }; use icann_rdap_common::iana::IanaRegistryType; use reqwest::Client; diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index 5cfc753..e2ba8ee 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -21,7 +21,7 @@ use write::FmtWrite; use write::PagerWrite; use clap::{ArgGroup, Parser, ValueEnum}; -use icann_rdap_client::rdap::qtype::QueryType; +use icann_rdap_client::rdap::QueryType; use icann_rdap_common::VERSION; use query::OutputType; use reqwest::Client; diff --git a/icann-rdap-cli/src/bin/rdap/query.rs b/icann-rdap-cli/src/bin/rdap/query.rs index e04be1c..4daeda0 100644 --- a/icann-rdap-cli/src/bin/rdap/query.rs +++ b/icann-rdap-cli/src/bin/rdap/query.rs @@ -11,8 +11,8 @@ use tracing::info; use icann_rdap_client::{ gtld::{GtldParams, ToGtldWhois}, md::{redacted::replace_redacted_items, MdOptions, MdParams, ToMd}, - rdap::rr::{RequestData, RequestResponse, RequestResponses, SourceType}, - rdap::{qtype::QueryType, request::ResponseData}, + rdap::{QueryType, ResponseData}, + rdap::{RequestData, RequestResponse, RequestResponses, SourceType}, }; use reqwest::Client; use termimad::{crossterm::style::Color::*, Alignment, MadSkin}; diff --git a/icann-rdap-cli/src/bin/rdap/request.rs b/icann-rdap-cli/src/bin/rdap/request.rs index 50f6b77..7669bbf 100644 --- a/icann-rdap-cli/src/bin/rdap/request.rs +++ b/icann-rdap-cli/src/bin/rdap/request.rs @@ -4,8 +4,7 @@ use std::{ }; use icann_rdap_client::rdap::{ - qtype::QueryType, - request::{rdap_url_request, ResponseData}, + QueryType, {rdap_url_request, ResponseData}, }; use icann_rdap_common::{httpdata::HttpData, response::GetSelfLink}; use pct_str::PctString; diff --git a/icann-rdap-cli/src/dirs/fcbs.rs b/icann-rdap-cli/src/dirs/fcbs.rs index d1f9726..037197a 100644 --- a/icann-rdap-cli/src/dirs/fcbs.rs +++ b/icann-rdap-cli/src/dirs/fcbs.rs @@ -96,7 +96,7 @@ where mod test { use icann_rdap_client::{ iana::{BootstrapStore, PreferredUrl}, - rdap::qtype::QueryType, + rdap::QueryType, }; use icann_rdap_common::{ httpdata::HttpData, diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 200cff9..7f52af9 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -7,8 +7,8 @@ use hickory_client::rr::{DNSClass, Name, RecordType}; use hickory_client::udp::UdpClientConnection; use icann_rdap_client::client::create_client_with_addr; use icann_rdap_client::iana::{qtype_to_bootstrap_url, BootstrapStore}; -use icann_rdap_client::{create_client, rdap_url_request, ClientConfig}; -use icann_rdap_client::{QueryType, RdapClientError}; +use icann_rdap_client::{create_client, rdap::rdap_url_request, ClientConfig}; +use icann_rdap_client::{rdap::QueryType, RdapClientError}; use icann_rdap_common::response::get_related_links; use icann_rdap_common::response::types::ExtensionId; use reqwest::header::HeaderValue; diff --git a/icann-rdap-cli/src/rt/results.rs b/icann-rdap-cli/src/rt/results.rs index 4403db7..854082c 100644 --- a/icann-rdap-cli/src/rt/results.rs +++ b/icann-rdap-cli/src/rt/results.rs @@ -4,7 +4,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use chrono::{DateTime, Utc}; use icann_rdap_client::{ md::{string::StringUtil, table::MultiPartTable, MdOptions}, - rdap::request::ResponseData, + rdap::ResponseData, RdapClientError, }; use icann_rdap_common::{ diff --git a/icann-rdap-cli/tests/integration/cache.rs b/icann-rdap-cli/tests/integration/cache.rs index f7d8c6a..2146d33 100644 --- a/icann-rdap-cli/tests/integration/cache.rs +++ b/icann-rdap-cli/tests/integration/cache.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::rdap::rr::RequestResponseOwned; +use icann_rdap_client::rdap::RequestResponseOwned; use icann_rdap_common::response::{domain::Domain, entity::Entity, RdapResponse}; use icann_rdap_srv::storage::StoreOps; diff --git a/icann-rdap-cli/tests/integration/source.rs b/icann-rdap-cli/tests/integration/source.rs index f9466fe..02bcbdd 100644 --- a/icann-rdap-cli/tests/integration/source.rs +++ b/icann-rdap-cli/tests/integration/source.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::rdap::rr::{RequestResponseOwned, SourceType}; +use icann_rdap_client::rdap::{RequestResponseOwned, SourceType}; use icann_rdap_common::response::network::Network; use icann_rdap_srv::storage::StoreOps; use rstest::rstest; diff --git a/icann-rdap-client/README.md b/icann-rdap-client/README.md index 9de1f36..710e98e 100644 --- a/icann-rdap-client/README.md +++ b/icann-rdap-client/README.md @@ -26,7 +26,8 @@ query using the IANA RDAP bootstrap files. To make a query using bootstrapping: ```rust,no_run use icann_rdap_client::*; -use icann_rdap_client::iana::MemoryBootstrapStore; +use icann_rdap_client::iana::*; +use icann_rdap_client::rdap::*; use std::str::FromStr; use tokio::main; @@ -61,6 +62,7 @@ To specify a base URL: ```rust,no_run use icann_rdap_client::*; +use icann_rdap_client::rdap::*; use std::str::FromStr; use tokio::main; diff --git a/icann-rdap-client/src/iana/bootstrap.rs b/icann-rdap-client/src/iana/bootstrap.rs index 16452e3..1eaf52d 100644 --- a/icann-rdap-client/src/iana/bootstrap.rs +++ b/icann-rdap-client/src/iana/bootstrap.rs @@ -11,7 +11,7 @@ use icann_rdap_common::{ }; use reqwest::Client; -use crate::{iana::iana_request::iana_request, QueryType, RdapClientError}; +use crate::{iana::iana_request::iana_request, rdap::QueryType, RdapClientError}; const SECONDS_IN_WEEK: i64 = 604800; @@ -362,7 +362,7 @@ mod test { iana::{IanaRegistry, IanaRegistryType}, }; - use crate::{iana::bootstrap::PreferredUrl, QueryType}; + use crate::{iana::bootstrap::PreferredUrl, rdap::QueryType}; use super::{BootstrapStore, MemoryBootstrapStore}; diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index 0b0eba6..2ad7294 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -20,14 +20,6 @@ pub mod rdap; pub use crate::client::create_client; #[doc(inline)] pub use crate::client::ClientConfig; -#[doc(inline)] -pub use crate::rdap::qtype::QueryType; -#[doc(inline)] -pub use crate::rdap::request::rdap_bootstrapped_request; -#[doc(inline)] -pub use crate::rdap::request::rdap_request; -#[doc(inline)] -pub use crate::rdap::request::rdap_url_request; /// Error returned by RDAP client functions and methods. #[derive(Error, Debug)] diff --git a/icann-rdap-client/src/rdap/mod.rs b/icann-rdap-client/src/rdap/mod.rs index 5e41bdc..1b393ac 100644 --- a/icann-rdap-client/src/rdap/mod.rs +++ b/icann-rdap-client/src/rdap/mod.rs @@ -1,6 +1,15 @@ //! Code for managing RDAP queries. -pub mod qtype; -pub mod registered_redactions; -pub mod request; -pub mod rr; +#[doc(inline)] +pub use qtype::*; +#[doc(inline)] +pub use registered_redactions::*; +#[doc(inline)] +pub use request::*; +#[doc(inline)] +pub use rr::*; + +pub(crate) mod qtype; +pub(crate) mod registered_redactions; +pub(crate) mod request; +pub(crate) mod rr; diff --git a/icann-rdap-client/src/rdap/registered_redactions.rs b/icann-rdap-client/src/rdap/registered_redactions.rs index 45e9c96..e093194 100644 --- a/icann-rdap-client/src/rdap/registered_redactions.rs +++ b/icann-rdap-client/src/rdap/registered_redactions.rs @@ -125,6 +125,7 @@ pub fn is_redaction_registered_for_role( false } +/// Same as [is_redaction_registered_for_role] but takes an array of [EntityRole] references. pub fn are_redactions_registered_for_roles( rdap_response: &RdapResponse, redaction_type: &[&RedactedName], diff --git a/icann-rdap-client/src/rdap/request.rs b/icann-rdap-client/src/rdap/request.rs index 4b4f50b..14026ff 100644 --- a/icann-rdap-client/src/rdap/request.rs +++ b/icann-rdap-client/src/rdap/request.rs @@ -27,7 +27,7 @@ use super::qtype::QueryType; /// ```no_run /// use icann_rdap_client::client::ClientConfig; /// use icann_rdap_client::client::create_client; -/// use icann_rdap_client::rdap::request::rdap_url_request; +/// use icann_rdap_client::rdap::rdap_url_request; /// use icann_rdap_client::RdapClientError; /// use std::str::FromStr; /// use tokio::main; @@ -131,8 +131,8 @@ pub async fn rdap_url_request(url: &str, client: &Client) -> Result { pub checks: Checks, } +/// A [Vec] of [RequestResponse]. pub type RequestResponses<'a> = Vec>; diff --git a/icann-rdap-srv/src/bin/rdap-srv-data.rs b/icann-rdap-srv/src/bin/rdap-srv-data.rs index b2667cb..b82b133 100644 --- a/icann-rdap-srv/src/bin/rdap-srv-data.rs +++ b/icann-rdap-srv/src/bin/rdap-srv-data.rs @@ -4,7 +4,7 @@ use chrono::Utc; use cidr::IpCidr; use cidr::IpInet; use clap::{Args, Parser, Subcommand}; -use icann_rdap_client::rdap::qtype::QueryType; +use icann_rdap_client::rdap::QueryType; use icann_rdap_common::contact::Contact; use icann_rdap_common::contact::PostalAddress; use icann_rdap_common::media_types::RDAP_MEDIA_TYPE; diff --git a/icann-rdap-srv/tests/integration/srv/bootstrap.rs b/icann-rdap-srv/tests/integration/srv/bootstrap.rs index e153f4e..f1efbe7 100644 --- a/icann-rdap-srv/tests/integration/srv/bootstrap.rs +++ b/icann-rdap-srv/tests/integration/srv/bootstrap.rs @@ -1,7 +1,7 @@ #![allow(non_snake_case)] use icann_rdap_client::client::{create_client, ClientConfig}; -use icann_rdap_client::rdap::{qtype::QueryType, request::rdap_request}; +use icann_rdap_client::rdap::{rdap_request, QueryType}; use icann_rdap_srv::storage::{ data::{AutnumId, DomainId, EntityId, NetworkId, NetworkIdType}, StoreOps, diff --git a/icann-rdap-srv/tests/integration/srv/domain.rs b/icann-rdap-srv/tests/integration/srv/domain.rs index bd505f6..26537fe 100644 --- a/icann-rdap-srv/tests/integration/srv/domain.rs +++ b/icann-rdap-srv/tests/integration/srv/domain.rs @@ -2,7 +2,7 @@ use icann_rdap_client::{ create_client, - rdap::{qtype::QueryType, request::rdap_request}, + rdap::{rdap_request, QueryType}, ClientConfig, RdapClientError, }; use icann_rdap_common::response::domain::Domain; diff --git a/icann-rdap-srv/tests/integration/srv/redirect.rs b/icann-rdap-srv/tests/integration/srv/redirect.rs index 13c5b62..383c065 100644 --- a/icann-rdap-srv/tests/integration/srv/redirect.rs +++ b/icann-rdap-srv/tests/integration/srv/redirect.rs @@ -2,7 +2,7 @@ use icann_rdap_client::{ create_client, - rdap::{qtype::QueryType, request::rdap_request}, + rdap::{rdap_request, QueryType}, ClientConfig, }; use icann_rdap_common::response::{ diff --git a/icann-rdap-srv/tests/integration/srv/srvhelp.rs b/icann-rdap-srv/tests/integration/srv/srvhelp.rs index 2359d37..2998ef7 100644 --- a/icann-rdap-srv/tests/integration/srv/srvhelp.rs +++ b/icann-rdap-srv/tests/integration/srv/srvhelp.rs @@ -2,7 +2,7 @@ use icann_rdap_client::{ create_client, - rdap::{qtype::QueryType, request::rdap_request}, + rdap::{rdap_request, QueryType}, ClientConfig, }; use icann_rdap_common::response::{ From e32312c21d37b4571130ce1a29573df93d6538b7 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 15:00:56 -0500 Subject: [PATCH 06/16] created client prelude --- icann-rdap-client/README.md | 7 ++----- icann-rdap-client/src/lib.rs | 20 ++++++++++++++++++++ icann-rdap-client/src/rdap/request.rs | 18 +++--------------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/icann-rdap-client/README.md b/icann-rdap-client/README.md index 710e98e..575ce19 100644 --- a/icann-rdap-client/README.md +++ b/icann-rdap-client/README.md @@ -25,9 +25,7 @@ is the process of finding the authoritative RDAP server to query using the IANA RDAP bootstrap files. To make a query using bootstrapping: ```rust,no_run -use icann_rdap_client::*; -use icann_rdap_client::iana::*; -use icann_rdap_client::rdap::*; +use icann_rdap_client::prelude::*; use std::str::FromStr; use tokio::main; @@ -61,8 +59,7 @@ async fn main() -> Result<(), RdapClientError> { To specify a base URL: ```rust,no_run -use icann_rdap_client::*; -use icann_rdap_client::rdap::*; +use icann_rdap_client::prelude::*; use std::str::FromStr; use tokio::main; diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index 2ad7294..b70e42e 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -16,6 +16,26 @@ pub mod iana; pub mod md; pub mod rdap; +/// Basics necesasry for a simple clients. +pub mod prelude { + #[doc(inline)] + pub use crate::client::create_client; + #[doc(inline)] + pub use crate::client::ClientConfig; + #[doc(inline)] + pub use crate::iana::MemoryBootstrapStore; + #[doc(inline)] + pub use crate::rdap::rdap_bootstrapped_request; + #[doc(inline)] + pub use crate::rdap::rdap_request; + #[doc(inline)] + pub use crate::rdap::rdap_url_request; + #[doc(inline)] + pub use crate::rdap::QueryType; + #[doc(inline)] + pub use crate::RdapClientError; +} + #[doc(inline)] pub use crate::client::create_client; #[doc(inline)] diff --git a/icann-rdap-client/src/rdap/request.rs b/icann-rdap-client/src/rdap/request.rs index 14026ff..14bda37 100644 --- a/icann-rdap-client/src/rdap/request.rs +++ b/icann-rdap-client/src/rdap/request.rs @@ -25,10 +25,7 @@ use super::qtype::QueryType; /// * client - a reference to a [reqwest::Client]. /// /// ```no_run -/// use icann_rdap_client::client::ClientConfig; -/// use icann_rdap_client::client::create_client; -/// use icann_rdap_client::rdap::rdap_url_request; -/// use icann_rdap_client::RdapClientError; +/// use icann_rdap_client::prelude::*; /// use std::str::FromStr; /// use tokio::main; /// @@ -129,11 +126,7 @@ pub async fn rdap_url_request(url: &str, client: &Client) -> Result Date: Sat, 4 Jan 2025 16:08:47 -0500 Subject: [PATCH 07/16] moved client to http module --- icann-rdap-cli/src/bin/rdap-test/main.rs | 2 +- icann-rdap-cli/src/bin/rdap/main.rs | 4 ++-- icann-rdap-cli/src/rt/exec.rs | 4 ++-- icann-rdap-client/src/{ => http}/client.rs | 0 icann-rdap-client/src/http/mod.rs | 6 ++++++ icann-rdap-client/src/lib.rs | 11 +++-------- icann-rdap-srv/src/bootstrap.rs | 2 +- icann-rdap-srv/tests/integration/srv/bootstrap.rs | 2 +- icann-rdap-srv/tests/integration/srv/domain.rs | 5 +++-- icann-rdap-srv/tests/integration/srv/redirect.rs | 4 ++-- icann-rdap-srv/tests/integration/srv/srvhelp.rs | 4 ++-- 11 files changed, 23 insertions(+), 21 deletions(-) rename icann-rdap-client/src/{ => http}/client.rs (100%) create mode 100644 icann-rdap-client/src/http/mod.rs diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index 2e60413..a567b50 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -11,7 +11,7 @@ use icann_rdap_cli::rt::exec::ExtensionGroup; use icann_rdap_cli::rt::exec::TestOptions; use icann_rdap_cli::rt::results::RunOutcome; use icann_rdap_cli::rt::results::TestResults; -use icann_rdap_client::client::ClientConfig; +use icann_rdap_client::http::ClientConfig; use icann_rdap_client::md::MdOptions; use icann_rdap_client::rdap::QueryType; use icann_rdap_common::check::traverse_checks; diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index e2ba8ee..f4085a5 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -3,8 +3,8 @@ use clap::builder::styling::AnsiColor; use clap::builder::Styles; use error::RdapCliError; use icann_rdap_cli::dirs; -use icann_rdap_client::client::create_client; -use icann_rdap_client::client::ClientConfig; +use icann_rdap_client::http::create_client; +use icann_rdap_client::http::ClientConfig; use icann_rdap_common::check::CheckClass; use query::InrBackupBootstrap; use query::ProcessType; diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 7f52af9..3bd7ddd 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -5,9 +5,9 @@ use std::str::FromStr; use hickory_client::client::{AsyncClient, ClientConnection, ClientHandle}; use hickory_client::rr::{DNSClass, Name, RecordType}; use hickory_client::udp::UdpClientConnection; -use icann_rdap_client::client::create_client_with_addr; +use icann_rdap_client::http::create_client_with_addr; use icann_rdap_client::iana::{qtype_to_bootstrap_url, BootstrapStore}; -use icann_rdap_client::{create_client, rdap::rdap_url_request, ClientConfig}; +use icann_rdap_client::{http::create_client, http::ClientConfig, rdap::rdap_url_request}; use icann_rdap_client::{rdap::QueryType, RdapClientError}; use icann_rdap_common::response::get_related_links; use icann_rdap_common::response::types::ExtensionId; diff --git a/icann-rdap-client/src/client.rs b/icann-rdap-client/src/http/client.rs similarity index 100% rename from icann-rdap-client/src/client.rs rename to icann-rdap-client/src/http/client.rs diff --git a/icann-rdap-client/src/http/mod.rs b/icann-rdap-client/src/http/mod.rs new file mode 100644 index 0000000..d8e86b7 --- /dev/null +++ b/icann-rdap-client/src/http/mod.rs @@ -0,0 +1,6 @@ +//! The HTTP layer of RDAP. + +#[doc(inline)] +pub use client::*; + +pub(crate) mod client; diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index b70e42e..4e0fa4c 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -10,8 +10,8 @@ use icann_rdap_common::{ }; use thiserror::Error; -pub mod client; pub mod gtld; +pub mod http; pub mod iana; pub mod md; pub mod rdap; @@ -19,9 +19,9 @@ pub mod rdap; /// Basics necesasry for a simple clients. pub mod prelude { #[doc(inline)] - pub use crate::client::create_client; + pub use crate::http::create_client; #[doc(inline)] - pub use crate::client::ClientConfig; + pub use crate::http::ClientConfig; #[doc(inline)] pub use crate::iana::MemoryBootstrapStore; #[doc(inline)] @@ -36,11 +36,6 @@ pub mod prelude { pub use crate::RdapClientError; } -#[doc(inline)] -pub use crate::client::create_client; -#[doc(inline)] -pub use crate::client::ClientConfig; - /// Error returned by RDAP client functions and methods. #[derive(Error, Debug)] pub enum RdapClientError { diff --git a/icann-rdap-srv/src/bootstrap.rs b/icann-rdap-srv/src/bootstrap.rs index 3c1cf6e..58da054 100644 --- a/icann-rdap-srv/src/bootstrap.rs +++ b/icann-rdap-srv/src/bootstrap.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, time::Duration}; -use icann_rdap_client::{create_client, iana::iana_request, ClientConfig}; +use icann_rdap_client::{http::create_client, http::ClientConfig, iana::iana_request}; use icann_rdap_common::{ httpdata::HttpData, iana::{IanaRegistry, IanaRegistryType}, diff --git a/icann-rdap-srv/tests/integration/srv/bootstrap.rs b/icann-rdap-srv/tests/integration/srv/bootstrap.rs index f1efbe7..0b25d89 100644 --- a/icann-rdap-srv/tests/integration/srv/bootstrap.rs +++ b/icann-rdap-srv/tests/integration/srv/bootstrap.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::client::{create_client, ClientConfig}; +use icann_rdap_client::http::{create_client, ClientConfig}; use icann_rdap_client::rdap::{rdap_request, QueryType}; use icann_rdap_srv::storage::{ data::{AutnumId, DomainId, EntityId, NetworkId, NetworkIdType}, diff --git a/icann-rdap-srv/tests/integration/srv/domain.rs b/icann-rdap-srv/tests/integration/srv/domain.rs index 26537fe..b77a94f 100644 --- a/icann-rdap-srv/tests/integration/srv/domain.rs +++ b/icann-rdap-srv/tests/integration/srv/domain.rs @@ -1,9 +1,10 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - create_client, + http::create_client, + http::ClientConfig, rdap::{rdap_request, QueryType}, - ClientConfig, RdapClientError, + RdapClientError, }; use icann_rdap_common::response::domain::Domain; use icann_rdap_srv::storage::{CommonConfig, StoreOps}; diff --git a/icann-rdap-srv/tests/integration/srv/redirect.rs b/icann-rdap-srv/tests/integration/srv/redirect.rs index 383c065..d62f93e 100644 --- a/icann-rdap-srv/tests/integration/srv/redirect.rs +++ b/icann-rdap-srv/tests/integration/srv/redirect.rs @@ -1,9 +1,9 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - create_client, + http::create_client, + http::ClientConfig, rdap::{rdap_request, QueryType}, - ClientConfig, }; use icann_rdap_common::response::{ error::Error, diff --git a/icann-rdap-srv/tests/integration/srv/srvhelp.rs b/icann-rdap-srv/tests/integration/srv/srvhelp.rs index 2998ef7..504cdf5 100644 --- a/icann-rdap-srv/tests/integration/srv/srvhelp.rs +++ b/icann-rdap-srv/tests/integration/srv/srvhelp.rs @@ -1,9 +1,9 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - create_client, + http::create_client, + http::ClientConfig, rdap::{rdap_request, QueryType}, - ClientConfig, }; use icann_rdap_common::response::{ help::Help, From 7d37e9fa64f9a0d2329c85638dea7562a45c89a5 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 17:18:36 -0500 Subject: [PATCH 08/16] renamed client to reqwestclient, etc... wrapped client is setup --- icann-rdap-cli/src/bin/rdap-test/main.rs | 4 +- icann-rdap-cli/src/bin/rdap/main.rs | 8 +- icann-rdap-cli/src/rt/exec.rs | 28 ++-- icann-rdap-client/README.md | 8 +- icann-rdap-client/src/http/mod.rs | 7 +- .../src/http/{client.rs => reqwest.rs} | 29 ++-- icann-rdap-client/src/http/wrapped.rs | 148 ++++++++++++++++++ icann-rdap-client/src/lib.rs | 4 +- icann-rdap-client/src/rdap/request.rs | 12 +- icann-rdap-srv/src/bootstrap.rs | 8 +- .../tests/integration/srv/bootstrap.rs | 46 +++--- .../tests/integration/srv/domain.rs | 20 +-- .../tests/integration/srv/redirect.rs | 28 ++-- .../tests/integration/srv/srvhelp.rs | 12 +- 14 files changed, 260 insertions(+), 102 deletions(-) rename icann-rdap-client/src/http/{client.rs => reqwest.rs} (89%) create mode 100644 icann-rdap-client/src/http/wrapped.rs diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index a567b50..ad8dfad 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -11,7 +11,7 @@ use icann_rdap_cli::rt::exec::ExtensionGroup; use icann_rdap_cli::rt::exec::TestOptions; use icann_rdap_cli::rt::results::RunOutcome; use icann_rdap_cli::rt::results::TestResults; -use icann_rdap_client::http::ClientConfig; +use icann_rdap_client::http::ReqwestClientConfig; use icann_rdap_client::md::MdOptions; use icann_rdap_client::rdap::QueryType; use icann_rdap_common::check::traverse_checks; @@ -369,7 +369,7 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> { allow_unregistered_extensions: cli.allow_unregistered_extensions, }; - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .user_agent_suffix("RT") .https_only(!cli.allow_http) .accept_invalid_host_names(cli.allow_invalid_host_names) diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index f4085a5..8c66c67 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -3,8 +3,8 @@ use clap::builder::styling::AnsiColor; use clap::builder::Styles; use error::RdapCliError; use icann_rdap_cli::dirs; -use icann_rdap_client::http::create_client; -use icann_rdap_client::http::ClientConfig; +use icann_rdap_client::http::create_reqwest_client; +use icann_rdap_client::http::ReqwestClientConfig; use icann_rdap_common::check::CheckClass; use query::InrBackupBootstrap; use query::ProcessType; @@ -554,13 +554,13 @@ pub async fn wrapped_main() -> Result<(), RdapCliError> { max_cache_age: cli.max_cache_age, }; - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .user_agent_suffix("CLI") .https_only(!cli.allow_http) .accept_invalid_host_names(cli.allow_invalid_host_names) .accept_invalid_certificates(cli.allow_invalid_certificates) .build(); - let rdap_client = create_client(&client_config); + let rdap_client = create_reqwest_client(&client_config); if let Ok(client) = rdap_client { if !use_pager { tracing_subscriber::fmt() diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 3bd7ddd..80c4548 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -5,9 +5,11 @@ use std::str::FromStr; use hickory_client::client::{AsyncClient, ClientConnection, ClientHandle}; use hickory_client::rr::{DNSClass, Name, RecordType}; use hickory_client::udp::UdpClientConnection; -use icann_rdap_client::http::create_client_with_addr; +use icann_rdap_client::http::create_reqwest_client_with_addr; use icann_rdap_client::iana::{qtype_to_bootstrap_url, BootstrapStore}; -use icann_rdap_client::{http::create_client, http::ClientConfig, rdap::rdap_url_request}; +use icann_rdap_client::{ + http::create_reqwest_client, http::ReqwestClientConfig, rdap::rdap_url_request, +}; use icann_rdap_client::{rdap::QueryType, RdapClientError}; use icann_rdap_common::response::get_related_links; use icann_rdap_common::response::types::ExtensionId; @@ -70,9 +72,9 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( bs: &BS, value: &QueryType, options: &TestOptions, - client_config: &ClientConfig, + client_config: &ReqwestClientConfig, ) -> Result { - let bs_client = create_client(client_config)?; + let bs_client = create_reqwest_client(client_config)?; // normalize extensions let extensions = normalize_extension_ids(options)?; @@ -97,7 +99,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( }; // if they URL to test is a referral if options.chase_referral { - let client = create_client(client_config)?; + let client = create_reqwest_client(client_config)?; let response_data = rdap_url_request(&query_url, &client).await?; query_url = get_related_links(&response_data.rdap) .first() @@ -126,7 +128,8 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run without origin let mut test_run = TestRun::new_v4(vec![], v4, port); if !options.skip_v4 { - let client = create_client_with_addr(client_config, host, test_run.socket_addr)?; + let client = + create_reqwest_client_with_addr(client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } @@ -135,10 +138,11 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run with origin let mut test_run = TestRun::new_v4(vec![RunFeature::OriginHeader], v4, port); if !options.skip_v4 && !options.skip_origin { - let client_config = ClientConfig::from_config(client_config) + let client_config = ReqwestClientConfig::from_config(client_config) .origin(HeaderValue::from_str(&options.origin_value)?) .build(); - let client = create_client_with_addr(&client_config, host, test_run.socket_addr)?; + let client = + create_reqwest_client_with_addr(&client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } @@ -148,7 +152,8 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run without origin let mut test_run = TestRun::new_v6(vec![], v6, port); if !options.skip_v6 { - let client = create_client_with_addr(client_config, host, test_run.socket_addr)?; + let client = + create_reqwest_client_with_addr(client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } @@ -157,10 +162,11 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run with origin let mut test_run = TestRun::new_v6(vec![RunFeature::OriginHeader], v6, port); if !options.skip_v6 { - let client_config = ClientConfig::from_config(client_config) + let client_config = ReqwestClientConfig::from_config(client_config) .origin(HeaderValue::from_str(&options.origin_value)?) .build(); - let client = create_client_with_addr(&client_config, host, test_run.socket_addr)?; + let client = + create_reqwest_client_with_addr(&client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } diff --git a/icann-rdap-client/README.md b/icann-rdap-client/README.md index 575ce19..5965d6a 100644 --- a/icann-rdap-client/README.md +++ b/icann-rdap-client/README.md @@ -38,8 +38,8 @@ async fn main() -> Result<(), RdapClientError> { let query = QueryType::from_str("icann.org")?; // create a client (from icann-rdap-common) - let config = ClientConfig::default(); - let client = create_client(&config)?; + let config = ReqwestClientConfig::default(); + let client = create_reqwest_client(&config)?; // ideally, keep store in same context as client let store = MemoryBootstrapStore::new(); @@ -72,8 +72,8 @@ async fn main() -> Result<(), RdapClientError> { let query = QueryType::from_str("icann.org")?; // create a client (from icann-rdap-common) - let config = ClientConfig::default(); - let client = create_client(&config)?; + let config = ReqwestClientConfig::default(); + let client = create_reqwest_client(&config)?; // issue the RDAP query let base_url = "https://rdap-bootstrap.arin.net/bootstrap"; diff --git a/icann-rdap-client/src/http/mod.rs b/icann-rdap-client/src/http/mod.rs index d8e86b7..9925631 100644 --- a/icann-rdap-client/src/http/mod.rs +++ b/icann-rdap-client/src/http/mod.rs @@ -1,6 +1,9 @@ //! The HTTP layer of RDAP. #[doc(inline)] -pub use client::*; +pub use reqwest::*; +#[doc(inline)] +pub use wrapped::*; -pub(crate) mod client; +pub(crate) mod reqwest; +pub(crate) mod wrapped; diff --git a/icann-rdap-client/src/http/client.rs b/icann-rdap-client/src/http/reqwest.rs similarity index 89% rename from icann-rdap-client/src/http/client.rs rename to icann-rdap-client/src/http/reqwest.rs index 76d0b95..bd13c10 100644 --- a/icann-rdap-client/src/http/client.rs +++ b/icann-rdap-client/src/http/reqwest.rs @@ -1,10 +1,9 @@ //! Creates a Reqwest client. use lazy_static::lazy_static; -use reqwest::{ - header::{self, HeaderValue}, - Client, -}; +pub use reqwest::header::{self, HeaderValue}; +pub use reqwest::Client as ReqwestClient; +pub use reqwest::Error as ReqwestError; use icann_rdap_common::media_types::{JSON_MEDIA_TYPE, RDAP_MEDIA_TYPE}; @@ -16,7 +15,7 @@ lazy_static! { } /// Configures the HTTP client. -pub struct ClientConfig { +pub struct ReqwestClientConfig { /// This string is appended to the user agent. It is provided so /// library users may identify their programs. pub user_agent_suffix: String, @@ -42,9 +41,9 @@ pub struct ClientConfig { pub origin: Option, } -impl Default for ClientConfig { +impl Default for ReqwestClientConfig { fn default() -> Self { - ClientConfig { + ReqwestClientConfig { user_agent_suffix: "library".to_string(), https_only: true, accept_invalid_host_names: false, @@ -57,7 +56,7 @@ impl Default for ClientConfig { } #[buildstructor::buildstructor] -impl ClientConfig { +impl ReqwestClientConfig { #[builder] pub fn new( user_agent_suffix: Option, @@ -68,7 +67,7 @@ impl ClientConfig { host: Option, origin: Option, ) -> Self { - let default = ClientConfig::default(); + let default = ReqwestClientConfig::default(); Self { user_agent_suffix: user_agent_suffix.unwrap_or(default.user_agent_suffix), https_only: https_only.unwrap_or(default.https_only), @@ -113,7 +112,7 @@ impl ClientConfig { /// uses cases creating only one client per process is /// necessary. #[cfg(not(target_arch = "wasm32"))] -pub fn create_client(config: &ClientConfig) -> Result { +pub fn create_reqwest_client(config: &ReqwestClientConfig) -> Result { let default_headers = default_headers(config); let mut client = reqwest::Client::builder(); @@ -142,11 +141,11 @@ pub fn create_client(config: &ClientConfig) -> Result { /// uses cases creating only one client per process is /// necessary. #[cfg(not(target_arch = "wasm32"))] -pub fn create_client_with_addr( - config: &ClientConfig, +pub fn create_reqwest_client_with_addr( + config: &ReqwestClientConfig, domain: &str, addr: SocketAddr, -) -> Result { +) -> Result { let default_headers = default_headers(config); let mut client = reqwest::Client::builder(); @@ -178,7 +177,7 @@ pub fn create_client_with_addr( /// Note that the WASM version does not set redirect policy, /// https_only, or TLS settings. #[cfg(target_arch = "wasm32")] -pub fn create_client(config: &ClientConfig) -> Result { +pub fn create_reqwest_client(config: &ReqwestClientConfig) -> Result { let default_headers = default_headers(config); let client = reqwest::Client::builder(); @@ -187,7 +186,7 @@ pub fn create_client(config: &ClientConfig) -> Result { Ok(client) } -fn default_headers(config: &ClientConfig) -> header::HeaderMap { +fn default_headers(config: &ReqwestClientConfig) -> header::HeaderMap { let mut default_headers = header::HeaderMap::new(); default_headers.insert( header::ACCEPT, diff --git a/icann-rdap-client/src/http/wrapped.rs b/icann-rdap-client/src/http/wrapped.rs new file mode 100644 index 0000000..45b8974 --- /dev/null +++ b/icann-rdap-client/src/http/wrapped.rs @@ -0,0 +1,148 @@ +//! Wrapped Client. + +pub use reqwest::header::HeaderValue; +pub use reqwest::Client as ReqwestClient; + +use super::create_reqwest_client; +#[cfg(not(target_arch = "wasm32"))] +use super::create_reqwest_client_with_addr; +use super::ReqwestClientConfig; +use crate::RdapClientError; +#[cfg(not(target_arch = "wasm32"))] +use std::net::SocketAddr; + +/// Used by the request functions. +#[derive(Clone, Copy)] +pub struct RequestOptions { + pub retry_seconds: u16, + pub max_retries: u16, +} + +impl Default for RequestOptions { + fn default() -> Self { + Self { + retry_seconds: 120, + max_retries: 2, + } + } +} + +/// Configures the HTTP client. +#[derive(Default)] +pub struct ClientConfig { + /// Config for the Reqwest client. + client_config: ReqwestClientConfig, + + /// Request options. + request_options: RequestOptions, +} + +#[buildstructor::buildstructor] +impl ClientConfig { + #[builder] + #[allow(clippy::too_many_arguments)] + pub fn new( + user_agent_suffix: Option, + https_only: Option, + accept_invalid_host_names: Option, + accept_invalid_certificates: Option, + follow_redirects: Option, + host: Option, + origin: Option, + retry_seconds: Option, + max_retries: Option, + ) -> Self { + let default_cc = ReqwestClientConfig::default(); + let default_ro = RequestOptions::default(); + Self { + client_config: ReqwestClientConfig { + user_agent_suffix: user_agent_suffix.unwrap_or(default_cc.user_agent_suffix), + https_only: https_only.unwrap_or(default_cc.https_only), + accept_invalid_host_names: accept_invalid_host_names + .unwrap_or(default_cc.accept_invalid_host_names), + accept_invalid_certificates: accept_invalid_certificates + .unwrap_or(default_cc.accept_invalid_certificates), + follow_redirects: follow_redirects.unwrap_or(default_cc.follow_redirects), + host, + origin, + }, + request_options: RequestOptions { + retry_seconds: retry_seconds.unwrap_or(default_ro.retry_seconds), + max_retries: max_retries.unwrap_or(default_ro.max_retries), + }, + } + } + + #[builder(entry = "from_config", exit = "build")] + #[allow(clippy::too_many_arguments)] + pub fn new_from_config( + &self, + user_agent_suffix: Option, + https_only: Option, + accept_invalid_host_names: Option, + accept_invalid_certificates: Option, + follow_redirects: Option, + host: Option, + origin: Option, + retry_seconds: Option, + max_retries: Option, + ) -> Self { + Self { + client_config: ReqwestClientConfig { + user_agent_suffix: user_agent_suffix + .unwrap_or(self.client_config.user_agent_suffix.clone()), + https_only: https_only.unwrap_or(self.client_config.https_only), + accept_invalid_host_names: accept_invalid_host_names + .unwrap_or(self.client_config.accept_invalid_host_names), + accept_invalid_certificates: accept_invalid_certificates + .unwrap_or(self.client_config.accept_invalid_certificates), + follow_redirects: follow_redirects.unwrap_or(self.client_config.follow_redirects), + host: host.map_or(self.client_config.host.clone(), Some), + origin: origin.map_or(self.client_config.origin.clone(), Some), + }, + request_options: RequestOptions { + retry_seconds: retry_seconds.unwrap_or(self.request_options.retry_seconds), + max_retries: max_retries.unwrap_or(self.request_options.max_retries), + }, + } + } +} + +/// A wrapper around Reqwest client to give additional features when used with the request functions. +pub struct Client { + /// The reqwest client. + pub(crate) reqwest_client: ReqwestClient, + + /// Request options. + pub(crate) request_options: RequestOptions, +} + +impl Client { + pub fn new(reqwest_client: ReqwestClient, request_options: RequestOptions) -> Self { + Self { + reqwest_client, + request_options, + } + } +} + +/// Creates a wrapped HTTP client. The wrapped +/// client holds its own connection pools, so in many +/// uses cases creating only one client per process is +/// necessary. +pub fn create_client(config: &ClientConfig) -> Result { + let client = create_reqwest_client(&config.client_config)?; + Ok(Client::new(client, config.request_options)) +} + +/// Creates a wrapped HTTP client. +/// This will direct the underlying client to connect to a specific socket. +#[cfg(not(target_arch = "wasm32"))] +pub fn create_client_with_addr( + config: &ClientConfig, + domain: &str, + addr: SocketAddr, +) -> Result { + let client = create_reqwest_client_with_addr(&config.client_config, domain, addr)?; + Ok(Client::new(client, config.request_options)) +} diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index 4e0fa4c..23c16c9 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -19,9 +19,9 @@ pub mod rdap; /// Basics necesasry for a simple clients. pub mod prelude { #[doc(inline)] - pub use crate::http::create_client; + pub use crate::http::create_reqwest_client; #[doc(inline)] - pub use crate::http::ClientConfig; + pub use crate::http::ReqwestClientConfig; #[doc(inline)] pub use crate::iana::MemoryBootstrapStore; #[doc(inline)] diff --git a/icann-rdap-client/src/rdap/request.rs b/icann-rdap-client/src/rdap/request.rs index 14bda37..8fff4e9 100644 --- a/icann-rdap-client/src/rdap/request.rs +++ b/icann-rdap-client/src/rdap/request.rs @@ -33,8 +33,8 @@ use super::qtype::QueryType; /// async fn main() -> Result<(), RdapClientError> { /// /// // create a client (from icann-rdap-common) -/// let config = ClientConfig::default(); -/// let client = create_client(&config)?; +/// let config = ReqwestClientConfig::default(); +/// let client = create_reqwest_client(&config)?; /// /// // issue the RDAP query /// let response = @@ -139,8 +139,8 @@ pub async fn rdap_url_request(url: &str, client: &Client) -> Result Result<(), RdapServerError> { if config.bootstrap { info!("Initializing IANA Bootstrap."); - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .user_agent_suffix("icann-rdap-srv") .build(); - let client = create_client(&client_config)?; + let client = create_reqwest_client(&client_config)?; // do one run of the bootstrapping before starting the thread. process_bootstrap(config, &client).await?; diff --git a/icann-rdap-srv/tests/integration/srv/bootstrap.rs b/icann-rdap-srv/tests/integration/srv/bootstrap.rs index 0b25d89..f261186 100644 --- a/icann-rdap-srv/tests/integration/srv/bootstrap.rs +++ b/icann-rdap-srv/tests/integration/srv/bootstrap.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::http::{create_client, ClientConfig}; +use icann_rdap_client::http::{create_reqwest_client, ReqwestClientConfig}; use icann_rdap_client::rdap::{rdap_request, QueryType}; use icann_rdap_srv::storage::{ data::{AutnumId, DomainId, EntityId, NetworkId, NetworkIdType}, @@ -26,11 +26,11 @@ async fn GIVEN_bootstrap_with_less_specific_domain_WHEN_query_domain_THEN_status tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -65,11 +65,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_domain_WHEN_query_domain_THEN_sho tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -93,11 +93,11 @@ async fn GIVEN_bootstrap_with_less_specific_ns_WHEN_query_ns_THEN_status_code_is tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ns("ns.foo.example").expect("invalid nameserver"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -132,11 +132,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_ns_WHEN_query_ns_THEN_should_pani tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ns("ns.foo.example").expect("invalid nameserver"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -164,11 +164,11 @@ async fn GIVEN_bootstrap_with_less_specific_ip_WHEN_query_ip_THEN_status_code_is tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ipv4cidr("10.0.0.0/24").expect("invalid CIDR"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -207,11 +207,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_ip_WHEN_query_ip_THEN_should_pani tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ipv4cidr("11.0.0.0/24").expect("invalid CIDR"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -239,11 +239,11 @@ async fn GIVEN_bootstrap_with_less_specific_autnum_WHEN_query_autnum_THEN_status tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::autnum("AS710").expect("invalid autnum"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -281,11 +281,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_autnum_WHEN_query_autnum_THEN_sho tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::autnum("AS1000").expect("invalid autnum"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -309,11 +309,11 @@ async fn GIVEN_bootstrap_with_specific_tag_WHEN_query_entity_THEN_status_code_is tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo-ARIN".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -348,11 +348,11 @@ async fn GIVEN_bootstrap_with_specific_tag_lowercase_WHEN_query_entity_THEN_stat tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo-arin".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -387,11 +387,11 @@ async fn GIVEN_bootstrap_with_no_specific_tag_WHEN_query_entity_THEN_should_pani tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo-arin".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; diff --git a/icann-rdap-srv/tests/integration/srv/domain.rs b/icann-rdap-srv/tests/integration/srv/domain.rs index b77a94f..139a73c 100644 --- a/icann-rdap-srv/tests/integration/srv/domain.rs +++ b/icann-rdap-srv/tests/integration/srv/domain.rs @@ -1,8 +1,8 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - http::create_client, - http::ClientConfig, + http::create_reqwest_client, + http::ReqwestClientConfig, rdap::{rdap_request, QueryType}, RdapClientError, }; @@ -22,11 +22,11 @@ async fn GIVEN_server_with_domain_WHEN_query_domain_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -52,11 +52,11 @@ async fn GIVEN_server_with_idn_WHEN_query_domain_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::domain("café.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -80,11 +80,11 @@ async fn GIVEN_server_with_domain_and_search_disabled_WHEN_query_domain_THEN_sta tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::DomainNameSearch("foo.*".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -109,11 +109,11 @@ async fn GIVEN_server_with_domain_and_search_enabled_WHEN_query_domain_THEN_stat tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::DomainNameSearch("foo.*".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await diff --git a/icann-rdap-srv/tests/integration/srv/redirect.rs b/icann-rdap-srv/tests/integration/srv/redirect.rs index d62f93e..3bb83aa 100644 --- a/icann-rdap-srv/tests/integration/srv/redirect.rs +++ b/icann-rdap-srv/tests/integration/srv/redirect.rs @@ -1,8 +1,8 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - http::create_client, - http::ClientConfig, + http::create_reqwest_client, + http::ReqwestClientConfig, rdap::{rdap_request, QueryType}, }; use icann_rdap_common::response::{ @@ -44,11 +44,11 @@ async fn GIVEN_domain_error_with_first_link_href_WHEN_query_THEN_status_code_is_ tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -94,11 +94,11 @@ async fn GIVEN_nameserver_error_with_first_link_href_WHEN_query_THEN_status_code tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ns("ns.foo.example").expect("invalid nameserver"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -143,11 +143,11 @@ async fn GIVEN_entity_error_with_first_link_href_WHEN_query_THEN_status_code_is_ tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -193,11 +193,11 @@ async fn GIVEN_autnum_error_with_first_link_href_WHEN_query_THEN_status_code_is_ tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::autnum("700").expect("invalid autnum"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -242,11 +242,11 @@ async fn GIVEN_network_cidr_error_with_first_link_href_WHEN_query_THEN_status_co tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ipv4("10.0.0.1").expect("invalid IP address"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -294,11 +294,11 @@ async fn GIVEN_network_addrs_error_with_first_link_href_WHEN_query_THEN_status_c tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::ipv4("10.0.0.1").expect("invalid IP address"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await diff --git a/icann-rdap-srv/tests/integration/srv/srvhelp.rs b/icann-rdap-srv/tests/integration/srv/srvhelp.rs index 504cdf5..33196da 100644 --- a/icann-rdap-srv/tests/integration/srv/srvhelp.rs +++ b/icann-rdap-srv/tests/integration/srv/srvhelp.rs @@ -1,8 +1,8 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - http::create_client, - http::ClientConfig, + http::create_reqwest_client, + http::ReqwestClientConfig, rdap::{rdap_request, QueryType}, }; use icann_rdap_common::response::{ @@ -32,11 +32,11 @@ async fn GIVEN_server_with_default_help_WHEN_query_help_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::Help; let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -65,12 +65,12 @@ async fn GIVEN_server_with_host_help_WHEN_query_help_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ClientConfig::builder() + let client_config = ReqwestClientConfig::builder() .https_only(false) .follow_redirects(false) .host(reqwest::header::HeaderValue::from_static("foo.example.com")) .build(); - let client = create_client(&client_config).expect("creating client"); + let client = create_reqwest_client(&client_config).expect("creating client"); let query = QueryType::Help; let response = rdap_request(&test_srv.rdap_base, &query, &client) .await From 844506377429e217c31d05540a7d41616c46416f Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 4 Jan 2025 20:50:59 -0500 Subject: [PATCH 09/16] wrapped client now in place of reqwest client --- icann-rdap-cli/src/bin/rdap-test/main.rs | 4 +- icann-rdap-cli/src/bin/rdap/bootstrap.rs | 2 +- icann-rdap-cli/src/bin/rdap/main.rs | 10 +-- icann-rdap-cli/src/bin/rdap/query.rs | 2 +- icann-rdap-cli/src/bin/rdap/request.rs | 6 +- icann-rdap-cli/src/rt/exec.rs | 28 +++---- icann-rdap-client/README.md | 13 +++- icann-rdap-client/src/http/wrapped.rs | 75 +++++++++++++++++++ icann-rdap-client/src/iana/bootstrap.rs | 3 +- icann-rdap-client/src/iana/iana_request.rs | 65 ++-------------- icann-rdap-client/src/lib.rs | 4 +- icann-rdap-client/src/rdap/request.rs | 75 +++---------------- icann-rdap-srv/src/bootstrap.rs | 8 +- .../tests/integration/srv/bootstrap.rs | 46 ++++++------ .../tests/integration/srv/domain.rs | 20 ++--- .../tests/integration/srv/redirect.rs | 27 ++++--- .../tests/integration/srv/srvhelp.rs | 11 ++- 17 files changed, 184 insertions(+), 215 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index ad8dfad..a567b50 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -11,7 +11,7 @@ use icann_rdap_cli::rt::exec::ExtensionGroup; use icann_rdap_cli::rt::exec::TestOptions; use icann_rdap_cli::rt::results::RunOutcome; use icann_rdap_cli::rt::results::TestResults; -use icann_rdap_client::http::ReqwestClientConfig; +use icann_rdap_client::http::ClientConfig; use icann_rdap_client::md::MdOptions; use icann_rdap_client::rdap::QueryType; use icann_rdap_common::check::traverse_checks; @@ -369,7 +369,7 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> { allow_unregistered_extensions: cli.allow_unregistered_extensions, }; - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .user_agent_suffix("RT") .https_only(!cli.allow_http) .accept_invalid_host_names(cli.allow_invalid_host_names) diff --git a/icann-rdap-cli/src/bin/rdap/bootstrap.rs b/icann-rdap-cli/src/bin/rdap/bootstrap.rs index 2e08a36..3ac0c66 100644 --- a/icann-rdap-cli/src/bin/rdap/bootstrap.rs +++ b/icann-rdap-cli/src/bin/rdap/bootstrap.rs @@ -1,5 +1,6 @@ use crate::error::RdapCliError; use icann_rdap_cli::dirs::fcbs::FileCacheBootstrapStore; +use icann_rdap_client::http::Client; use icann_rdap_client::iana::BootstrapStore; use icann_rdap_client::iana::PreferredUrl; use icann_rdap_client::{ @@ -7,7 +8,6 @@ use icann_rdap_client::{ rdap::QueryType, }; use icann_rdap_common::iana::IanaRegistryType; -use reqwest::Client; use tracing::debug; /// Defines the type of bootstrapping to use. diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index 8c66c67..68ffe58 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -3,8 +3,9 @@ use clap::builder::styling::AnsiColor; use clap::builder::Styles; use error::RdapCliError; use icann_rdap_cli::dirs; -use icann_rdap_client::http::create_reqwest_client; -use icann_rdap_client::http::ReqwestClientConfig; +use icann_rdap_client::http::create_client; +use icann_rdap_client::http::Client; +use icann_rdap_client::http::ClientConfig; use icann_rdap_common::check::CheckClass; use query::InrBackupBootstrap; use query::ProcessType; @@ -24,7 +25,6 @@ use clap::{ArgGroup, Parser, ValueEnum}; use icann_rdap_client::rdap::QueryType; use icann_rdap_common::VERSION; use query::OutputType; -use reqwest::Client; use tokio::{join, task::spawn_blocking}; use crate::query::do_query; @@ -554,13 +554,13 @@ pub async fn wrapped_main() -> Result<(), RdapCliError> { max_cache_age: cli.max_cache_age, }; - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .user_agent_suffix("CLI") .https_only(!cli.allow_http) .accept_invalid_host_names(cli.allow_invalid_host_names) .accept_invalid_certificates(cli.allow_invalid_certificates) .build(); - let rdap_client = create_reqwest_client(&client_config); + let rdap_client = create_client(&client_config); if let Ok(client) = rdap_client { if !use_pager { tracing_subscriber::fmt() diff --git a/icann-rdap-cli/src/bin/rdap/query.rs b/icann-rdap-cli/src/bin/rdap/query.rs index 4daeda0..83adfd4 100644 --- a/icann-rdap-cli/src/bin/rdap/query.rs +++ b/icann-rdap-cli/src/bin/rdap/query.rs @@ -1,3 +1,4 @@ +use icann_rdap_client::http::Client; use icann_rdap_common::check::traverse_checks; use icann_rdap_common::check::CheckClass; use icann_rdap_common::check::CheckParams; @@ -14,7 +15,6 @@ use icann_rdap_client::{ rdap::{QueryType, ResponseData}, rdap::{RequestData, RequestResponse, RequestResponses, SourceType}, }; -use reqwest::Client; use termimad::{crossterm::style::Color::*, Alignment, MadSkin}; use crate::bootstrap::get_base_url; diff --git a/icann-rdap-cli/src/bin/rdap/request.rs b/icann-rdap-cli/src/bin/rdap/request.rs index 7669bbf..83dbf42 100644 --- a/icann-rdap-cli/src/bin/rdap/request.rs +++ b/icann-rdap-cli/src/bin/rdap/request.rs @@ -3,13 +3,13 @@ use std::{ io::{BufRead, BufReader}, }; -use icann_rdap_client::rdap::{ - QueryType, {rdap_url_request, ResponseData}, +use icann_rdap_client::{ + http::Client, + rdap::{rdap_url_request, QueryType, ResponseData}, }; use icann_rdap_common::{httpdata::HttpData, response::GetSelfLink}; use pct_str::PctString; use pct_str::URIReserved; -use reqwest::Client; use tracing::{debug, info}; use crate::{dirs::rdap_cache_path, error::RdapCliError, query::ProcessingParams}; diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 80c4548..3bd7ddd 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -5,11 +5,9 @@ use std::str::FromStr; use hickory_client::client::{AsyncClient, ClientConnection, ClientHandle}; use hickory_client::rr::{DNSClass, Name, RecordType}; use hickory_client::udp::UdpClientConnection; -use icann_rdap_client::http::create_reqwest_client_with_addr; +use icann_rdap_client::http::create_client_with_addr; use icann_rdap_client::iana::{qtype_to_bootstrap_url, BootstrapStore}; -use icann_rdap_client::{ - http::create_reqwest_client, http::ReqwestClientConfig, rdap::rdap_url_request, -}; +use icann_rdap_client::{http::create_client, http::ClientConfig, rdap::rdap_url_request}; use icann_rdap_client::{rdap::QueryType, RdapClientError}; use icann_rdap_common::response::get_related_links; use icann_rdap_common::response::types::ExtensionId; @@ -72,9 +70,9 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( bs: &BS, value: &QueryType, options: &TestOptions, - client_config: &ReqwestClientConfig, + client_config: &ClientConfig, ) -> Result { - let bs_client = create_reqwest_client(client_config)?; + let bs_client = create_client(client_config)?; // normalize extensions let extensions = normalize_extension_ids(options)?; @@ -99,7 +97,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( }; // if they URL to test is a referral if options.chase_referral { - let client = create_reqwest_client(client_config)?; + let client = create_client(client_config)?; let response_data = rdap_url_request(&query_url, &client).await?; query_url = get_related_links(&response_data.rdap) .first() @@ -128,8 +126,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run without origin let mut test_run = TestRun::new_v4(vec![], v4, port); if !options.skip_v4 { - let client = - create_reqwest_client_with_addr(client_config, host, test_run.socket_addr)?; + let client = create_client_with_addr(client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } @@ -138,11 +135,10 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run with origin let mut test_run = TestRun::new_v4(vec![RunFeature::OriginHeader], v4, port); if !options.skip_v4 && !options.skip_origin { - let client_config = ReqwestClientConfig::from_config(client_config) + let client_config = ClientConfig::from_config(client_config) .origin(HeaderValue::from_str(&options.origin_value)?) .build(); - let client = - create_reqwest_client_with_addr(&client_config, host, test_run.socket_addr)?; + let client = create_client_with_addr(&client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } @@ -152,8 +148,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run without origin let mut test_run = TestRun::new_v6(vec![], v6, port); if !options.skip_v6 { - let client = - create_reqwest_client_with_addr(client_config, host, test_run.socket_addr)?; + let client = create_client_with_addr(client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } @@ -162,11 +157,10 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run with origin let mut test_run = TestRun::new_v6(vec![RunFeature::OriginHeader], v6, port); if !options.skip_v6 { - let client_config = ReqwestClientConfig::from_config(client_config) + let client_config = ClientConfig::from_config(client_config) .origin(HeaderValue::from_str(&options.origin_value)?) .build(); - let client = - create_reqwest_client_with_addr(&client_config, host, test_run.socket_addr)?; + let client = create_client_with_addr(&client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); } diff --git a/icann-rdap-client/README.md b/icann-rdap-client/README.md index 5965d6a..1c52fe1 100644 --- a/icann-rdap-client/README.md +++ b/icann-rdap-client/README.md @@ -38,8 +38,11 @@ async fn main() -> Result<(), RdapClientError> { let query = QueryType::from_str("icann.org")?; // create a client (from icann-rdap-common) - let config = ReqwestClientConfig::default(); - let client = create_reqwest_client(&config)?; + let config = ClientConfig::default(); + // or let config = ClientConfig::builder().build(); + + let client = create_client(&config)?; + // ideally, keep store in same context as client let store = MemoryBootstrapStore::new(); @@ -72,8 +75,10 @@ async fn main() -> Result<(), RdapClientError> { let query = QueryType::from_str("icann.org")?; // create a client (from icann-rdap-common) - let config = ReqwestClientConfig::default(); - let client = create_reqwest_client(&config)?; + let config = ClientConfig::builder().build(); + // or let config = ClientConfig::default(); + + let client = create_client(&config)?; // issue the RDAP query let base_url = "https://rdap-bootstrap.arin.net/bootstrap"; diff --git a/icann-rdap-client/src/http/wrapped.rs b/icann-rdap-client/src/http/wrapped.rs index 45b8974..d3c2b0d 100644 --- a/icann-rdap-client/src/http/wrapped.rs +++ b/icann-rdap-client/src/http/wrapped.rs @@ -1,7 +1,13 @@ //! Wrapped Client. +use icann_rdap_common::httpdata::HttpData; pub use reqwest::header::HeaderValue; +use reqwest::header::{ + ACCESS_CONTROL_ALLOW_ORIGIN, CACHE_CONTROL, CONTENT_TYPE, EXPIRES, LOCATION, RETRY_AFTER, + STRICT_TRANSPORT_SECURITY, +}; pub use reqwest::Client as ReqwestClient; +pub use reqwest::Error as ReqwestError; use super::create_reqwest_client; #[cfg(not(target_arch = "wasm32"))] @@ -146,3 +152,72 @@ pub fn create_client_with_addr( let client = create_reqwest_client_with_addr(&config.client_config, domain, addr)?; Ok(Client::new(client, config.request_options)) } + +pub(crate) struct WrappedResponse { + pub(crate) http_data: HttpData, + pub(crate) text: String, +} + +pub(crate) async fn wrapped_request( + url: &str, + client: &Client, +) -> Result { + let response = client + .reqwest_client + .get(url) + .send() + .await? + .error_for_status()?; + let content_type = response + .headers() + .get(CONTENT_TYPE) + .map(|value| value.to_str().unwrap().to_string()); + let expires = response + .headers() + .get(EXPIRES) + .map(|value| value.to_str().unwrap().to_string()); + let cache_control = response + .headers() + .get(CACHE_CONTROL) + .map(|value| value.to_str().unwrap().to_string()); + let location = response + .headers() + .get(LOCATION) + .map(|value| value.to_str().unwrap().to_string()); + let access_control_allow_origin = response + .headers() + .get(ACCESS_CONTROL_ALLOW_ORIGIN) + .map(|value| value.to_str().unwrap().to_string()); + let strict_transport_security = response + .headers() + .get(STRICT_TRANSPORT_SECURITY) + .map(|value| value.to_str().unwrap().to_string()); + let retry_after = response + .headers() + .get(RETRY_AFTER) + .map(|value| value.to_str().unwrap().to_string()); + let content_length = response.content_length(); + let status_code = response.status().as_u16(); + let url = response.url().to_owned(); + let text = response.text().await?; + + let http_data = HttpData::now() + .status_code(status_code) + .and_location(location) + .and_content_length(content_length) + .and_content_type(content_type) + .scheme(url.scheme()) + .host( + url.host_str() + .expect("URL has no host. This shouldn't happen.") + .to_owned(), + ) + .and_expires(expires) + .and_cache_control(cache_control) + .and_access_control_allow_origin(access_control_allow_origin) + .and_strict_transport_security(strict_transport_security) + .and_retry_after(retry_after) + .build(); + + Ok(WrappedResponse { http_data, text }) +} diff --git a/icann-rdap-client/src/iana/bootstrap.rs b/icann-rdap-client/src/iana/bootstrap.rs index 1eaf52d..49762fe 100644 --- a/icann-rdap-client/src/iana/bootstrap.rs +++ b/icann-rdap-client/src/iana/bootstrap.rs @@ -9,9 +9,8 @@ use icann_rdap_common::{ IanaRegistryType, }, }; -use reqwest::Client; -use crate::{iana::iana_request::iana_request, rdap::QueryType, RdapClientError}; +use crate::{http::Client, iana::iana_request::iana_request, rdap::QueryType, RdapClientError}; const SECONDS_IN_WEEK: i64 = 604800; diff --git a/icann-rdap-client/src/iana/iana_request.rs b/icann-rdap-client/src/iana/iana_request.rs index 2038013..12b1ef7 100644 --- a/icann-rdap-client/src/iana/iana_request.rs +++ b/icann-rdap-client/src/iana/iana_request.rs @@ -4,16 +4,12 @@ use icann_rdap_common::httpdata::HttpData; use icann_rdap_common::iana::IanaRegistry; use icann_rdap_common::iana::IanaRegistryType; use icann_rdap_common::iana::RdapBootstrapRegistry; -use reqwest::header::ACCESS_CONTROL_ALLOW_ORIGIN; -use reqwest::header::RETRY_AFTER; -use reqwest::header::STRICT_TRANSPORT_SECURITY; -use reqwest::{ - header::{CACHE_CONTROL, CONTENT_TYPE, EXPIRES, LOCATION}, - Client, -}; use serde::{Deserialize, Serialize}; use thiserror::Error; +use crate::http::wrapped_request; +use crate::http::Client; + /// Response from getting an IANA registry. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct IanaResponse { @@ -37,57 +33,12 @@ pub async fn iana_request( client: &Client, ) -> Result { let url = registry_type.url(); - let response = client.get(url).send().await?.error_for_status()?; - let content_type = response - .headers() - .get(CONTENT_TYPE) - .map(|value| value.to_str().unwrap().to_string()); - let expires = response - .headers() - .get(EXPIRES) - .map(|value| value.to_str().unwrap().to_string()); - let cache_control = response - .headers() - .get(CACHE_CONTROL) - .map(|value| value.to_str().unwrap().to_string()); - let location = response - .headers() - .get(LOCATION) - .map(|value| value.to_str().unwrap().to_string()); - let access_control_allow_origin = response - .headers() - .get(ACCESS_CONTROL_ALLOW_ORIGIN) - .map(|value| value.to_str().unwrap().to_string()); - let strict_transport_security = response - .headers() - .get(STRICT_TRANSPORT_SECURITY) - .map(|value| value.to_str().unwrap().to_string()); - let retry_after = response - .headers() - .get(RETRY_AFTER) - .map(|value| value.to_str().unwrap().to_string()); - let status_code = response.status().as_u16(); - let content_length = response.content_length(); - let url = response.url().to_owned(); - let text = response.text().await?; + + let wrapped_response = wrapped_request(url, client).await?; + let text = wrapped_response.text; + let http_data = wrapped_response.http_data; + let json: RdapBootstrapRegistry = serde_json::from_str(&text)?; - let http_data = HttpData::now() - .scheme(url.scheme()) - .host( - url.host_str() - .expect("URL has no host. This shouldn't happen.") - .to_owned(), - ) - .status_code(status_code) - .and_location(location) - .and_content_length(content_length) - .and_content_type(content_type) - .and_expires(expires) - .and_cache_control(cache_control) - .and_access_control_allow_origin(access_control_allow_origin) - .and_strict_transport_security(strict_transport_security) - .and_retry_after(retry_after) - .build(); Ok(IanaResponse { registry: IanaRegistry::RdapBootstrapRegistry(json), registry_type, diff --git a/icann-rdap-client/src/lib.rs b/icann-rdap-client/src/lib.rs index 23c16c9..4e0fa4c 100644 --- a/icann-rdap-client/src/lib.rs +++ b/icann-rdap-client/src/lib.rs @@ -19,9 +19,9 @@ pub mod rdap; /// Basics necesasry for a simple clients. pub mod prelude { #[doc(inline)] - pub use crate::http::create_reqwest_client; + pub use crate::http::create_client; #[doc(inline)] - pub use crate::http::ReqwestClientConfig; + pub use crate::http::ClientConfig; #[doc(inline)] pub use crate::iana::MemoryBootstrapStore; #[doc(inline)] diff --git a/icann-rdap-client/src/rdap/request.rs b/icann-rdap-client/src/rdap/request.rs index 8fff4e9..8acb6ae 100644 --- a/icann-rdap-client/src/rdap/request.rs +++ b/icann-rdap-client/src/rdap/request.rs @@ -1,17 +1,11 @@ //! Functions to make RDAP requests. use icann_rdap_common::{httpdata::HttpData, iana::IanaRegistryType, response::RdapResponse}; -use reqwest::{ - header::{ - ACCESS_CONTROL_ALLOW_ORIGIN, CACHE_CONTROL, CONTENT_TYPE, EXPIRES, LOCATION, RETRY_AFTER, - STRICT_TRANSPORT_SECURITY, - }, - Client, -}; use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::{ + http::{wrapped_request, Client}, iana::bootstrap::{qtype_to_bootstrap_url, BootstrapStore}, RdapClientError, }; @@ -33,8 +27,8 @@ use super::qtype::QueryType; /// async fn main() -> Result<(), RdapClientError> { /// /// // create a client (from icann-rdap-common) -/// let config = ReqwestClientConfig::default(); -/// let client = create_reqwest_client(&config)?; +/// let config = ClientConfig::default(); +/// let client = create_client(&config)?; /// /// // issue the RDAP query /// let response = @@ -47,57 +41,10 @@ use super::qtype::QueryType; /// } /// ``` pub async fn rdap_url_request(url: &str, client: &Client) -> Result { - let response = client.get(url).send().await?.error_for_status()?; - let content_type = response - .headers() - .get(CONTENT_TYPE) - .map(|value| value.to_str().unwrap().to_string()); - let expires = response - .headers() - .get(EXPIRES) - .map(|value| value.to_str().unwrap().to_string()); - let cache_control = response - .headers() - .get(CACHE_CONTROL) - .map(|value| value.to_str().unwrap().to_string()); - let location = response - .headers() - .get(LOCATION) - .map(|value| value.to_str().unwrap().to_string()); - let access_control_allow_origin = response - .headers() - .get(ACCESS_CONTROL_ALLOW_ORIGIN) - .map(|value| value.to_str().unwrap().to_string()); - let strict_transport_security = response - .headers() - .get(STRICT_TRANSPORT_SECURITY) - .map(|value| value.to_str().unwrap().to_string()); - let retry_after = response - .headers() - .get(RETRY_AFTER) - .map(|value| value.to_str().unwrap().to_string()); - let content_length = response.content_length(); - let status_code = response.status().as_u16(); - let url = response.url().to_owned(); - let text = response.text().await?; - - let http_data = HttpData::now() - .status_code(status_code) - .and_location(location) - .and_content_length(content_length) - .and_content_type(content_type) - .scheme(url.scheme()) - .host( - url.host_str() - .expect("URL has no host. This shouldn't happen.") - .to_owned(), - ) - .and_expires(expires) - .and_cache_control(cache_control) - .and_access_control_allow_origin(access_control_allow_origin) - .and_strict_transport_security(strict_transport_security) - .and_retry_after(retry_after) - .build(); + let wrapped_response = wrapped_request(url, client).await?; + // for convenience purposes + let text = wrapped_response.text; + let http_data = wrapped_response.http_data; let json: Result = serde_json::from_str(&text); if let Ok(rdap_json) = json { @@ -139,8 +86,8 @@ pub async fn rdap_url_request(url: &str, client: &Client) -> Result Result<(), RdapServerError> { if config.bootstrap { info!("Initializing IANA Bootstrap."); - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .user_agent_suffix("icann-rdap-srv") .build(); - let client = create_reqwest_client(&client_config)?; + let client = create_client(&client_config)?; // do one run of the bootstrapping before starting the thread. process_bootstrap(config, &client).await?; diff --git a/icann-rdap-srv/tests/integration/srv/bootstrap.rs b/icann-rdap-srv/tests/integration/srv/bootstrap.rs index f261186..0b25d89 100644 --- a/icann-rdap-srv/tests/integration/srv/bootstrap.rs +++ b/icann-rdap-srv/tests/integration/srv/bootstrap.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use icann_rdap_client::http::{create_reqwest_client, ReqwestClientConfig}; +use icann_rdap_client::http::{create_client, ClientConfig}; use icann_rdap_client::rdap::{rdap_request, QueryType}; use icann_rdap_srv::storage::{ data::{AutnumId, DomainId, EntityId, NetworkId, NetworkIdType}, @@ -26,11 +26,11 @@ async fn GIVEN_bootstrap_with_less_specific_domain_WHEN_query_domain_THEN_status tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -65,11 +65,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_domain_WHEN_query_domain_THEN_sho tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -93,11 +93,11 @@ async fn GIVEN_bootstrap_with_less_specific_ns_WHEN_query_ns_THEN_status_code_is tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ns("ns.foo.example").expect("invalid nameserver"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -132,11 +132,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_ns_WHEN_query_ns_THEN_should_pani tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ns("ns.foo.example").expect("invalid nameserver"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -164,11 +164,11 @@ async fn GIVEN_bootstrap_with_less_specific_ip_WHEN_query_ip_THEN_status_code_is tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ipv4cidr("10.0.0.0/24").expect("invalid CIDR"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -207,11 +207,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_ip_WHEN_query_ip_THEN_should_pani tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ipv4cidr("11.0.0.0/24").expect("invalid CIDR"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -239,11 +239,11 @@ async fn GIVEN_bootstrap_with_less_specific_autnum_WHEN_query_autnum_THEN_status tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::autnum("AS710").expect("invalid autnum"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -281,11 +281,11 @@ async fn GIVEN_bootstrap_with_no_less_specific_autnum_WHEN_query_autnum_THEN_sho tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::autnum("AS1000").expect("invalid autnum"); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -309,11 +309,11 @@ async fn GIVEN_bootstrap_with_specific_tag_WHEN_query_entity_THEN_status_code_is tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo-ARIN".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -348,11 +348,11 @@ async fn GIVEN_bootstrap_with_specific_tag_lowercase_WHEN_query_entity_THEN_stat tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo-arin".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -387,11 +387,11 @@ async fn GIVEN_bootstrap_with_no_specific_tag_WHEN_query_entity_THEN_should_pani tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo-arin".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; diff --git a/icann-rdap-srv/tests/integration/srv/domain.rs b/icann-rdap-srv/tests/integration/srv/domain.rs index 139a73c..b77a94f 100644 --- a/icann-rdap-srv/tests/integration/srv/domain.rs +++ b/icann-rdap-srv/tests/integration/srv/domain.rs @@ -1,8 +1,8 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - http::create_reqwest_client, - http::ReqwestClientConfig, + http::create_client, + http::ClientConfig, rdap::{rdap_request, QueryType}, RdapClientError, }; @@ -22,11 +22,11 @@ async fn GIVEN_server_with_domain_WHEN_query_domain_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -52,11 +52,11 @@ async fn GIVEN_server_with_idn_WHEN_query_domain_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::domain("café.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -80,11 +80,11 @@ async fn GIVEN_server_with_domain_and_search_disabled_WHEN_query_domain_THEN_sta tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::DomainNameSearch("foo.*".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client).await; @@ -109,11 +109,11 @@ async fn GIVEN_server_with_domain_and_search_enabled_WHEN_query_domain_THEN_stat tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::DomainNameSearch("foo.*".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await diff --git a/icann-rdap-srv/tests/integration/srv/redirect.rs b/icann-rdap-srv/tests/integration/srv/redirect.rs index 3bb83aa..9e5e645 100644 --- a/icann-rdap-srv/tests/integration/srv/redirect.rs +++ b/icann-rdap-srv/tests/integration/srv/redirect.rs @@ -1,8 +1,7 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - http::create_reqwest_client, - http::ReqwestClientConfig, + http::{create_client, ClientConfig}, rdap::{rdap_request, QueryType}, }; use icann_rdap_common::response::{ @@ -44,11 +43,11 @@ async fn GIVEN_domain_error_with_first_link_href_WHEN_query_THEN_status_code_is_ tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::domain("foo.example").expect("invalid domain name"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -94,11 +93,11 @@ async fn GIVEN_nameserver_error_with_first_link_href_WHEN_query_THEN_status_code tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ns("ns.foo.example").expect("invalid nameserver"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -143,11 +142,11 @@ async fn GIVEN_entity_error_with_first_link_href_WHEN_query_THEN_status_code_is_ tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::Entity("foo".to_string()); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -193,11 +192,11 @@ async fn GIVEN_autnum_error_with_first_link_href_WHEN_query_THEN_status_code_is_ tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::autnum("700").expect("invalid autnum"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -242,11 +241,11 @@ async fn GIVEN_network_cidr_error_with_first_link_href_WHEN_query_THEN_status_co tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ipv4("10.0.0.1").expect("invalid IP address"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -294,11 +293,11 @@ async fn GIVEN_network_addrs_error_with_first_link_href_WHEN_query_THEN_status_c tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::ipv4("10.0.0.1").expect("invalid IP address"); let response = rdap_request(&test_srv.rdap_base, &query, &client) .await diff --git a/icann-rdap-srv/tests/integration/srv/srvhelp.rs b/icann-rdap-srv/tests/integration/srv/srvhelp.rs index 33196da..4d4b81b 100644 --- a/icann-rdap-srv/tests/integration/srv/srvhelp.rs +++ b/icann-rdap-srv/tests/integration/srv/srvhelp.rs @@ -1,8 +1,7 @@ #![allow(non_snake_case)] use icann_rdap_client::{ - http::create_reqwest_client, - http::ReqwestClientConfig, + http::{create_client, ClientConfig}, rdap::{rdap_request, QueryType}, }; use icann_rdap_common::response::{ @@ -32,11 +31,11 @@ async fn GIVEN_server_with_default_help_WHEN_query_help_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::Help; let response = rdap_request(&test_srv.rdap_base, &query, &client) .await @@ -65,12 +64,12 @@ async fn GIVEN_server_with_host_help_WHEN_query_help_THEN_status_code_200() { tx.commit().await.expect("tx commit"); // WHEN - let client_config = ReqwestClientConfig::builder() + let client_config = ClientConfig::builder() .https_only(false) .follow_redirects(false) .host(reqwest::header::HeaderValue::from_static("foo.example.com")) .build(); - let client = create_reqwest_client(&client_config).expect("creating client"); + let client = create_client(&client_config).expect("creating client"); let query = QueryType::Help; let response = rdap_request(&test_srv.rdap_base, &query, &client) .await From 30a830b828411c61455b7f464da132fdbb4753dc Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sun, 5 Jan 2025 16:03:22 -0500 Subject: [PATCH 10/16] honoring retry-after --- Cargo.lock | 1 + icann-rdap-client/Cargo.toml | 3 + icann-rdap-client/src/http/wrapped.rs | 123 ++++++++++++++++++++------ 3 files changed, 102 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa91f30..8cff5fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1485,6 +1485,7 @@ dependencies = [ "strum_macros", "thiserror", "tokio", + "tracing", ] [[package]] diff --git a/icann-rdap-client/Cargo.toml b/icann-rdap-client/Cargo.toml index a886a2a..371e924 100644 --- a/icann-rdap-client/Cargo.toml +++ b/icann-rdap-client/Cargo.toml @@ -29,7 +29,10 @@ serde_json.workspace = true strum.workspace = true strum_macros.workspace = true thiserror.workspace = true +tracing.workspace = true +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio.workspace = true [dev-dependencies] diff --git a/icann-rdap-client/src/http/wrapped.rs b/icann-rdap-client/src/http/wrapped.rs index d3c2b0d..a413671 100644 --- a/icann-rdap-client/src/http/wrapped.rs +++ b/icann-rdap-client/src/http/wrapped.rs @@ -10,25 +10,29 @@ pub use reqwest::Client as ReqwestClient; pub use reqwest::Error as ReqwestError; use super::create_reqwest_client; -#[cfg(not(target_arch = "wasm32"))] -use super::create_reqwest_client_with_addr; use super::ReqwestClientConfig; use crate::RdapClientError; + #[cfg(not(target_arch = "wasm32"))] -use std::net::SocketAddr; +use { + super::create_reqwest_client_with_addr, chrono::DateTime, chrono::Utc, reqwest::StatusCode, + std::net::SocketAddr, tracing::debug, tracing::info, +}; /// Used by the request functions. #[derive(Clone, Copy)] pub struct RequestOptions { - pub retry_seconds: u16, - pub max_retries: u16, + pub(crate) max_retry_seconds: u32, + pub(crate) default_retry_seconds: u32, + pub(crate) max_retries: u16, } impl Default for RequestOptions { fn default() -> Self { Self { - retry_seconds: 120, - max_retries: 2, + max_retry_seconds: 120, + default_retry_seconds: 120, + max_retries: 1, } } } @@ -55,8 +59,10 @@ impl ClientConfig { follow_redirects: Option, host: Option, origin: Option, - retry_seconds: Option, - max_retries: Option, + // TODO uncomment after deciding to expose this in the API. + // max_retry_seconds: Option, + // default_retry_seconds: Option, + // max_retries: Option, ) -> Self { let default_cc = ReqwestClientConfig::default(); let default_ro = RequestOptions::default(); @@ -72,10 +78,14 @@ impl ClientConfig { host, origin, }, - request_options: RequestOptions { - retry_seconds: retry_seconds.unwrap_or(default_ro.retry_seconds), - max_retries: max_retries.unwrap_or(default_ro.max_retries), - }, + request_options: default_ro, + // TODO uncomment after deciding to expose this in the API. + // request_options: RequestOptions { + // max_retry_seconds: max_retry_seconds.unwrap_or(default_ro.max_retry_seconds), + // default_retry_seconds: default_retry_seconds + // .unwrap_or(default_ro.default_retry_seconds), + // max_retries: max_retries.unwrap_or(default_ro.max_retries), + // }, } } @@ -90,8 +100,10 @@ impl ClientConfig { follow_redirects: Option, host: Option, origin: Option, - retry_seconds: Option, - max_retries: Option, + // TODO uncomment after deciding to expose this in the API. + // max_retry_seconds: Option, + // default_retry_seconds: Option, + // max_retries: Option, ) -> Self { Self { client_config: ReqwestClientConfig { @@ -106,10 +118,15 @@ impl ClientConfig { host: host.map_or(self.client_config.host.clone(), Some), origin: origin.map_or(self.client_config.origin.clone(), Some), }, - request_options: RequestOptions { - retry_seconds: retry_seconds.unwrap_or(self.request_options.retry_seconds), - max_retries: max_retries.unwrap_or(self.request_options.max_retries), - }, + request_options: RequestOptions::default(), + // TODO uncomment after deciding to expose this in the API. + // request_options: RequestOptions { + // max_retry_seconds: max_retry_seconds + // .unwrap_or(self.request_options.max_retry_seconds), + // default_retry_seconds: default_retry_seconds + // .unwrap_or(self.request_options.default_retry_seconds), + // max_retries: max_retries.unwrap_or(self.request_options.max_retries), + // }, } } } @@ -162,12 +179,68 @@ pub(crate) async fn wrapped_request( url: &str, client: &Client, ) -> Result { - let response = client - .reqwest_client - .get(url) - .send() - .await? - .error_for_status()?; + // send request and loop for possible retries + #[allow(unused_mut)] //because of wasm32 exclusion below + let mut response = client.reqwest_client.get(url).send().await?; + + // this doesn't work on wasm32 because tokio doesn't work on wasm + #[cfg(not(target_arch = "wasm32"))] + { + let mut tries: u16 = 0; + loop { + debug!("HTTP version: {:?}", response.version()); + // loop if HTTP 429 + if matches!(response.status(), StatusCode::TOO_MANY_REQUESTS) { + let retry_after = response + .headers() + .get(RETRY_AFTER) + .map(|value| value.to_str().unwrap().to_string()) + .unwrap_or(client.request_options.default_retry_seconds.to_string()); + let mut wait_time_seconds = + if let Ok(date) = DateTime::parse_from_rfc2822(&retry_after) { + (date.with_timezone(&Utc) - Utc::now()).num_seconds() as u64 + } else if let Ok(seconds) = retry_after.parse::() { + seconds + } else { + debug!( + "Unable to parse retry-after header value. Using {}", + client.request_options.default_retry_seconds + ); + client.request_options.default_retry_seconds.into() + }; + if wait_time_seconds == 0 { + debug!("Given {wait_time_seconds} for retry-after. Does not make sense."); + break; + } + if wait_time_seconds > client.request_options.max_retry_seconds as u64 { + info!( + "Server is asking to wait longer than configured max of {}.", + client.request_options.max_retry_seconds + ); + wait_time_seconds = client.request_options.max_retry_seconds as u64; + } + info!("Server says to wait {wait_time_seconds} seconds."); + tokio::time::sleep(tokio::time::Duration::from_secs(wait_time_seconds + 1)).await; + tries += 1; + if tries > client.request_options.max_retries { + info!("Max query retries reached."); + break; + } else { + // send the query again + response = client.reqwest_client.get(url).send().await?; + } + + // else don't repeat the request + } else { + break; + } + } + } + + // throw an error if not 200 OK + let response = response.error_for_status()?; + + // get the response let content_type = response .headers() .get(CONTENT_TYPE) From 45480c98c5b32ef640976dec3759554fdb2560ed Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Mon, 6 Jan 2025 05:44:57 -0500 Subject: [PATCH 11/16] adding rdap-test to the zip file --- .github/workflows/release-rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml index 1ab0696..291c133 100644 --- a/.github/workflows/release-rust.yml +++ b/.github/workflows/release-rust.yml @@ -64,7 +64,7 @@ jobs: shell: bash run: | cd target/${{ matrix.target }}/release - tar czvf ../../../icann-rdap-${{ matrix.target }}.tar.gz rdap rdap-srv rdap-srv-data rdap-srv-store rdap-srv-test-data + tar czvf ../../../icann-rdap-${{ matrix.target }}.tar.gz rdap rdap-test rdap-srv rdap-srv-data rdap-srv-store rdap-srv-test-data cd - - name: Publish uses: softprops/action-gh-release@v1 From 07be3e2690202b986eb9eed07cbee5958399a097 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Mon, 6 Jan 2025 07:05:38 -0500 Subject: [PATCH 12/16] added one-addr feature to rdap-test --- icann-rdap-cli/src/bin/rdap-test/main.rs | 7 +++++++ icann-rdap-cli/src/rt/exec.rs | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index a567b50..c6a9bdd 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -133,6 +133,12 @@ struct Cli { #[arg(long, required = false, env = "RDAP_TEST_SKIP_ORIGIN")] skip_origin: bool, + /// Only test one address. + /// + /// Only test one address per address family. + #[arg(long, required = false, env = "RDAP_TEST_ONE_ADDR")] + one_addr: bool, + /// Origin header value. /// /// Specifies the origin header value. @@ -367,6 +373,7 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> { expect_extensions: cli.expect_extensions, expect_groups, allow_unregistered_extensions: cli.allow_unregistered_extensions, + one_addr: cli.one_addr, }; let client_config = ClientConfig::builder() diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 3bd7ddd..71a2f33 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -31,6 +31,7 @@ pub struct TestOptions { pub expect_extensions: Vec, pub expect_groups: Vec, pub allow_unregistered_extensions: bool, + pub one_addr: bool, } #[derive(Clone)] @@ -122,10 +123,11 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( let dns_data = get_dns_records(host).await?; let mut test_results = TestResults::new(query_url.clone(), dns_data.clone()); + let mut more_runs = true; for v4 in dns_data.v4_addrs { // test run without origin let mut test_run = TestRun::new_v4(vec![], v4, port); - if !options.skip_v4 { + if !options.skip_v4 && more_runs { let client = create_client_with_addr(client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); @@ -134,7 +136,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run with origin let mut test_run = TestRun::new_v4(vec![RunFeature::OriginHeader], v4, port); - if !options.skip_v4 && !options.skip_origin { + if !options.skip_v4 && !options.skip_origin && more_runs { let client_config = ClientConfig::from_config(client_config) .origin(HeaderValue::from_str(&options.origin_value)?) .build(); @@ -143,11 +145,16 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( test_run = test_run.end(rdap_response, options); } test_results.add_test_run(test_run); + if options.one_addr { + more_runs = false; + } } + + let mut more_runs = true; for v6 in dns_data.v6_addrs { // test run without origin let mut test_run = TestRun::new_v6(vec![], v6, port); - if !options.skip_v6 { + if !options.skip_v6 && more_runs { let client = create_client_with_addr(client_config, host, test_run.socket_addr)?; let rdap_response = rdap_url_request(&query_url, &client).await; test_run = test_run.end(rdap_response, options); @@ -156,7 +163,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( // test run with origin let mut test_run = TestRun::new_v6(vec![RunFeature::OriginHeader], v6, port); - if !options.skip_v6 { + if !options.skip_v6 && !options.skip_origin && more_runs { let client_config = ClientConfig::from_config(client_config) .origin(HeaderValue::from_str(&options.origin_value)?) .build(); @@ -165,6 +172,9 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( test_run = test_run.end(rdap_response, options); } test_results.add_test_run(test_run); + if options.one_addr { + more_runs = false; + } } test_results.end(options); From e47a0865d0946ed7026d90c3f7ea2f1ffe3627f1 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Mon, 6 Jan 2025 11:40:21 -0500 Subject: [PATCH 13/16] added timeout value and changed some of the retry messages --- icann-rdap-cli/src/bin/rdap-test/main.rs | 13 ++++++++++ icann-rdap-cli/src/bin/rdap/main.rs | 13 ++++++++++ icann-rdap-client/src/http/reqwest.rs | 31 +++++++++++++++++++++--- icann-rdap-client/src/http/wrapped.rs | 22 ++++++++++++----- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index c6a9bdd..27fc36a 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -115,6 +115,18 @@ struct Cli { )] allow_invalid_certificates: bool, + /// Set the query timeout. + /// + /// This values specifies, in seconds, the total time to connect and read all + /// the data from a connection. + #[arg( + long, + required = false, + env = "RDAP_TEST_TIMEOUT_SECS", + default_value = "60" + )] + timeout_secs: u64, + /// Skip v4. /// /// Skip testing of IPv4 connections. @@ -382,6 +394,7 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> { .accept_invalid_host_names(cli.allow_invalid_host_names) .accept_invalid_certificates(cli.allow_invalid_certificates) .follow_redirects(cli.follow_redirects) + .timeout_secs(cli.timeout_secs) .build(); // execute tests diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index 68ffe58..70c971e 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -255,6 +255,18 @@ struct Cli { )] allow_invalid_certificates: bool, + /// Set the query timeout. + /// + /// This values specifies, in seconds, the total time to connect and read all + /// the data from a connection. + #[arg( + long, + required = false, + env = "RDAP_TIMEOUT_SECS", + default_value = "60" + )] + timeout_secs: u64, + /// Reset. /// /// Removes the cache files and resets the config file. @@ -559,6 +571,7 @@ pub async fn wrapped_main() -> Result<(), RdapCliError> { .https_only(!cli.allow_http) .accept_invalid_host_names(cli.allow_invalid_host_names) .accept_invalid_certificates(cli.allow_invalid_certificates) + .timeout_secs(cli.timeout_secs) .build(); let rdap_client = create_client(&client_config); if let Ok(client) = rdap_client { diff --git a/icann-rdap-client/src/http/reqwest.rs b/icann-rdap-client/src/http/reqwest.rs index bd13c10..44ad54a 100644 --- a/icann-rdap-client/src/http/reqwest.rs +++ b/icann-rdap-client/src/http/reqwest.rs @@ -1,14 +1,14 @@ //! Creates a Reqwest client. -use lazy_static::lazy_static; pub use reqwest::header::{self, HeaderValue}; pub use reqwest::Client as ReqwestClient; pub use reqwest::Error as ReqwestError; use icann_rdap_common::media_types::{JSON_MEDIA_TYPE, RDAP_MEDIA_TYPE}; +use lazy_static::lazy_static; #[cfg(not(target_arch = "wasm32"))] -use {icann_rdap_common::VERSION, std::net::SocketAddr}; +use {icann_rdap_common::VERSION, std::net::SocketAddr, std::time::Duration}; lazy_static! { static ref ACCEPT_HEADER_VALUES: String = format!("{RDAP_MEDIA_TYPE}, {JSON_MEDIA_TYPE}"); @@ -16,20 +16,31 @@ lazy_static! { /// Configures the HTTP client. pub struct ReqwestClientConfig { - /// This string is appended to the user agent. It is provided so + /// This string is appended to the user agent. + /// + /// It is provided so /// library users may identify their programs. + /// This is ignored on wasm32. pub user_agent_suffix: String, /// If set to true, connections will be required to use HTTPS. + /// + /// This is ignored on wasm32. pub https_only: bool, /// If set to true, invalid host names will be accepted. + /// + /// This is ignored on wasm32. pub accept_invalid_host_names: bool, /// If set to true, invalid certificates will be accepted. + /// + /// This is ignored on wasm32. pub accept_invalid_certificates: bool, /// If true, HTTP redirects will be followed. + /// + /// This is ignored on wasm32. pub follow_redirects: bool, /// Specify Host @@ -39,6 +50,13 @@ pub struct ReqwestClientConfig { /// /// Most browsers ignore this by default. pub origin: Option, + + /// Query timeout in seconds. + /// + /// This corresponds to the total timeout of the request (connection plus reading all the data). + /// + /// This is ignored on wasm32. + pub timeout_secs: u64, } impl Default for ReqwestClientConfig { @@ -51,6 +69,7 @@ impl Default for ReqwestClientConfig { follow_redirects: true, host: None, origin: None, + timeout_secs: 60, } } } @@ -66,6 +85,7 @@ impl ReqwestClientConfig { follow_redirects: Option, host: Option, origin: Option, + timeout_secs: Option, ) -> Self { let default = ReqwestClientConfig::default(); Self { @@ -78,6 +98,7 @@ impl ReqwestClientConfig { follow_redirects: follow_redirects.unwrap_or(default.follow_redirects), host, origin, + timeout_secs: timeout_secs.unwrap_or(default.timeout_secs), } } @@ -92,6 +113,7 @@ impl ReqwestClientConfig { follow_redirects: Option, host: Option, origin: Option, + timeout_secs: Option, ) -> Self { Self { user_agent_suffix: user_agent_suffix.unwrap_or(self.user_agent_suffix.clone()), @@ -103,6 +125,7 @@ impl ReqwestClientConfig { follow_redirects: follow_redirects.unwrap_or(self.follow_redirects), host: host.map_or(self.host.clone(), Some), origin: origin.map_or(self.origin.clone(), Some), + timeout_secs: timeout_secs.unwrap_or(self.timeout_secs), } } } @@ -123,6 +146,7 @@ pub fn create_reqwest_client(config: &ReqwestClientConfig) -> Result, host: Option, origin: Option, + timeout_secs: Option, // TODO uncomment after deciding to expose this in the API. // max_retry_seconds: Option, // default_retry_seconds: Option, @@ -77,6 +78,7 @@ impl ClientConfig { follow_redirects: follow_redirects.unwrap_or(default_cc.follow_redirects), host, origin, + timeout_secs: timeout_secs.unwrap_or(default_cc.timeout_secs), }, request_options: default_ro, // TODO uncomment after deciding to expose this in the API. @@ -100,6 +102,7 @@ impl ClientConfig { follow_redirects: Option, host: Option, origin: Option, + timeout_secs: Option, // TODO uncomment after deciding to expose this in the API. // max_retry_seconds: Option, // default_retry_seconds: Option, @@ -117,6 +120,7 @@ impl ClientConfig { follow_redirects: follow_redirects.unwrap_or(self.client_config.follow_redirects), host: host.map_or(self.client_config.host.clone(), Some), origin: origin.map_or(self.client_config.origin.clone(), Some), + timeout_secs: timeout_secs.unwrap_or(self.client_config.timeout_secs), }, request_options: RequestOptions::default(), // TODO uncomment after deciding to expose this in the API. @@ -191,25 +195,31 @@ pub(crate) async fn wrapped_request( debug!("HTTP version: {:?}", response.version()); // loop if HTTP 429 if matches!(response.status(), StatusCode::TOO_MANY_REQUESTS) { - let retry_after = response + let retry_after_header = response .headers() .get(RETRY_AFTER) - .map(|value| value.to_str().unwrap().to_string()) - .unwrap_or(client.request_options.default_retry_seconds.to_string()); + .map(|value| value.to_str().unwrap().to_string()); + let retry_after = if let Some(rt) = retry_after_header { + info!("Server says too many requests and to retry-after '{rt}'."); + rt + } else { + info!("Server says too many requests but does not offer 'retry-after' value."); + client.request_options.default_retry_seconds.to_string() + }; let mut wait_time_seconds = if let Ok(date) = DateTime::parse_from_rfc2822(&retry_after) { (date.with_timezone(&Utc) - Utc::now()).num_seconds() as u64 } else if let Ok(seconds) = retry_after.parse::() { seconds } else { - debug!( + info!( "Unable to parse retry-after header value. Using {}", client.request_options.default_retry_seconds ); client.request_options.default_retry_seconds.into() }; if wait_time_seconds == 0 { - debug!("Given {wait_time_seconds} for retry-after. Does not make sense."); + info!("Given {wait_time_seconds} for retry-after. Does not make sense."); break; } if wait_time_seconds > client.request_options.max_retry_seconds as u64 { @@ -219,7 +229,7 @@ pub(crate) async fn wrapped_request( ); wait_time_seconds = client.request_options.max_retry_seconds as u64; } - info!("Server says to wait {wait_time_seconds} seconds."); + info!("Waiting {wait_time_seconds} seconds to retry."); tokio::time::sleep(tokio::time::Duration::from_secs(wait_time_seconds + 1)).await; tries += 1; if tries > client.request_options.max_retries { From 91ed18c27903e917426a4c5c59ccf315f75af6c1 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Mon, 6 Jan 2025 17:17:12 -0500 Subject: [PATCH 14/16] adding parameters to control the retries --- icann-rdap-cli/src/bin/rdap-test/main.rs | 43 +++++++++++++++- icann-rdap-cli/src/bin/rdap/main.rs | 36 ++++++++++++++ icann-rdap-client/src/http/wrapped.rs | 63 ++++++++++-------------- 3 files changed, 105 insertions(+), 37 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index 27fc36a..54fc02a 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -88,7 +88,7 @@ struct Cli { /// /// When given, allows connections to RDAP servers using HTTP. /// Otherwise, only HTTPS is allowed. - #[arg(short = 'T', long, required = false, env = "RDAP_ALLOW_HTTP")] + #[arg(short = 'T', long, required = false, env = "RDAP_TEST_ALLOW_HTTP")] allow_http: bool, /// Allow invalid host names. @@ -115,6 +115,44 @@ struct Cli { )] allow_invalid_certificates: bool, + /// Maximum retry wait time. + /// + /// Sets the maximum number of seconds to wait before retrying a query when + /// a server has sent an HTTP 429 status code with a retry-after value. + /// That is, the value to used is no greater than this setting. + #[arg( + long, + required = false, + env = "RDAP_TEST_MAX_RETRY_SECS", + default_value = "120" + )] + max_retry_secs: u32, + + /// Default retry wait time. + /// + /// Sets the number of seconds to wait before retrying a query when + /// a server has sent an HTTP 429 status code without a retry-after value + /// or when the retry-after value does not make sense. + #[arg( + long, + required = false, + env = "RDAP_TEST_DEF_RETRY_SECS", + default_value = "60" + )] + def_retry_secs: u32, + + /// Maximum number of retries. + /// + /// This sets the maximum number of retries when a server signals too many + /// requests have been sent using an HTTP 429 status code. + #[arg( + long, + required = false, + env = "RDAP_TEST_MAX_RETRIES", + default_value = "1" + )] + max_retries: u16, + /// Set the query timeout. /// /// This values specifies, in seconds, the total time to connect and read all @@ -395,6 +433,9 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> { .accept_invalid_certificates(cli.allow_invalid_certificates) .follow_redirects(cli.follow_redirects) .timeout_secs(cli.timeout_secs) + .max_retry_secs(cli.max_retry_secs) + .def_retry_secs(cli.def_retry_secs) + .max_retries(cli.max_retries) .build(); // execute tests diff --git a/icann-rdap-cli/src/bin/rdap/main.rs b/icann-rdap-cli/src/bin/rdap/main.rs index 70c971e..f089b2d 100644 --- a/icann-rdap-cli/src/bin/rdap/main.rs +++ b/icann-rdap-cli/src/bin/rdap/main.rs @@ -267,6 +267,39 @@ struct Cli { )] timeout_secs: u64, + /// Maximum retry wait time. + /// + /// Sets the maximum number of seconds to wait before retrying a query when + /// a server has sent an HTTP 429 status code with a retry-after value. + /// That is, the value to used is no greater than this setting. + #[arg( + long, + required = false, + env = "RDAP_MAX_RETRY_SECS", + default_value = "120" + )] + max_retry_secs: u32, + + /// Default retry wait time. + /// + /// Sets the number of seconds to wait before retrying a query when + /// a server has sent an HTTP 429 status code without a retry-after value + /// or when the retry-after value does not make sense. + #[arg( + long, + required = false, + env = "RDAP_DEF_RETRY_SECS", + default_value = "60" + )] + def_retry_secs: u32, + + /// Maximum number of retries. + /// + /// This sets the maximum number of retries when a server signals too many + /// requests have been sent using an HTTP 429 status code. + #[arg(long, required = false, env = "RDAP_MAX_RETRIES", default_value = "1")] + max_retries: u16, + /// Reset. /// /// Removes the cache files and resets the config file. @@ -572,6 +605,9 @@ pub async fn wrapped_main() -> Result<(), RdapCliError> { .accept_invalid_host_names(cli.allow_invalid_host_names) .accept_invalid_certificates(cli.allow_invalid_certificates) .timeout_secs(cli.timeout_secs) + .max_retry_secs(cli.max_retry_secs) + .def_retry_secs(cli.def_retry_secs) + .max_retries(cli.max_retries) .build(); let rdap_client = create_client(&client_config); if let Ok(client) = rdap_client { diff --git a/icann-rdap-client/src/http/wrapped.rs b/icann-rdap-client/src/http/wrapped.rs index 0945bc2..cc7e463 100644 --- a/icann-rdap-client/src/http/wrapped.rs +++ b/icann-rdap-client/src/http/wrapped.rs @@ -22,16 +22,16 @@ use { /// Used by the request functions. #[derive(Clone, Copy)] pub struct RequestOptions { - pub(crate) max_retry_seconds: u32, - pub(crate) default_retry_seconds: u32, + pub(crate) max_retry_secs: u32, + pub(crate) def_retry_secs: u32, pub(crate) max_retries: u16, } impl Default for RequestOptions { fn default() -> Self { Self { - max_retry_seconds: 120, - default_retry_seconds: 120, + max_retry_secs: 120, + def_retry_secs: 60, max_retries: 1, } } @@ -60,10 +60,9 @@ impl ClientConfig { host: Option, origin: Option, timeout_secs: Option, - // TODO uncomment after deciding to expose this in the API. - // max_retry_seconds: Option, - // default_retry_seconds: Option, - // max_retries: Option, + max_retry_secs: Option, + def_retry_secs: Option, + max_retries: Option, ) -> Self { let default_cc = ReqwestClientConfig::default(); let default_ro = RequestOptions::default(); @@ -80,14 +79,11 @@ impl ClientConfig { origin, timeout_secs: timeout_secs.unwrap_or(default_cc.timeout_secs), }, - request_options: default_ro, - // TODO uncomment after deciding to expose this in the API. - // request_options: RequestOptions { - // max_retry_seconds: max_retry_seconds.unwrap_or(default_ro.max_retry_seconds), - // default_retry_seconds: default_retry_seconds - // .unwrap_or(default_ro.default_retry_seconds), - // max_retries: max_retries.unwrap_or(default_ro.max_retries), - // }, + request_options: RequestOptions { + max_retry_secs: max_retry_secs.unwrap_or(default_ro.max_retry_secs), + def_retry_secs: def_retry_secs.unwrap_or(default_ro.def_retry_secs), + max_retries: max_retries.unwrap_or(default_ro.max_retries), + }, } } @@ -103,10 +99,9 @@ impl ClientConfig { host: Option, origin: Option, timeout_secs: Option, - // TODO uncomment after deciding to expose this in the API. - // max_retry_seconds: Option, - // default_retry_seconds: Option, - // max_retries: Option, + max_retry_secs: Option, + def_retry_secs: Option, + max_retries: Option, ) -> Self { Self { client_config: ReqwestClientConfig { @@ -122,15 +117,11 @@ impl ClientConfig { origin: origin.map_or(self.client_config.origin.clone(), Some), timeout_secs: timeout_secs.unwrap_or(self.client_config.timeout_secs), }, - request_options: RequestOptions::default(), - // TODO uncomment after deciding to expose this in the API. - // request_options: RequestOptions { - // max_retry_seconds: max_retry_seconds - // .unwrap_or(self.request_options.max_retry_seconds), - // default_retry_seconds: default_retry_seconds - // .unwrap_or(self.request_options.default_retry_seconds), - // max_retries: max_retries.unwrap_or(self.request_options.max_retries), - // }, + request_options: RequestOptions { + max_retry_secs: max_retry_secs.unwrap_or(self.request_options.max_retry_secs), + def_retry_secs: def_retry_secs.unwrap_or(self.request_options.def_retry_secs), + max_retries: max_retries.unwrap_or(self.request_options.max_retries), + }, } } } @@ -204,7 +195,7 @@ pub(crate) async fn wrapped_request( rt } else { info!("Server says too many requests but does not offer 'retry-after' value."); - client.request_options.default_retry_seconds.to_string() + client.request_options.def_retry_secs.to_string() }; let mut wait_time_seconds = if let Ok(date) = DateTime::parse_from_rfc2822(&retry_after) { @@ -214,20 +205,20 @@ pub(crate) async fn wrapped_request( } else { info!( "Unable to parse retry-after header value. Using {}", - client.request_options.default_retry_seconds + client.request_options.def_retry_secs ); - client.request_options.default_retry_seconds.into() + client.request_options.def_retry_secs.into() }; if wait_time_seconds == 0 { info!("Given {wait_time_seconds} for retry-after. Does not make sense."); - break; + wait_time_seconds = client.request_options.def_retry_secs as u64; } - if wait_time_seconds > client.request_options.max_retry_seconds as u64 { + if wait_time_seconds > client.request_options.max_retry_secs as u64 { info!( "Server is asking to wait longer than configured max of {}.", - client.request_options.max_retry_seconds + client.request_options.max_retry_secs ); - wait_time_seconds = client.request_options.max_retry_seconds as u64; + wait_time_seconds = client.request_options.max_retry_secs as u64; } info!("Waiting {wait_time_seconds} seconds to retry."); tokio::time::sleep(tokio::time::Duration::from_secs(wait_time_seconds + 1)).await; From 560802e95964e7e612ea0f3148baac8097682ef4 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Tue, 7 Jan 2025 13:08:12 -0500 Subject: [PATCH 15/16] added option to change dns resolver --- icann-rdap-cli/src/bin/rdap-test/main.rs | 12 ++++++++++++ icann-rdap-cli/src/rt/exec.rs | 10 +++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/icann-rdap-cli/src/bin/rdap-test/main.rs b/icann-rdap-cli/src/bin/rdap-test/main.rs index 54fc02a..94087af 100644 --- a/icann-rdap-cli/src/bin/rdap-test/main.rs +++ b/icann-rdap-cli/src/bin/rdap-test/main.rs @@ -84,6 +84,17 @@ struct Cli { )] log_level: LogLevel, + /// DNS Resolver + /// + /// Specifies the address and port of the DNS resolver to query. + #[arg( + long, + required = false, + env = "RDAP_TEST_DNS_RESOLVER", + default_value = "8.8.8.8:53" + )] + dns_resolver: String, + /// Allow HTTP connections. /// /// When given, allows connections to RDAP servers using HTTP. @@ -424,6 +435,7 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> { expect_groups, allow_unregistered_extensions: cli.allow_unregistered_extensions, one_addr: cli.one_addr, + dns_resolver: Some(cli.dns_resolver), }; let client_config = ClientConfig::builder() diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 71a2f33..14420b6 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -32,6 +32,7 @@ pub struct TestOptions { pub expect_groups: Vec, pub allow_unregistered_extensions: bool, pub one_addr: bool, + pub dns_resolver: Option, } #[derive(Clone)] @@ -81,6 +82,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( expect_extensions: extensions, expect_groups: options.expect_groups.clone(), origin_value: options.origin_value.clone(), + dns_resolver: options.dns_resolver.clone(), ..*options }; @@ -120,7 +122,7 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( .ok_or(TestExecutionError::NoHostToResolve)?; info!("Testing {query_url}"); - let dns_data = get_dns_records(host).await?; + let dns_data = get_dns_records(host, options).await?; let mut test_results = TestResults::new(query_url.clone(), dns_data.clone()); let mut more_runs = true; @@ -182,8 +184,10 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( Ok(test_results) } -async fn get_dns_records(host: &str) -> Result { - let conn = UdpClientConnection::new("8.8.8.8:53".parse()?) +async fn get_dns_records(host: &str, options: &TestOptions) -> Result { + let def_dns_resolver = "8.8.8.8:53".to_string(); + let dns_resolver = options.dns_resolver.as_ref().unwrap_or(&def_dns_resolver); + let conn = UdpClientConnection::new(dns_resolver.parse()?) .unwrap() .new_stream(None); let (mut client, bg) = AsyncClient::connect(conn).await.unwrap(); From d77a0de91ca0a9ab894cbe2088e09599468b77c0 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 11 Jan 2025 12:08:44 -0500 Subject: [PATCH 16/16] don't do dns lookup if given ip addrs, test_jig and tests for rdap-test --- icann-rdap-cli/src/rt/exec.rs | 18 +++++ icann-rdap-cli/tests/integration/main.rs | 7 +- .../tests/integration/{ => rdap_cmd}/cache.rs | 2 +- .../tests/integration/{ => rdap_cmd}/check.rs | 2 +- .../tests/integration/rdap_cmd/mod.rs | 5 ++ .../integration/{ => rdap_cmd}/queries.rs | 20 ++--- .../integration/{ => rdap_cmd}/source.rs | 2 +- .../tests/integration/{ => rdap_cmd}/url.rs | 4 +- .../tests/integration/rdap_test_cmd/mod.rs | 1 + .../tests/integration/rdap_test_cmd/url.rs | 31 +++++++ icann-rdap-cli/tests/integration/test_jig.rs | 80 ++++++++++++------- 11 files changed, 123 insertions(+), 49 deletions(-) rename icann-rdap-cli/tests/integration/{ => rdap_cmd}/cache.rs (97%) rename icann-rdap-cli/tests/integration/{ => rdap_cmd}/check.rs (92%) create mode 100644 icann-rdap-cli/tests/integration/rdap_cmd/mod.rs rename icann-rdap-cli/tests/integration/{ => rdap_cmd}/queries.rs (91%) rename icann-rdap-cli/tests/integration/{ => rdap_cmd}/source.rs (96%) rename icann-rdap-cli/tests/integration/{ => rdap_cmd}/url.rs (93%) create mode 100644 icann-rdap-cli/tests/integration/rdap_test_cmd/mod.rs create mode 100644 icann-rdap-cli/tests/integration/rdap_test_cmd/url.rs diff --git a/icann-rdap-cli/src/rt/exec.rs b/icann-rdap-cli/src/rt/exec.rs index 14420b6..93360b0 100644 --- a/icann-rdap-cli/src/rt/exec.rs +++ b/icann-rdap-cli/src/rt/exec.rs @@ -1,5 +1,6 @@ //! Function to execute tests. +use std::net::{Ipv4Addr, Ipv6Addr}; use std::str::FromStr; use hickory_client::client::{AsyncClient, ClientConnection, ClientHandle}; @@ -185,6 +186,23 @@ pub async fn execute_tests<'a, BS: BootstrapStore>( } async fn get_dns_records(host: &str, options: &TestOptions) -> Result { + // short circuit dns if these are ip addresses + if let Ok(ip4) = Ipv4Addr::from_str(host) { + return Ok(DnsData { + v4_cname: None, + v6_cname: None, + v4_addrs: vec![ip4], + v6_addrs: vec![], + }); + } else if let Ok(ip6) = Ipv6Addr::from_str(host.trim_start_matches('[').trim_end_matches(']')) { + return Ok(DnsData { + v4_cname: None, + v6_cname: None, + v4_addrs: vec![], + v6_addrs: vec![ip6], + }); + } + let def_dns_resolver = "8.8.8.8:53".to_string(); let dns_resolver = options.dns_resolver.as_ref().unwrap_or(&def_dns_resolver); let conn = UdpClientConnection::new(dns_resolver.parse()?) diff --git a/icann-rdap-cli/tests/integration/main.rs b/icann-rdap-cli/tests/integration/main.rs index 60d200c..722b307 100644 --- a/icann-rdap-cli/tests/integration/main.rs +++ b/icann-rdap-cli/tests/integration/main.rs @@ -1,6 +1,3 @@ -mod cache; -mod check; -mod queries; -mod source; +mod rdap_cmd; +mod rdap_test_cmd; mod test_jig; -mod url; diff --git a/icann-rdap-cli/tests/integration/cache.rs b/icann-rdap-cli/tests/integration/rdap_cmd/cache.rs similarity index 97% rename from icann-rdap-cli/tests/integration/cache.rs rename to icann-rdap-cli/tests/integration/rdap_cmd/cache.rs index 2146d33..405266f 100644 --- a/icann-rdap-cli/tests/integration/cache.rs +++ b/icann-rdap-cli/tests/integration/rdap_cmd/cache.rs @@ -9,7 +9,7 @@ use crate::test_jig::TestJig; #[tokio::test(flavor = "multi_thread")] async fn GIVEN_domain_with_entity_WHEN_retreived_from_cache_THEN_is_domain() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain( &Domain::basic() diff --git a/icann-rdap-cli/tests/integration/check.rs b/icann-rdap-cli/tests/integration/rdap_cmd/check.rs similarity index 92% rename from icann-rdap-cli/tests/integration/check.rs rename to icann-rdap-cli/tests/integration/rdap_cmd/check.rs index 2b53e14..be04399 100644 --- a/icann-rdap-cli/tests/integration/check.rs +++ b/icann-rdap-cli/tests/integration/rdap_cmd/check.rs @@ -8,7 +8,7 @@ use crate::test_jig::TestJig; #[tokio::test(flavor = "multi_thread")] async fn GIVEN_domain_with_check_WHEN_query_THEN_failure() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain(&Domain::basic().ldh_name("foo.example").build()) .await diff --git a/icann-rdap-cli/tests/integration/rdap_cmd/mod.rs b/icann-rdap-cli/tests/integration/rdap_cmd/mod.rs new file mode 100644 index 0000000..758e119 --- /dev/null +++ b/icann-rdap-cli/tests/integration/rdap_cmd/mod.rs @@ -0,0 +1,5 @@ +mod cache; +mod check; +mod queries; +mod source; +mod url; diff --git a/icann-rdap-cli/tests/integration/queries.rs b/icann-rdap-cli/tests/integration/rdap_cmd/queries.rs similarity index 91% rename from icann-rdap-cli/tests/integration/queries.rs rename to icann-rdap-cli/tests/integration/rdap_cmd/queries.rs index 1400492..fc96461 100644 --- a/icann-rdap-cli/tests/integration/queries.rs +++ b/icann-rdap-cli/tests/integration/rdap_cmd/queries.rs @@ -16,7 +16,7 @@ use crate::test_jig::TestJig; #[tokio::test(flavor = "multi_thread")] async fn GIVEN_domain_WHEN_query_THEN_success(#[case] db_domain: &str, #[case] q_domain: &str) { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain(&Domain::basic().ldh_name(db_domain).build()) .await @@ -34,7 +34,7 @@ async fn GIVEN_domain_WHEN_query_THEN_success(#[case] db_domain: &str, #[case] q #[tokio::test(flavor = "multi_thread")] async fn GIVEN_tld_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain(&Domain::basic().ldh_name("example").build()) .await @@ -53,7 +53,7 @@ async fn GIVEN_tld_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_entity_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_entity(&Entity::basic().handle("foo").build()) .await @@ -71,7 +71,7 @@ async fn GIVEN_entity_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_nameserver_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_nameserver( &Nameserver::basic() @@ -94,7 +94,7 @@ async fn GIVEN_nameserver_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_autnum_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_autnum(&Autnum::basic().autnum_range(700..710).build()) .await @@ -112,7 +112,7 @@ async fn GIVEN_autnum_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_network_ip_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_network( &Network::basic() @@ -138,7 +138,7 @@ async fn GIVEN_network_ip_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_network_cidr_WHEN_query_THEN_success(#[case] db_cidr: &str, #[case] q_cidr: &str) { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_network( &Network::basic() @@ -161,7 +161,7 @@ async fn GIVEN_network_cidr_WHEN_query_THEN_success(#[case] db_cidr: &str, #[cas #[tokio::test(flavor = "multi_thread")] async fn GIVEN_url_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain(&Domain::basic().ldh_name("foo.example").build()) .await @@ -180,7 +180,7 @@ async fn GIVEN_url_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_idn_WHEN_query_a_label_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain(&Domain::basic().ldh_name("xn--caf-dma.example").build()) .await @@ -198,7 +198,7 @@ async fn GIVEN_idn_WHEN_query_a_label_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_domain_WHEN_search_domain_names_THEN_success() { // GIVEN - let mut test_jig = TestJig::new_with_enable_domain_name_search().await; + let mut test_jig = TestJig::new_rdap_with_dn_search().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_domain(&Domain::basic().ldh_name("foo.example").build()) .await diff --git a/icann-rdap-cli/tests/integration/source.rs b/icann-rdap-cli/tests/integration/rdap_cmd/source.rs similarity index 96% rename from icann-rdap-cli/tests/integration/source.rs rename to icann-rdap-cli/tests/integration/rdap_cmd/source.rs index 02bcbdd..1e69cb4 100644 --- a/icann-rdap-cli/tests/integration/source.rs +++ b/icann-rdap-cli/tests/integration/rdap_cmd/source.rs @@ -16,7 +16,7 @@ async fn GIVEN_inr_query_WHEN_query_THEN_source_is_rir( #[case] q_cidr: &str, ) { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_network( &Network::basic() diff --git a/icann-rdap-cli/tests/integration/url.rs b/icann-rdap-cli/tests/integration/rdap_cmd/url.rs similarity index 93% rename from icann-rdap-cli/tests/integration/url.rs rename to icann-rdap-cli/tests/integration/rdap_cmd/url.rs index 98e0c34..9b91047 100644 --- a/icann-rdap-cli/tests/integration/url.rs +++ b/icann-rdap-cli/tests/integration/rdap_cmd/url.rs @@ -8,7 +8,7 @@ use crate::test_jig::TestJig; #[tokio::test(flavor = "multi_thread")] async fn GIVEN_url_used_with_base_url_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_network( &Network::basic() @@ -32,7 +32,7 @@ async fn GIVEN_url_used_with_base_url_WHEN_query_THEN_success() { #[tokio::test(flavor = "multi_thread")] async fn GIVEN_url_used_with_no_base_url_WHEN_query_THEN_success() { // GIVEN - let mut test_jig = TestJig::new().await; + let mut test_jig = TestJig::new_rdap().await; test_jig.cmd.env_remove("RDAP_BASE_URL"); let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); tx.add_network( diff --git a/icann-rdap-cli/tests/integration/rdap_test_cmd/mod.rs b/icann-rdap-cli/tests/integration/rdap_test_cmd/mod.rs new file mode 100644 index 0000000..ec3f638 --- /dev/null +++ b/icann-rdap-cli/tests/integration/rdap_test_cmd/mod.rs @@ -0,0 +1 @@ +mod url; diff --git a/icann-rdap-cli/tests/integration/rdap_test_cmd/url.rs b/icann-rdap-cli/tests/integration/rdap_test_cmd/url.rs new file mode 100644 index 0000000..cdd119b --- /dev/null +++ b/icann-rdap-cli/tests/integration/rdap_test_cmd/url.rs @@ -0,0 +1,31 @@ +#![allow(non_snake_case)] + +use icann_rdap_common::response::network::Network; +use icann_rdap_srv::storage::StoreOps; + +use crate::test_jig::TestJig; + +#[tokio::test(flavor = "multi_thread")] +async fn GIVEN_url_WHEN_test_THEN_success() { + // GIVEN + let mut test_jig = TestJig::new_rdap_test().await; + test_jig.cmd.env_remove("RDAP_BASE_URL"); + let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); + tx.add_network( + &Network::basic() + .cidr("10.0.0.0/24") + .build() + .expect("cidr parsing"), + ) + .await + .expect("add network in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let url = format!("{}/ip/10.0.0.1", test_jig.rdap_base); + test_jig.cmd.arg(url); + + // THEN + let assert = test_jig.cmd.assert(); + assert.success(); +} diff --git a/icann-rdap-cli/tests/integration/test_jig.rs b/icann-rdap-cli/tests/integration/test_jig.rs index a762784..17e319b 100644 --- a/icann-rdap-cli/tests/integration/test_jig.rs +++ b/icann-rdap-cli/tests/integration/test_jig.rs @@ -10,28 +10,39 @@ use test_dir::DirBuilder; use test_dir::FileType; use test_dir::TestDir; +pub enum CommandType { + Rdap, + RdapTest, +} + pub struct TestJig { pub mem: Mem, pub cmd: Command, + pub cmd_type: CommandType, pub rdap_base: String, // pass ownership to the test so the directories are dropped when the test is done. - _test_dir: TestDir, + test_dir: TestDir, } impl TestJig { - pub async fn new() -> TestJig { + pub async fn new_rdap() -> TestJig { let common_config = CommonConfig::default(); - TestJig::new_common_config(common_config).await + TestJig::new_common_config(common_config, CommandType::Rdap).await } - pub async fn new_with_enable_domain_name_search() -> TestJig { + pub async fn new_rdap_with_dn_search() -> TestJig { let common_config = CommonConfig::builder() .domain_search_by_name_enable(true) .build(); - TestJig::new_common_config(common_config).await + TestJig::new_common_config(common_config, CommandType::Rdap).await } - pub async fn new_common_config(common_config: CommonConfig) -> TestJig { + pub async fn new_rdap_test() -> TestJig { + let common_config = CommonConfig::default(); + TestJig::new_common_config(common_config, CommandType::RdapTest).await + } + + pub async fn new_common_config(common_config: CommonConfig, cmd_type: CommandType) -> TestJig { let mem = Mem::new(MemConfig::builder().common_config(common_config).build()); let app_state = AppState { storage: mem.clone(), @@ -51,35 +62,46 @@ impl TestJig { let test_dir = TestDir::temp() .create("cache", FileType::Dir) .create("config", FileType::Dir); - let mut cmd = Command::cargo_bin("rdap").expect("cannot find rdap cmd"); - cmd.env_clear() - .timeout(Duration::from_secs(2)) - .env("RDAP_BASE_URL", rdap_base.clone()) - .env("RDAP_PAGING", "none") - .env("RDAP_OUTPUT", "json-extra") - .env("RDAP_LOG", "debug") - .env("RDAP_ALLOW_HTTP", "true") - .env("XDG_CACHE_HOME", test_dir.path("cache")) - .env("XDG_CONFIG_HOME", test_dir.path("config")); - TestJig { + let cmd = Command::new("sh"); //throw away + let jig = TestJig { mem, cmd, + cmd_type, rdap_base, - _test_dir: test_dir, - } + test_dir, + }; + jig.new_cmd() } + /// Creates a new command from an existing one but resetting necessary environment variables. + /// + /// Using the function allows the test jig to stay up but a new command to be executed. pub fn new_cmd(self) -> TestJig { - let mut cmd = Command::cargo_bin("rdap").expect("cannot find rdap cmd"); - cmd.env_clear() - .timeout(Duration::from_secs(2)) - .env("RDAP_BASE_URL", self.rdap_base.clone()) - .env("RDAP_PAGING", "none") - .env("RDAP_OUTPUT", "json-extra") - .env("RDAP_LOG", "debug") - .env("RDAP_ALLOW_HTTP", "true") - .env("XDG_CACHE_HOME", self._test_dir.path("cache")) - .env("XDG_CONFIG_HOME", self._test_dir.path("config")); + let cmd = match self.cmd_type { + CommandType::Rdap => { + let mut cmd = Command::cargo_bin("rdap").expect("cannot find rdap cmd"); + cmd.env_clear() + .timeout(Duration::from_secs(2)) + .env("RDAP_BASE_URL", self.rdap_base.clone()) + .env("RDAP_PAGING", "none") + .env("RDAP_OUTPUT", "json-extra") + .env("RDAP_LOG", "debug") + .env("RDAP_ALLOW_HTTP", "true") + .env("XDG_CACHE_HOME", self.test_dir.path("cache")) + .env("XDG_CONFIG_HOME", self.test_dir.path("config")); + cmd + } + CommandType::RdapTest => { + let mut cmd = Command::cargo_bin("rdap-test").expect("cannot find rdap-test cmd"); + cmd.env_clear() + .timeout(Duration::from_secs(2)) + .env("RDAP_TEST_LOG", "debug") + .env("RDAP_TEST_ALLOW_HTTP", "true") + .env("XDG_CACHE_HOME", self.test_dir.path("cache")) + .env("XDG_CONFIG_HOME", self.test_dir.path("config")); + cmd + } + }; TestJig { cmd, ..self } } }