Merge pull request #95 from icann/rdap_test

bypass dns lookup if ipaddr and integration test for rda-test

.github/workflows/release-rust.yml

@@ -64,7 +64,7 @@ jobs:
         shell: bash
         run: |
           cd target/${{ matrix.target }}/release          
-          tar czvf ../../../icann-rdap-${{ matrix.target }}.tar.gz rdap rdap-srv rdap-srv-data rdap-srv-store rdap-srv-test-data 
+          tar czvf ../../../icann-rdap-${{ matrix.target }}.tar.gz rdap rdap-test rdap-srv rdap-srv-data rdap-srv-store rdap-srv-test-data 
           cd -
       - name: Publish
         uses: softprops/action-gh-release@v1

icann-rdap-cli/src/bin/rdap-test/error.rs

@@ -1,7 +1,7 @@
 use std::process::{ExitCode, Termination};
 
 use icann_rdap_cli::rt::exec::TestExecutionError;
-use icann_rdap_client::iana_request::IanaResponseError;
+use icann_rdap_client::iana::IanaResponseError;
 use icann_rdap_client::RdapClientError;
 use thiserror::Error;
 

icann-rdap-cli/src/bin/rdap-test/main.rs

@@ -11,9 +11,9 @@ use icann_rdap_cli::rt::exec::ExtensionGroup;
 use icann_rdap_cli::rt::exec::TestOptions;
 use icann_rdap_cli::rt::results::RunOutcome;
 use icann_rdap_cli::rt::results::TestResults;
-use icann_rdap_client::client::ClientConfig;
+use icann_rdap_client::http::ClientConfig;
 use icann_rdap_client::md::MdOptions;
-use icann_rdap_client::QueryType;
+use icann_rdap_client::rdap::QueryType;
 use icann_rdap_common::check::traverse_checks;
 use icann_rdap_common::check::CheckClass;
 use termimad::crossterm::style::Color::*;
@@ -84,11 +84,22 @@ struct Cli {
     )]
     log_level: LogLevel,
 
+    /// DNS Resolver
+    ///
+    /// Specifies the address and port of the DNS resolver to query.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_TEST_DNS_RESOLVER",
+        default_value = "8.8.8.8:53"
+    )]
+    dns_resolver: String,
+
     /// Allow HTTP connections.
     ///
     /// When given, allows connections to RDAP servers using HTTP.
     /// Otherwise, only HTTPS is allowed.
-    #[arg(short = 'T', long, required = false, env = "RDAP_ALLOW_HTTP")]
+    #[arg(short = 'T', long, required = false, env = "RDAP_TEST_ALLOW_HTTP")]
     allow_http: bool,
 
     /// Allow invalid host names.
@@ -115,6 +126,56 @@ struct Cli {
     )]
     allow_invalid_certificates: bool,
 
+    /// Maximum retry wait time.
+    ///
+    /// Sets the maximum number of seconds to wait before retrying a query when
+    /// a server has sent an HTTP 429 status code with a retry-after value.
+    /// That is, the value to used is no greater than this setting.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_TEST_MAX_RETRY_SECS",
+        default_value = "120"
+    )]
+    max_retry_secs: u32,
+
+    /// Default retry wait time.
+    ///
+    /// Sets the number of seconds to wait before retrying a query when
+    /// a server has sent an HTTP 429 status code without a retry-after value
+    /// or when the retry-after value does not make sense.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_TEST_DEF_RETRY_SECS",
+        default_value = "60"
+    )]
+    def_retry_secs: u32,
+
+    /// Maximum number of retries.
+    ///
+    /// This sets the maximum number of retries when a server signals too many
+    /// requests have been sent using an HTTP 429 status code.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_TEST_MAX_RETRIES",
+        default_value = "1"
+    )]
+    max_retries: u16,
+
+    /// Set the query timeout.
+    ///
+    /// This values specifies, in seconds, the total time to connect and read all
+    /// the data from a connection.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_TEST_TIMEOUT_SECS",
+        default_value = "60"
+    )]
+    timeout_secs: u64,
+
     /// Skip v4.
     ///
     /// Skip testing of IPv4 connections.
@@ -133,6 +194,12 @@ struct Cli {
     #[arg(long, required = false, env = "RDAP_TEST_SKIP_ORIGIN")]
     skip_origin: bool,
 
+    /// Only test one address.
+    ///
+    /// Only test one address per address family.
+    #[arg(long, required = false, env = "RDAP_TEST_ONE_ADDR")]
+    one_addr: bool,
+
     /// Origin header value.
     ///
     /// Specifies the origin header value.
@@ -367,6 +434,8 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> {
         expect_extensions: cli.expect_extensions,
         expect_groups,
         allow_unregistered_extensions: cli.allow_unregistered_extensions,
+        one_addr: cli.one_addr,
+        dns_resolver: Some(cli.dns_resolver),
     };
 
     let client_config = ClientConfig::builder()
@@ -375,6 +444,10 @@ pub async fn wrapped_main() -> Result<(), RdapTestError> {
         .accept_invalid_host_names(cli.allow_invalid_host_names)
         .accept_invalid_certificates(cli.allow_invalid_certificates)
         .follow_redirects(cli.follow_redirects)
+        .timeout_secs(cli.timeout_secs)
+        .max_retry_secs(cli.max_retry_secs)
+        .def_retry_secs(cli.def_retry_secs)
+        .max_retries(cli.max_retries)
         .build();
 
     // execute tests

icann-rdap-cli/src/bin/rdap/bootstrap.rs

@@ -1,11 +1,13 @@
 use crate::error::RdapCliError;
 use icann_rdap_cli::dirs::fcbs::FileCacheBootstrapStore;
-use icann_rdap_client::query::{
-    bootstrap::{fetch_bootstrap, qtype_to_bootstrap_url, BootstrapStore, PreferredUrl},
-    qtype::QueryType,
+use icann_rdap_client::http::Client;
+use icann_rdap_client::iana::BootstrapStore;
+use icann_rdap_client::iana::PreferredUrl;
+use icann_rdap_client::{
+    iana::{fetch_bootstrap, qtype_to_bootstrap_url},
+    rdap::QueryType,
 };
 use icann_rdap_common::iana::IanaRegistryType;
-use reqwest::Client;
 use tracing::debug;
 
 /// Defines the type of bootstrapping to use.

icann-rdap-cli/src/bin/rdap/error.rs

@@ -1,6 +1,6 @@
 use std::process::{ExitCode, Termination};
 
-use icann_rdap_client::iana_request::IanaResponseError;
+use icann_rdap_client::iana::IanaResponseError;
 use icann_rdap_client::RdapClientError;
 use minus::MinusError;
 use thiserror::Error;

icann-rdap-cli/src/bin/rdap/main.rs

@@ -3,8 +3,9 @@ use clap::builder::styling::AnsiColor;
 use clap::builder::Styles;
 use error::RdapCliError;
 use icann_rdap_cli::dirs;
-use icann_rdap_client::client::create_client;
-use icann_rdap_client::client::ClientConfig;
+use icann_rdap_client::http::create_client;
+use icann_rdap_client::http::Client;
+use icann_rdap_client::http::ClientConfig;
 use icann_rdap_common::check::CheckClass;
 use query::InrBackupBootstrap;
 use query::ProcessType;
@@ -21,10 +22,9 @@ use write::FmtWrite;
 use write::PagerWrite;
 
 use clap::{ArgGroup, Parser, ValueEnum};
-use icann_rdap_client::query::qtype::QueryType;
+use icann_rdap_client::rdap::QueryType;
 use icann_rdap_common::VERSION;
 use query::OutputType;
-use reqwest::Client;
 use tokio::{join, task::spawn_blocking};
 
 use crate::query::do_query;
@@ -255,6 +255,51 @@ struct Cli {
     )]
     allow_invalid_certificates: bool,
 
+    /// Set the query timeout.
+    ///
+    /// This values specifies, in seconds, the total time to connect and read all
+    /// the data from a connection.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_TIMEOUT_SECS",
+        default_value = "60"
+    )]
+    timeout_secs: u64,
+
+    /// Maximum retry wait time.
+    ///
+    /// Sets the maximum number of seconds to wait before retrying a query when
+    /// a server has sent an HTTP 429 status code with a retry-after value.
+    /// That is, the value to used is no greater than this setting.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_MAX_RETRY_SECS",
+        default_value = "120"
+    )]
+    max_retry_secs: u32,
+
+    /// Default retry wait time.
+    ///
+    /// Sets the number of seconds to wait before retrying a query when
+    /// a server has sent an HTTP 429 status code without a retry-after value
+    /// or when the retry-after value does not make sense.
+    #[arg(
+        long,
+        required = false,
+        env = "RDAP_DEF_RETRY_SECS",
+        default_value = "60"
+    )]
+    def_retry_secs: u32,
+
+    /// Maximum number of retries.
+    ///
+    /// This sets the maximum number of retries when a server signals too many
+    /// requests have been sent using an HTTP 429 status code.
+    #[arg(long, required = false, env = "RDAP_MAX_RETRIES", default_value = "1")]
+    max_retries: u16,
+
     /// Reset.
     ///
     /// Removes the cache files and resets the config file.
@@ -559,6 +604,10 @@ pub async fn wrapped_main() -> Result<(), RdapCliError> {
         .https_only(!cli.allow_http)
         .accept_invalid_host_names(cli.allow_invalid_host_names)
         .accept_invalid_certificates(cli.allow_invalid_certificates)
+        .timeout_secs(cli.timeout_secs)
+        .max_retry_secs(cli.max_retry_secs)
+        .def_retry_secs(cli.def_retry_secs)
+        .max_retries(cli.max_retries)
         .build();
     let rdap_client = create_client(&client_config);
     if let Ok(client) = rdap_client {

icann-rdap-cli/src/bin/rdap/query.rs

@@ -1,3 +1,4 @@
+use icann_rdap_client::http::Client;
 use icann_rdap_common::check::traverse_checks;
 use icann_rdap_common::check::CheckClass;
 use icann_rdap_common::check::CheckParams;
@@ -11,10 +12,9 @@ use tracing::info;
 use icann_rdap_client::{
     gtld::{GtldParams, ToGtldWhois},
     md::{redacted::replace_redacted_items, MdOptions, MdParams, ToMd},
-    query::{qtype::QueryType, request::ResponseData},
-    rr::{RequestData, RequestResponse, RequestResponses, SourceType},
+    rdap::{QueryType, ResponseData},
+    rdap::{RequestData, RequestResponse, RequestResponses, SourceType},
 };
-use reqwest::Client;
 use termimad::{crossterm::style::Color::*, Alignment, MadSkin};
 
 use crate::bootstrap::get_base_url;

icann-rdap-cli/src/bin/rdap/request.rs

@@ -3,14 +3,13 @@ use std::{
     io::{BufRead, BufReader},
 };
 
-use icann_rdap_client::query::{
-    qtype::QueryType,
-    request::{rdap_url_request, ResponseData},
+use icann_rdap_client::{
+    http::Client,
+    rdap::{rdap_url_request, QueryType, ResponseData},
 };
 use icann_rdap_common::{httpdata::HttpData, response::GetSelfLink};
 use pct_str::PctString;
 use pct_str::URIReserved;
-use reqwest::Client;
 use tracing::{debug, info};
 
 use crate::{dirs::rdap_cache_path, error::RdapCliError, query::ProcessingParams};

icann-rdap-cli/src/dirs/fcbs.rs

@@ -4,7 +4,7 @@ use std::{
     path::PathBuf,
 };
 
-use icann_rdap_client::query::bootstrap::{BootstrapStore, RegistryHasNotExpired};
+use icann_rdap_client::iana::{BootstrapStore, RegistryHasNotExpired};
 use icann_rdap_common::{
     httpdata::HttpData,
     iana::{BootstrapRegistry, IanaRegistry, IanaRegistryType},
@@ -94,9 +94,9 @@ where
 #[cfg(test)]
 #[allow(non_snake_case)]
 mod test {
-    use icann_rdap_client::query::{
-        bootstrap::{BootstrapStore, PreferredUrl},
-        qtype::QueryType,
+    use icann_rdap_client::{
+        iana::{BootstrapStore, PreferredUrl},
+        rdap::QueryType,
     };
     use icann_rdap_common::{
         httpdata::HttpData,