refactor: Use libtest-mimic to run integration tests. (#137)

.github/workflows/ci.yml

@@ -38,16 +38,6 @@ jobs:
 
   unit:
     runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Update Rust Stable
-        run: rustup update stable
-      - name: Test
-        run: cargo test --all-targets --all-features
-
-  integration-tests:
-    runs-on: ubuntu-latest
-    needs: unit
     steps:
       - uses: actions/checkout@v3
       - name: Update Rust Stable
@@ -59,9 +49,5 @@ jobs:
         uses: abatilo/actions-poetry@v2
         with:
           poetry-version: "1.5.1"
-      - name: Build rust
-        run: cargo build --all-features
-      - name: Run tests
-        run: |
-          cd tests/integration
-          bash run.sh
+      - name: Test
+        run: cargo test --all-targets --all-features

Cargo.toml

@@ -17,7 +17,6 @@ async-trait = "0.1"
 apache-avro = { version = "0.15", features = ["derive"] }
 arrow = { version = ">=40, <45" }
 bytes = "1"
-futures = "0.3"
 opendal = ">=0.37, <0.40"
 uuid = { version = "1", features = ["v4"] }
 serde = "1"
@@ -35,5 +34,8 @@ env_logger = "0.10.0"
 csv = "1"
 url = "2"
 regex = "1"
-clap = { version = "4", features = ["derive"]}
+clap = { version = "4", features = ["derive"] }
 ordered-float = "3.7.0"
+confique = "0.2"
+libtest-mimic = "0.6"
+futures = { version = "0.3", features = ["executor"]}

rest_api/src/apis/configuration.rs

@@ -4,12 +4,10 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
-
 #[derive(Debug, Clone)]
 pub struct Configuration {
     pub base_path: String,
@@ -30,7 +28,6 @@ pub struct ApiKey {
     pub key: String,
 }
 
-
 impl Configuration {
     pub fn new() -> Configuration {
         Configuration::default()
@@ -47,7 +44,6 @@ impl Default for Configuration {
             oauth_access_token: None,
             bearer_access_token: None,
             api_key: None,
-
         }
     }
 }

rest_api/src/apis/configuration_api_api.rs

@@ -4,16 +4,14 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
 use reqwest;
 
+use super::{configuration, Error};
 use crate::apis::ResponseContent;
-use super::{Error, configuration};
-
 
 /// struct for typed errors of method [`get_config`]
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -28,21 +26,26 @@ pub enum GetConfigError {
     UnknownValue(serde_json::Value),
 }
 
-
-///  All REST clients should first call this route to get catalog configuration properties from the server to configure the catalog and its HTTP client. Configuration from the server consists of two sets of key/value pairs. - defaults -  properties that should be used as default configuration; applied before client configuration - overrides - properties that should be used to override client configuration; applied after defaults and client configuration  Catalog configuration is constructed by setting the defaults, then client- provided configuration, and finally overrides. The final property set is then used to configure the catalog.  For example, a default configuration property might set the size of the client pool, which can be replaced with a client-specific setting. An override might be used to set the warehouse location, which is stored on the server rather than in client configuration.  Common catalog configuration settings are documented at https://iceberg.apache.org/configuration/#catalog-properties 
-pub async fn get_config(configuration: &configuration::Configuration, warehouse: Option<&str>) -> Result<crate::models::CatalogConfig, Error<GetConfigError>> {
+///  All REST clients should first call this route to get catalog configuration properties from the server to configure the catalog and its HTTP client. Configuration from the server consists of two sets of key/value pairs. - defaults -  properties that should be used as default configuration; applied before client configuration - overrides - properties that should be used to override client configuration; applied after defaults and client configuration  Catalog configuration is constructed by setting the defaults, then client- provided configuration, and finally overrides. The final property set is then used to configure the catalog.  For example, a default configuration property might set the size of the client pool, which can be replaced with a client-specific setting. An override might be used to set the warehouse location, which is stored on the server rather than in client configuration.  Common catalog configuration settings are documented at https://iceberg.apache.org/configuration/#catalog-properties
+pub async fn get_config(
+    configuration: &configuration::Configuration,
+    warehouse: Option<&str>,
+) -> Result<crate::models::CatalogConfig, Error<GetConfigError>> {
     let local_var_configuration = configuration;
 
     let local_var_client = &local_var_configuration.client;
 
     let local_var_uri_str = format!("{}/v1/config", local_var_configuration.base_path);
-    let mut local_var_req_builder = local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str());
+    let mut local_var_req_builder =
+        local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str());
 
     if let Some(ref local_var_str) = warehouse {
-        local_var_req_builder = local_var_req_builder.query(&[("warehouse", &local_var_str.to_string())]);
+        local_var_req_builder =
+            local_var_req_builder.query(&[("warehouse", &local_var_str.to_string())]);
     }
     if let Some(ref local_var_user_agent) = local_var_configuration.user_agent {
-        local_var_req_builder = local_var_req_builder.header(reqwest::header::USER_AGENT, local_var_user_agent.clone());
+        local_var_req_builder =
+            local_var_req_builder.header(reqwest::header::USER_AGENT, local_var_user_agent.clone());
     }
     if let Some(ref local_var_token) = local_var_configuration.oauth_access_token {
         local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned());
@@ -60,9 +63,13 @@ pub async fn get_config(configuration: &configuration::Configuration, warehouse:
     if !local_var_status.is_client_error() && !local_var_status.is_server_error() {
         serde_json::from_str(&local_var_content).map_err(Error::from)
     } else {
-        let local_var_entity: Option<GetConfigError> = serde_json::from_str(&local_var_content).ok();
-        let local_var_error = ResponseContent { status: local_var_status, content: local_var_content, entity: local_var_entity };
+        let local_var_entity: Option<GetConfigError> =
+            serde_json::from_str(&local_var_content).ok();
+        let local_var_error = ResponseContent {
+            status: local_var_status,
+            content: local_var_content,
+            entity: local_var_entity,
+        };
         Err(Error::ResponseError(local_var_error))
     }
 }
-

rest_api/src/apis/mod.rs

@@ -16,7 +16,7 @@ pub enum Error<T> {
     ResponseError(ResponseContent<T>),
 }
 
-impl <T> fmt::Display for Error<T> {
+impl<T> fmt::Display for Error<T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let (module, e) = match self {
             Error::Reqwest(e) => ("reqwest", e.to_string()),
@@ -28,7 +28,7 @@ impl <T> fmt::Display for Error<T> {
     }
 }
 
-impl <T: fmt::Debug> error::Error for Error<T> {
+impl<T: fmt::Debug> error::Error for Error<T> {
     fn source(&self) -> Option<&(dyn error::Error + 'static)> {
         Some(match self {
             Error::Reqwest(e) => e,
@@ -39,19 +39,19 @@ impl <T: fmt::Debug> error::Error for Error<T> {
     }
 }
 
-impl <T> From<reqwest::Error> for Error<T> {
+impl<T> From<reqwest::Error> for Error<T> {
     fn from(e: reqwest::Error) -> Self {
         Error::Reqwest(e)
     }
 }
 
-impl <T> From<serde_json::Error> for Error<T> {
+impl<T> From<serde_json::Error> for Error<T> {
     fn from(e: serde_json::Error) -> Self {
         Error::Serde(e)
     }
 }
 
-impl <T> From<std::io::Error> for Error<T> {
+impl<T> From<std::io::Error> for Error<T> {
     fn from(e: std::io::Error) -> Self {
         Error::Io(e)
     }
@@ -78,8 +78,10 @@ pub fn parse_deep_object(prefix: &str, value: &serde_json::Value) -> Vec<(String
                             value,
                         ));
                     }
-                },
-                serde_json::Value::String(s) => params.push((format!("{}[{}]", prefix, key), s.clone())),
+                }
+                serde_json::Value::String(s) => {
+                    params.push((format!("{}[{}]", prefix, key), s.clone()))
+                }
                 _ => params.push((format!("{}[{}]", prefix, key), value.to_string())),
             }
         }

rest_api/src/apis/o_auth2_api_api.rs

@@ -4,16 +4,14 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
 use reqwest;
 
+use super::{configuration, Error};
 use crate::apis::ResponseContent;
-use super::{Error, configuration};
-
 
 /// struct for typed errors of method [`get_token`]
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -25,18 +23,30 @@ pub enum GetTokenError {
     UnknownValue(serde_json::Value),
 }
 
-
 /// Exchange credentials for a token using the OAuth2 client credentials flow or token exchange.  This endpoint is used for three purposes - 1. To exchange client credentials (client ID and secret) for an access token This uses the client credentials flow. 2. To exchange a client token and an identity token for a more specific access token This uses the token exchange flow. 3. To exchange an access token for one with the same claims and a refreshed expiration period This uses the token exchange flow.  For example, a catalog client may be configured with client credentials from the OAuth2 Authorization flow. This client would exchange its client ID and secret for an access token using the client credentials request with this endpoint (1). Subsequent requests would then use that access token.  Some clients may also handle sessions that have additional user context. These clients would use the token exchange flow to exchange a user token (the \"subject\" token) from the session for a more specific access token for that user, using the catalog's access token as the \"actor\" token (2). The user ID token is the \"subject\" token and can be any token type allowed by the OAuth2 token exchange flow, including a unsecured JWT token with a sub claim. This request should use the catalog's bearer token in the \"Authorization\" header.  Clients may also use the token exchange flow to refresh a token that is about to expire by sending a token exchange request (3). The request's \"subject\" token should be the expiring token. This request should use the subject token in the \"Authorization\" header.
-pub async fn get_token(configuration: &configuration::Configuration, grant_type: &str, client_id: &str, client_secret: &str, subject_token: &str, subject_token_type: crate::models::TokenType, scope: Option<&str>, requested_token_type: Option<crate::models::TokenType>, actor_token: Option<&str>, actor_token_type: Option<crate::models::TokenType>) -> Result<crate::models::OAuthTokenResponse, Error<GetTokenError>> {
+pub async fn get_token(
+    configuration: &configuration::Configuration,
+    grant_type: &str,
+    client_id: &str,
+    client_secret: &str,
+    subject_token: &str,
+    subject_token_type: crate::models::TokenType,
+    scope: Option<&str>,
+    requested_token_type: Option<crate::models::TokenType>,
+    actor_token: Option<&str>,
+    actor_token_type: Option<crate::models::TokenType>,
+) -> Result<crate::models::OAuthTokenResponse, Error<GetTokenError>> {
     let local_var_configuration = configuration;
 
     let local_var_client = &local_var_configuration.client;
 
     let local_var_uri_str = format!("{}/v1/oauth/tokens", local_var_configuration.base_path);
-    let mut local_var_req_builder = local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str());
+    let mut local_var_req_builder =
+        local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str());
 
     if let Some(ref local_var_user_agent) = local_var_configuration.user_agent {
-        local_var_req_builder = local_var_req_builder.header(reqwest::header::USER_AGENT, local_var_user_agent.clone());
+        local_var_req_builder =
+            local_var_req_builder.header(reqwest::header::USER_AGENT, local_var_user_agent.clone());
     }
     if let Some(ref local_var_token) = local_var_configuration.oauth_access_token {
         local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned());
@@ -74,8 +84,11 @@ pub async fn get_token(configuration: &configuration::Configuration, grant_type:
         serde_json::from_str(&local_var_content).map_err(Error::from)
     } else {
         let local_var_entity: Option<GetTokenError> = serde_json::from_str(&local_var_content).ok();
-        let local_var_error = ResponseContent { status: local_var_status, content: local_var_content, entity: local_var_entity };
+        let local_var_error = ResponseContent {
+            status: local_var_status,
+            content: local_var_content,
+            entity: local_var_entity,
+        };
         Err(Error::ResponseError(local_var_error))
     }
 }
-

rest_api/src/models/add_partition_spec_update.rs

@@ -4,13 +4,10 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
-
-
 #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)]
 pub struct AddPartitionSpecUpdate {
     #[serde(rename = "action")]
@@ -28,7 +25,7 @@ impl AddPartitionSpecUpdate {
     }
 }
 
-/// 
+///
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
 pub enum Action {
     #[serde(rename = "upgrade-format-version")]
@@ -66,4 +63,3 @@ impl Default for Action {
         Self::UpgradeFormatVersion
     }
 }
-

rest_api/src/models/add_partition_spec_update_all_of.rs

@@ -4,13 +4,10 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
-
-
 #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)]
 pub struct AddPartitionSpecUpdateAllOf {
     #[serde(rename = "spec")]
@@ -24,5 +21,3 @@ impl AddPartitionSpecUpdateAllOf {
         }
     }
 }
-
-

rest_api/src/models/add_schema_update.rs

@@ -4,13 +4,10 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
-
-
 #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)]
 pub struct AddSchemaUpdate {
     #[serde(rename = "action")]
@@ -28,7 +25,7 @@ impl AddSchemaUpdate {
     }
 }
 
-/// 
+///
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
 pub enum Action {
     #[serde(rename = "upgrade-format-version")]
@@ -66,4 +63,3 @@ impl Default for Action {
         Self::UpgradeFormatVersion
     }
 }
-

rest_api/src/models/add_schema_update_all_of.rs

@@ -4,13 +4,10 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
-
-
 #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)]
 pub struct AddSchemaUpdateAllOf {
     #[serde(rename = "schema")]
@@ -24,5 +21,3 @@ impl AddSchemaUpdateAllOf {
         }
     }
 }
-
-

rest_api/src/models/add_snapshot_update.rs

@@ -4,13 +4,10 @@
  * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2.
  *
  * The version of the OpenAPI document: 0.0.1
- * 
+ *
  * Generated by: https://openapi-generator.tech
  */
 
-
-
-
 #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)]
 pub struct AddSnapshotUpdate {
     #[serde(rename = "action")]
@@ -28,7 +25,7 @@ impl AddSnapshotUpdate {
     }
 }
 
-/// 
+///
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
 pub enum Action {
     #[serde(rename = "upgrade-format-version")]
@@ -66,4 +63,3 @@ impl Default for Action {
         Self::UpgradeFormatVersion
     }
 }
-