Skip to content

Commit

Permalink
Merge pull request #1 from quackscience/refactor
Browse files Browse the repository at this point in the history 
Refactor and Deduplicate functions
  • Loading branch information
@lmangani
lmangani authored Oct 20, 2024
2 parents 4cddecc + abcde70 commit d8b7a91
Showing 1 changed file with 90 additions and 133 deletions.
223 changes: 90 additions & 133 deletions src/http_client_extension.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,89 +15,103 @@

namespace duckdb {

// Helper function to parse URL and setup client
static std::pair<duckdb_httplib_openssl::Client, std::string> SetupHttpClient(const std::string &url) {
std::string scheme, domain, path;
size_t pos = url.find("://");
std::string mod_url = url;
if (pos != std::string::npos) {
scheme = mod_url.substr(0, pos);
mod_url.erase(0, pos + 3);
}

pos = mod_url.find("/");
if (pos != std::string::npos) {
domain = mod_url.substr(0, pos);
path = mod_url.substr(pos);
} else {
domain = mod_url;
path = "/";
}

// Create client and set a reasonable timeout (e.g., 10 seconds)
duckdb_httplib_openssl::Client client(domain.c_str());
client.set_read_timeout(10, 0); // 10 seconds
client.set_follow_location(true); // Follow redirects

return std::make_pair(std::move(client), path);
}

static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std::string &request_type) {
std::string err_message = "HTTP " + request_type + " request failed. ";

switch (res.error()) {
case duckdb_httplib_openssl::Error::Connection:
err_message += "Connection error.";
break;
case duckdb_httplib_openssl::Error::BindIPAddress:
err_message += "Failed to bind IP address.";
break;
case duckdb_httplib_openssl::Error::Read:
err_message += "Error reading response.";
break;
case duckdb_httplib_openssl::Error::Write:
err_message += "Error writing request.";
break;
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
err_message += "Too many redirects.";
break;
case duckdb_httplib_openssl::Error::Canceled:
err_message += "Request was canceled.";
break;
case duckdb_httplib_openssl::Error::SSLConnection:
err_message += "SSL connection failed.";
break;
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
err_message += "Failed to load SSL certificates.";
break;
case duckdb_httplib_openssl::Error::SSLServerVerification:
err_message += "SSL server verification failed.";
break;
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
err_message += "Unsupported characters in multipart boundary.";
break;
case duckdb_httplib_openssl::Error::Compression:
err_message += "Error during compression.";
break;
default:
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
}


static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vector &result) {
D_ASSERT(args.data.size() == 1);

UnaryExecutor::Execute<string_t, string_t>(args.data[0], result, args.size(), [&](string_t input) {
std::string url = input.GetString();

// Parse the URL to extract the domain and path
std::string scheme, domain, path;
size_t pos = url.find("://");
if (pos != std::string::npos) {
scheme = url.substr(0, pos);
url.erase(0, pos + 3);
}

pos = url.find("/");
if (pos != std::string::npos) {
domain = url.substr(0, pos);
path = url.substr(pos);
} else {
domain = url;
path = "/";
}

// Create client and set a reasonable timeout (e.g., 10 seconds)
duckdb_httplib_openssl::Client client(domain.c_str());
client.set_read_timeout(10, 0); // 10 seconds

// Follow redirects
client.set_follow_location(true);
// Use helper to setup client and parse URL
auto client_and_path = SetupHttpClient(url);
auto &client = client_and_path.first;
auto &path = client_and_path.second;

// Make the GET request
auto res = client.Get(path.c_str());
if (res) {
if (res->status == 200) {
return StringVector::AddString(result, res->body);
} else {
throw std::runtime_error("HTTP error: " + std::to_string(res->status) + " - " + res->reason);
throw std::runtime_error("HTTP GET error: " + std::to_string(res->status) + " - " + res->reason);
}
} else {
// Handle the error case
std::string err_message = "HTTP request failed. ";

// Convert httplib error codes to a descriptive message
switch (res.error()) {
case duckdb_httplib_openssl::Error::Connection:
err_message += "Connection error.";
break;
case duckdb_httplib_openssl::Error::BindIPAddress:
err_message += "Failed to bind IP address.";
break;
case duckdb_httplib_openssl::Error::Read:
err_message += "Error reading response.";
break;
case duckdb_httplib_openssl::Error::Write:
err_message += "Error writing request.";
break;
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
err_message += "Too many redirects.";
break;
case duckdb_httplib_openssl::Error::Canceled:
err_message += "Request was canceled.";
break;
case duckdb_httplib_openssl::Error::SSLConnection:
err_message += "SSL connection failed.";
break;
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
err_message += "Failed to load SSL certificates.";
break;
case duckdb_httplib_openssl::Error::SSLServerVerification:
err_message += "SSL server verification failed.";
break;
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
err_message += "Unsupported characters in multipart boundary.";
break;
case duckdb_httplib_openssl::Error::Compression:
err_message += "Error during compression.";
break;
default:
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
// Handle errors
HandleHttpError(res, "GET");
}
// Ensure a return value in case of an error
return string_t();
});
}

Expand All @@ -113,28 +127,10 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
[&](string_t url, string_t headers, string_t body) {
std::string url_str = url.GetString();

// Parse the URL to extract the domain and path
std::string scheme, domain, path;
size_t pos = url_str.find("://");
if (pos != std::string::npos) {
scheme = url_str.substr(0, pos);
url_str.erase(0, pos + 3);
}

pos = url_str.find("/");
if (pos != std::string::npos) {
domain = url_str.substr(0, pos);
path = url_str.substr(pos);
} else {
domain = url_str;
path = "/";
}

// Create the client and set a timeout (e.g., 10 seconds)
duckdb_httplib_openssl::Client client(domain.c_str());
client.set_read_timeout(10, 0); // 10 seconds
// Follow redirects
client.set_follow_location(true);
// Use helper to setup client and parse URL
auto client_and_path = SetupHttpClient(url_str);
auto &client = client_and_path.first;
auto &path = client_and_path.second;

// Prepare headers
duckdb_httplib_openssl::Headers header_map;
Expand All @@ -160,53 +156,14 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
if (res->status == 200) {
return StringVector::AddString(result, res->body);
} else {
throw std::runtime_error("HTTP error: " + std::to_string(res->status) + " - " + res->reason);
throw std::runtime_error("HTTP POST error: " + std::to_string(res->status) + " - " + res->reason);
}
} else {
// Handle the error case
std::string err_message = "HTTP POST request failed. ";

// Convert httplib error codes to a descriptive message
switch (res.error()) {
case duckdb_httplib_openssl::Error::Connection:
err_message += "Connection error.";
break;
case duckdb_httplib_openssl::Error::BindIPAddress:
err_message += "Failed to bind IP address.";
break;
case duckdb_httplib_openssl::Error::Read:
err_message += "Error reading response.";
break;
case duckdb_httplib_openssl::Error::Write:
err_message += "Error writing request.";
break;
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
err_message += "Too many redirects.";
break;
case duckdb_httplib_openssl::Error::Canceled:
err_message += "Request was canceled.";
break;
case duckdb_httplib_openssl::Error::SSLConnection:
err_message += "SSL connection failed.";
break;
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
err_message += "Failed to load SSL certificates.";
break;
case duckdb_httplib_openssl::Error::SSLServerVerification:
err_message += "SSL server verification failed.";
break;
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
err_message += "Unsupported characters in multipart boundary.";
break;
case duckdb_httplib_openssl::Error::Compression:
err_message += "Error during compression.";
break;
default:
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
// Handle errors
HandleHttpError(res, "POST");
}
// Ensure a return value in case of an error
return string_t();
});
}

Expand Down

0 comments on commit d8b7a91

Please sign in to comment.