Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor and Deduplicate functions #1

Merged
merged 5 commits into from
Oct 20, 2024
Merged

Refactor and Deduplicate functions #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
22b8e99
post follow
lmangani Oct 20, 2024
b105508
Merge remote-tracking branch 'refs/remotes/origin/main'
lmangani Oct 20, 2024
037b60c
Merge remote-tracking branch 'refs/remotes/origin/main'
lmangani Oct 20, 2024
dd65a86
cleanup
lmangani Oct 20, 2024
abcde70
refactor, deduplicate
lmangani Oct 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 90 additions & 133 deletions src/http_client_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,89 +15,103 @@

namespace duckdb {

// Helper function to parse URL and setup client
static std::pair<duckdb_httplib_openssl::Client, std::string> SetupHttpClient(const std::string &url) {
std::string scheme, domain, path;
size_t pos = url.find("://");
std::string mod_url = url;
if (pos != std::string::npos) {
scheme = mod_url.substr(0, pos);
mod_url.erase(0, pos + 3);
}

pos = mod_url.find("/");
if (pos != std::string::npos) {
domain = mod_url.substr(0, pos);
path = mod_url.substr(pos);
} else {
domain = mod_url;
path = "/";
}

// Create client and set a reasonable timeout (e.g., 10 seconds)
duckdb_httplib_openssl::Client client(domain.c_str());
client.set_read_timeout(10, 0); // 10 seconds
client.set_follow_location(true); // Follow redirects

return std::make_pair(std::move(client), path);
}

static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std::string &request_type) {
std::string err_message = "HTTP " + request_type + " request failed. ";

switch (res.error()) {
case duckdb_httplib_openssl::Error::Connection:
err_message += "Connection error.";
break;
case duckdb_httplib_openssl::Error::BindIPAddress:
err_message += "Failed to bind IP address.";
break;
case duckdb_httplib_openssl::Error::Read:
err_message += "Error reading response.";
break;
case duckdb_httplib_openssl::Error::Write:
err_message += "Error writing request.";
break;
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
err_message += "Too many redirects.";
break;
case duckdb_httplib_openssl::Error::Canceled:
err_message += "Request was canceled.";
break;
case duckdb_httplib_openssl::Error::SSLConnection:
err_message += "SSL connection failed.";
break;
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
err_message += "Failed to load SSL certificates.";
break;
case duckdb_httplib_openssl::Error::SSLServerVerification:
err_message += "SSL server verification failed.";
break;
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
err_message += "Unsupported characters in multipart boundary.";
break;
case duckdb_httplib_openssl::Error::Compression:
err_message += "Error during compression.";
break;
default:
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
}


static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vector &result) {
D_ASSERT(args.data.size() == 1);

UnaryExecutor::Execute<string_t, string_t>(args.data[0], result, args.size(), [&](string_t input) {
std::string url = input.GetString();

// Parse the URL to extract the domain and path
std::string scheme, domain, path;
size_t pos = url.find("://");
if (pos != std::string::npos) {
scheme = url.substr(0, pos);
url.erase(0, pos + 3);
}

pos = url.find("/");
if (pos != std::string::npos) {
domain = url.substr(0, pos);
path = url.substr(pos);
} else {
domain = url;
path = "/";
}

// Create client and set a reasonable timeout (e.g., 10 seconds)
duckdb_httplib_openssl::Client client(domain.c_str());
client.set_read_timeout(10, 0); // 10 seconds

// Follow redirects
client.set_follow_location(true);
// Use helper to setup client and parse URL
auto client_and_path = SetupHttpClient(url);
auto &client = client_and_path.first;
auto &path = client_and_path.second;

// Make the GET request
auto res = client.Get(path.c_str());
if (res) {
if (res->status == 200) {
return StringVector::AddString(result, res->body);
} else {
throw std::runtime_error("HTTP error: " + std::to_string(res->status) + " - " + res->reason);
throw std::runtime_error("HTTP GET error: " + std::to_string(res->status) + " - " + res->reason);
}
} else {
// Handle the error case
std::string err_message = "HTTP request failed. ";

// Convert httplib error codes to a descriptive message
switch (res.error()) {
case duckdb_httplib_openssl::Error::Connection:
err_message += "Connection error.";
break;
case duckdb_httplib_openssl::Error::BindIPAddress:
err_message += "Failed to bind IP address.";
break;
case duckdb_httplib_openssl::Error::Read:
err_message += "Error reading response.";
break;
case duckdb_httplib_openssl::Error::Write:
err_message += "Error writing request.";
break;
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
err_message += "Too many redirects.";
break;
case duckdb_httplib_openssl::Error::Canceled:
err_message += "Request was canceled.";
break;
case duckdb_httplib_openssl::Error::SSLConnection:
err_message += "SSL connection failed.";
break;
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
err_message += "Failed to load SSL certificates.";
break;
case duckdb_httplib_openssl::Error::SSLServerVerification:
err_message += "SSL server verification failed.";
break;
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
err_message += "Unsupported characters in multipart boundary.";
break;
case duckdb_httplib_openssl::Error::Compression:
err_message += "Error during compression.";
break;
default:
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
// Handle errors
HandleHttpError(res, "GET");
}
// Ensure a return value in case of an error
return string_t();
});
}

Expand All @@ -113,28 +127,10 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
[&](string_t url, string_t headers, string_t body) {
std::string url_str = url.GetString();

// Parse the URL to extract the domain and path
std::string scheme, domain, path;
size_t pos = url_str.find("://");
if (pos != std::string::npos) {
scheme = url_str.substr(0, pos);
url_str.erase(0, pos + 3);
}

pos = url_str.find("/");
if (pos != std::string::npos) {
domain = url_str.substr(0, pos);
path = url_str.substr(pos);
} else {
domain = url_str;
path = "/";
}

// Create the client and set a timeout (e.g., 10 seconds)
duckdb_httplib_openssl::Client client(domain.c_str());
client.set_read_timeout(10, 0); // 10 seconds
// Follow redirects
client.set_follow_location(true);
// Use helper to setup client and parse URL
auto client_and_path = SetupHttpClient(url_str);
auto &client = client_and_path.first;
auto &path = client_and_path.second;

// Prepare headers
duckdb_httplib_openssl::Headers header_map;
Expand All @@ -160,53 +156,14 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
if (res->status == 200) {
return StringVector::AddString(result, res->body);
} else {
throw std::runtime_error("HTTP error: " + std::to_string(res->status) + " - " + res->reason);
throw std::runtime_error("HTTP POST error: " + std::to_string(res->status) + " - " + res->reason);
}
} else {
// Handle the error case
std::string err_message = "HTTP POST request failed. ";

// Convert httplib error codes to a descriptive message
switch (res.error()) {
case duckdb_httplib_openssl::Error::Connection:
err_message += "Connection error.";
break;
case duckdb_httplib_openssl::Error::BindIPAddress:
err_message += "Failed to bind IP address.";
break;
case duckdb_httplib_openssl::Error::Read:
err_message += "Error reading response.";
break;
case duckdb_httplib_openssl::Error::Write:
err_message += "Error writing request.";
break;
case duckdb_httplib_openssl::Error::ExceedRedirectCount:
err_message += "Too many redirects.";
break;
case duckdb_httplib_openssl::Error::Canceled:
err_message += "Request was canceled.";
break;
case duckdb_httplib_openssl::Error::SSLConnection:
err_message += "SSL connection failed.";
break;
case duckdb_httplib_openssl::Error::SSLLoadingCerts:
err_message += "Failed to load SSL certificates.";
break;
case duckdb_httplib_openssl::Error::SSLServerVerification:
err_message += "SSL server verification failed.";
break;
case duckdb_httplib_openssl::Error::UnsupportedMultipartBoundaryChars:
err_message += "Unsupported characters in multipart boundary.";
break;
case duckdb_httplib_openssl::Error::Compression:
err_message += "Error during compression.";
break;
default:
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
// Handle errors
HandleHttpError(res, "POST");
}
// Ensure a return value in case of an error
return string_t();
});
}

Expand Down
Loading
Loading