Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Runtime Config #399

Merged
merged 2 commits into from
Apr 29, 2024
+191 −116
Merged

Runtime Config #399

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7052329
feat: tidying up CLI and file configuration for runtime config. Added…
Taepper Apr 25, 2024
5546bcf
feat: also get Runtime Config options from environment variables
Taepper Apr 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions include/silo_api/runtime_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,30 @@
#include <filesystem>
#include <optional>

#include <Poco/Util/AbstractConfiguration.h>

#include "silo/preprocessing/preprocessing_config.h"

namespace silo_api {

const std::string DATA_DIRECTORY_OPTION = "dataDirectory";
const std::string DATA_DIRECTORY_ENV_OPTION = "SILO_DATA_DIRECTORY";
const std::string MAX_CONNECTIONS_OPTION = "maxQueuedHttpConnections";
const std::string MAX_CONNECTIONS_ENV_OPTION = "SILO_MAX_QUEUED_HTTP_CONNECTIONS";
const std::string PARALLEL_THREADS_OPTION = "threadsForHttpConnections";
const std::string PARALLEL_THREADS_ENV_OPTION = "SILO_THREADS_FOR_HTTP_CONNECTIONS";
const std::string PORT_OPTION = "port";
const std::string PORT_ENV_OPTION = "SILO_PORT";

struct RuntimeConfig {
std::optional<std::filesystem::path> data_directory;
std::filesystem::path data_directory = silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory;
int32_t max_connections = 64;
int32_t parallel_threads = 4;
uint16_t port = 8081;

static RuntimeConfig readFromFile(const std::filesystem::path& config_path);
void overwriteFromFile(const std::filesystem::path& config_path);
void overwriteFromEnvironmentVariables();
void overwriteFromCommandLineArguments(const Poco::Util::AbstractConfiguration& config);
};

} // namespace silo_api
165 changes: 73 additions & 92 deletions src/silo_api/api.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
static const std::string ESTIMATED_STARTUP_TIME_IN_MINUTES_OPTION = "estimatedStartupTimeInMinutes";
static const std::string PREPROCESSING_CONFIG_OPTION = "preprocessingConfig";
static const std::string DATABASE_CONFIG_OPTION = "databaseConfig";
static const std::string DATA_DIRECTORY_OPTION = "dataDirectory";
static const std::string API_OPTION = "api";
static const std::string PREPROCESSING_OPTION = "preprocessing";

Expand Down Expand Up @@ -76,107 +75,91 @@ silo::config::DatabaseConfig databaseConfig(const Poco::Util::AbstractConfigurat
return silo::config::ConfigRepository().getValidatedConfig("database_config.yaml");
}

std::filesystem::path dataDirectory(
const Poco::Util::AbstractConfiguration& config,
const silo_api::RuntimeConfig& runtime_config
) {
if (config.hasProperty(DATA_DIRECTORY_OPTION)) {
SPDLOG_DEBUG(
"Using dataDirectory passed via command line argument: {}",
config.getString(DATA_DIRECTORY_OPTION)
);
return config.getString(DATA_DIRECTORY_OPTION);
}
if (runtime_config.data_directory.has_value()) {
SPDLOG_DEBUG(
"Using dataDirectory from runtime config file: {}",
runtime_config.data_directory.value().string()
);
return runtime_config.data_directory.value();
}

SPDLOG_DEBUG(
"dataDirectory not found in specified. Using default value: {}",
silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory
);
return silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory;
}

class SiloServer : public Poco::Util::ServerApplication {
protected:
[[maybe_unused]] void defineOptions(Poco::Util::OptionSet& options) override {
ServerApplication::defineOptions(options);

options.addOption(
Poco::Util::Option("help", "h", "display help information on command line arguments")
Poco::Util::Option()
.fullName("help")
.shortName("h")
.description("display help information on command line arguments")
.required(false)
.repeatable(false)
.callback(Poco::Util::OptionCallback<SiloServer>(this, &SiloServer::displayHelp))
);

options.addOption(
Poco::Util::Option(
PREPROCESSING_CONFIG_OPTION, "pc", "path to the preprocessing config file"
)
.required(false)
.repeatable(false)
.argument("PATH")
.binding(PREPROCESSING_CONFIG_OPTION)
);
options.addOption(Poco::Util::Option()
.fullName(PREPROCESSING_CONFIG_OPTION)
.description("path to the preprocessing config file")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(PREPROCESSING_CONFIG_OPTION));

options.addOption(
Poco::Util::Option(DATABASE_CONFIG_OPTION, "dc", "path to the database config file")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(DATABASE_CONFIG_OPTION)
);
options.addOption(Poco::Util::Option()
.fullName(DATABASE_CONFIG_OPTION)
.description("path to the database config file")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(DATABASE_CONFIG_OPTION));

options.addOption(
Poco::Util::Option(DATA_DIRECTORY_OPTION, "d", "path to the preprocessed data")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(DATA_DIRECTORY_OPTION)
);
options.addOption(Poco::Util::Option()
.fullName(silo_api::DATA_DIRECTORY_OPTION)
.shortName("d")
.description("path to the preprocessed data")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(silo_api::DATA_DIRECTORY_OPTION));

options.addOption(Poco::Util::Option(
"maxQueuedHttpConnections", "mqhc", "maximum number of http connections"
)
options.addOption(Poco::Util::Option()
.fullName(silo_api::PORT_OPTION)
.description("port to listen to requests")
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding(silo_api::PORT_OPTION));

options.addOption(Poco::Util::Option()
.fullName("maxQueuedHttpConnections")
.description("maximum number of http connections")
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding("maxQueuedHttpConnections"));

options.addOption(
Poco::Util::Option(
"threadsForHttpConnections", "tfhc", "number of threads for http connections"
)
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding("threadsForHttpConnections")
);
options.addOption(Poco::Util::Option()
.fullName("threadsForHttpConnections")
.description("number of threads for http connections")
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding("threadsForHttpConnections"));

options.addOption(Poco::Util::Option()
.fullName(API_OPTION)
.shortName("a")
.description("Execution mode: start the SILO web interface")
.required(false)
.repeatable(false)
.binding(API_OPTION)
.group("executionMode"));

options.addOption(
Poco::Util::Option(API_OPTION, "a", "Execution mode: start the SILO web interface")
Poco::Util::Option()
.fullName(PREPROCESSING_OPTION)
.shortName("p")
.description("Execution mode: trigger the preprocessing pipeline to generate a "
"partitioned dataset that can be read by the database")
.required(false)
.repeatable(false)
.binding(API_OPTION)
.binding(PREPROCESSING_OPTION)
.group("executionMode")
);

options.addOption(Poco::Util::Option(
PREPROCESSING_OPTION,
"p",
"Execution mode: trigger the preprocessing pipeline to generate a "
"partitioned dataset that can be read by the database"
)
.required(false)
.repeatable(false)
.binding(PREPROCESSING_OPTION)
.group("executionMode"));

options.addOption(
Poco::Util::Option(
ESTIMATED_STARTUP_TIME_IN_MINUTES_OPTION,
Expand Down Expand Up @@ -209,8 +192,7 @@ class SiloServer : public Poco::Util::ServerApplication {
return handlePreprocessing();
}

std::cout << "No execution mode specified."
<< "\n\n";
std::cout << "No execution mode specified.\n\n";
displayHelp("", "");
return Application::EXIT_USAGE;
}
Expand All @@ -230,37 +212,36 @@ class SiloServer : public Poco::Util::ServerApplication {

int handleApi() {
SPDLOG_INFO("Starting SILO API");
const int port = 8081;

silo_api::RuntimeConfig runtime_config;
if (std::filesystem::exists("./runtime_config.yaml")) {
runtime_config = silo_api::RuntimeConfig::readFromFile("./runtime_config.yaml");
runtime_config.overwriteFromFile("./runtime_config.yaml");
}

const auto data_directory = dataDirectory(config(), runtime_config);
runtime_config.overwriteFromEnvironmentVariables();
runtime_config.overwriteFromCommandLineArguments(config());

silo_api::DatabaseMutex database_mutex;

const Poco::Net::ServerSocket server_socket(port);
const Poco::Net::ServerSocket server_socket(runtime_config.port);

const silo_api::DatabaseDirectoryWatcher watcher(data_directory, database_mutex);
const silo_api::DatabaseDirectoryWatcher watcher(
runtime_config.data_directory, database_mutex
);

auto* const poco_parameter = new Poco::Net::HTTPServerParams;
const auto max_connections = config().getInt("maxQueuedHttpConnections", 64);
SPDLOG_INFO("Using {} queued http connections", max_connections);
poco_parameter->setMaxQueued(max_connections);

const auto threads = config().getInt("threadsForHttpConnections", 4);
SPDLOG_INFO("Using {} threads for http connections", threads);
poco_parameter->setMaxThreads(threads);
SPDLOG_INFO("Using {} queued http connections", runtime_config.max_connections);
poco_parameter->setMaxQueued(runtime_config.max_connections);

SPDLOG_INFO("Using {} threads for http connections", runtime_config.parallel_threads);
poco_parameter->setMaxThreads(runtime_config.parallel_threads);

Poco::Net::HTTPServer server(
new silo_api::SiloRequestHandlerFactory(database_mutex, getStartupConfig()),
server_socket,
poco_parameter
);

SPDLOG_INFO("Listening on port {}", port);
SPDLOG_INFO("Listening on port {}", runtime_config.port);

server.start();
waitForTerminationRequest();
Expand Down
Loading