Skip to content

Commit

Permalink
feat: tidying up CLI and file configuration for runtime config. Added…
Browse files Browse the repository at this point in the history 
… option for specifying the port
  • Loading branch information
@Taepper
Taepper committed Apr 25, 2024
1 parent 7bb6d28 commit df8a9e7
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 112 deletions.
16 changes: 14 additions & 2 deletions include/silo_api/runtime_config.h
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,26 @@
#pragma once

#include <Poco/Util/AbstractConfiguration.h>
#include <filesystem>
#include <optional>

#include "silo/preprocessing/preprocessing_config.h"

namespace silo_api {

static const std::string DATA_DIRECTORY_OPTION = "dataDirectory";
static const std::string MAX_CONNECTIONS_OPTION = "maxQueuedHttpConnections";
static const std::string PARALLEL_THREADS_OPTION = "threadsForHttpConnections";
static const std::string PORT_OPTION = "port";

struct RuntimeConfig {
std::optional<std::filesystem::path> data_directory;
std::filesystem::path data_directory = silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory;
uint32_t max_connections = 64;
uint32_t parallel_threads = 4;
uint16_t port = 8081;

static RuntimeConfig readFromFile(const std::filesystem::path& config_path);
void overwriteFromFile(const std::filesystem::path& config_path);
void overwriteFromCommandLineArguments(const Poco::Util::AbstractConfiguration& config);
};

} // namespace silo_api
158 changes: 70 additions & 88 deletions src/silo_api/api.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
static const std::string ESTIMATED_STARTUP_TIME_IN_MINUTES_OPTION = "estimatedStartupTimeInMinutes";
static const std::string PREPROCESSING_CONFIG_OPTION = "preprocessingConfig";
static const std::string DATABASE_CONFIG_OPTION = "databaseConfig";
static const std::string DATA_DIRECTORY_OPTION = "dataDirectory";
static const std::string API_OPTION = "api";
static const std::string PREPROCESSING_OPTION = "preprocessing";

Expand Down Expand Up @@ -76,107 +75,91 @@ silo::config::DatabaseConfig databaseConfig(const Poco::Util::AbstractConfigurat
return silo::config::ConfigRepository().getValidatedConfig("database_config.yaml");
}

std::filesystem::path dataDirectory(
const Poco::Util::AbstractConfiguration& config,
const silo_api::RuntimeConfig& runtime_config
) {
if (config.hasProperty(DATA_DIRECTORY_OPTION)) {
SPDLOG_DEBUG(
"Using dataDirectory passed via command line argument: {}",
config.getString(DATA_DIRECTORY_OPTION)
);
return config.getString(DATA_DIRECTORY_OPTION);
}
if (runtime_config.data_directory.has_value()) {
SPDLOG_DEBUG(
"Using dataDirectory from runtime config file: {}",
runtime_config.data_directory.value().string()
);
return runtime_config.data_directory.value();
}

SPDLOG_DEBUG(
"dataDirectory not found in specified. Using default value: {}",
silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory
);
return silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory;
}

class SiloServer : public Poco::Util::ServerApplication {
protected:
[[maybe_unused]] void defineOptions(Poco::Util::OptionSet& options) override {
ServerApplication::defineOptions(options);

options.addOption(
Poco::Util::Option("help", "h", "display help information on command line arguments")
Poco::Util::Option()
.fullName("help")
.shortName("h")
.description("display help information on command line arguments")
.required(false)
.repeatable(false)
.callback(Poco::Util::OptionCallback<SiloServer>(this, &SiloServer::displayHelp))
);

options.addOption(
Poco::Util::Option(
PREPROCESSING_CONFIG_OPTION, "pc", "path to the preprocessing config file"
)
.required(false)
.repeatable(false)
.argument("PATH")
.binding(PREPROCESSING_CONFIG_OPTION)
);
options.addOption(Poco::Util::Option()
.fullName(PREPROCESSING_CONFIG_OPTION)
.description("path to the preprocessing config file")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(PREPROCESSING_CONFIG_OPTION));

options.addOption(
Poco::Util::Option(DATABASE_CONFIG_OPTION, "dc", "path to the database config file")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(DATABASE_CONFIG_OPTION)
);
options.addOption(Poco::Util::Option()
.fullName(DATABASE_CONFIG_OPTION)
.description("path to the database config file")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(DATABASE_CONFIG_OPTION));

options.addOption(
Poco::Util::Option(DATA_DIRECTORY_OPTION, "d", "path to the preprocessed data")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(DATA_DIRECTORY_OPTION)
);
options.addOption(Poco::Util::Option()
.fullName(silo_api::DATA_DIRECTORY_OPTION)
.shortName("d")
.description("path to the preprocessed data")
.required(false)
.repeatable(false)
.argument("PATH")
.binding(silo_api::DATA_DIRECTORY_OPTION));

options.addOption(Poco::Util::Option(
"maxQueuedHttpConnections", "mqhc", "maximum number of http connections"
)
options.addOption(Poco::Util::Option()
.fullName(silo_api::PORT_OPTION)
.description("port to listen to requests")
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding(silo_api::PORT_OPTION));

options.addOption(Poco::Util::Option()
.fullName("maxQueuedHttpConnections")
.description("maximum number of http connections")
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding("maxQueuedHttpConnections"));

options.addOption(
Poco::Util::Option(
"threadsForHttpConnections", "tfhc", "number of threads for http connections"
)
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding("threadsForHttpConnections")
);
options.addOption(Poco::Util::Option()
.fullName("threadsForHttpConnections")
.description("number of threads for http connections")
.required(false)
.repeatable(false)
.argument("NUMBER")
.binding("threadsForHttpConnections"));

options.addOption(Poco::Util::Option()
.fullName(API_OPTION)
.shortName("a")
.description("Execution mode: start the SILO web interface")
.required(false)
.repeatable(false)
.binding(API_OPTION)
.group("executionMode"));

options.addOption(
Poco::Util::Option(API_OPTION, "a", "Execution mode: start the SILO web interface")
Poco::Util::Option()
.fullName(PREPROCESSING_OPTION)
.shortName("p")
.description("Execution mode: trigger the preprocessing pipeline to generate a "
"partitioned dataset that can be read by the database")
.required(false)
.repeatable(false)
.binding(API_OPTION)
.binding(PREPROCESSING_OPTION)
.group("executionMode")
);

options.addOption(Poco::Util::Option(
PREPROCESSING_OPTION,
"p",
"Execution mode: trigger the preprocessing pipeline to generate a "
"partitioned dataset that can be read by the database"
)
.required(false)
.repeatable(false)
.binding(PREPROCESSING_OPTION)
.group("executionMode"));

options.addOption(
Poco::Util::Option(
ESTIMATED_STARTUP_TIME_IN_MINUTES_OPTION,
Expand Down Expand Up @@ -209,8 +192,7 @@ class SiloServer : public Poco::Util::ServerApplication {
return handlePreprocessing();
}

std::cout << "No execution mode specified."
<< "\n\n";
std::cout << "No execution mode specified.\n\n";
displayHelp("", "");
return Application::EXIT_USAGE;
}
Expand All @@ -234,25 +216,25 @@ class SiloServer : public Poco::Util::ServerApplication {

silo_api::RuntimeConfig runtime_config;
if (std::filesystem::exists("./runtime_config.yaml")) {
runtime_config = silo_api::RuntimeConfig::readFromFile("./runtime_config.yaml");
runtime_config.overwriteFromFile("./runtime_config.yaml");
}

const auto data_directory = dataDirectory(config(), runtime_config);
runtime_config.overwriteFromCommandLineArguments(config());

silo_api::DatabaseMutex database_mutex;

const Poco::Net::ServerSocket server_socket(port);

const silo_api::DatabaseDirectoryWatcher watcher(data_directory, database_mutex);
const silo_api::DatabaseDirectoryWatcher watcher(
runtime_config.data_directory, database_mutex
);

auto* const poco_parameter = new Poco::Net::HTTPServerParams;
const auto max_connections = config().getInt("maxQueuedHttpConnections", 64);
SPDLOG_INFO("Using {} queued http connections", max_connections);
poco_parameter->setMaxQueued(max_connections);

const auto threads = config().getInt("threadsForHttpConnections", 4);
SPDLOG_INFO("Using {} threads for http connections", threads);
poco_parameter->setMaxThreads(threads);
SPDLOG_INFO("Using {} queued http connections", runtime_config.max_connections);
poco_parameter->setMaxQueued(runtime_config.max_connections);

SPDLOG_INFO("Using {} threads for http connections", runtime_config.parallel_threads);
poco_parameter->setMaxThreads(runtime_config.parallel_threads);

Poco::Net::HTTPServer server(
new silo_api::SiloRequestHandlerFactory(database_mutex, getStartupConfig()),
Expand Down
79 changes: 60 additions & 19 deletions src/silo_api/runtime_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,35 +6,76 @@
#include <spdlog/spdlog.h>
#include <yaml-cpp/yaml.h>

namespace YAML {

template <>
struct convert<silo_api::RuntimeConfig> {
static bool decode(const Node& node, silo_api::RuntimeConfig& config) {
config = silo_api::RuntimeConfig{
node["dataDirectory"]
? std::optional<std::filesystem::path>(node["dataDirectory"].as<std::string>())
: std::nullopt
};

return true;
}
};

} // namespace YAML

namespace silo_api {

RuntimeConfig RuntimeConfig::readFromFile(const std::filesystem::path& config_path) {
void RuntimeConfig::overwriteFromFile(const std::filesystem::path& config_path) {
SPDLOG_INFO("Reading runtime config from {}", config_path.string());

try {
return YAML::LoadFile(config_path.string()).as<RuntimeConfig>();
YAML::Node node = YAML::LoadFile(config_path.string());
if (node[DATA_DIRECTORY_OPTION]) {
SPDLOG_DEBUG(
"Using dataDirectory passed via config file: {}",
node[DATA_DIRECTORY_OPTION].as<std::string>()
);
data_directory = node[DATA_DIRECTORY_OPTION].as<std::string>();
}
if (node[MAX_CONNECTIONS_OPTION]) {
SPDLOG_DEBUG(
"Using maximum queued http connections passed via config file: {}",
node[MAX_CONNECTIONS_OPTION].as<uint32_t>()
);
max_connections = node[MAX_CONNECTIONS_OPTION].as<uint32_t>();
}
if (node[PARALLEL_THREADS_OPTION]) {
SPDLOG_DEBUG(
"Using parallel threads for accepting http connections as passed via config file: {}",
node[PARALLEL_THREADS_OPTION].as<uint32_t>()
);
parallel_threads = node[PARALLEL_THREADS_OPTION].as<uint32_t>();
}
if (node[PORT_OPTION]) {
SPDLOG_DEBUG("Using port passed via config file: {}", node[PORT_OPTION].as<uint16_t>());
port = node[PORT_OPTION].as<uint16_t>();
}
} catch (const YAML::Exception& e) {
throw std::runtime_error(
"Failed to read runtime config from " + config_path.string() + ": " + std::string(e.what())
);
}
}

void RuntimeConfig::overwriteFromCommandLineArguments(
const Poco::Util::AbstractConfiguration& config
) {
if (config.hasProperty(DATA_DIRECTORY_OPTION)) {
SPDLOG_DEBUG(
"Using dataDirectory passed via command line argument: {}",
config.getString(DATA_DIRECTORY_OPTION)
);
data_directory = config.getString(DATA_DIRECTORY_OPTION);
}
if (config.hasProperty(MAX_CONNECTIONS_OPTION)) {
SPDLOG_DEBUG(
"Using maximum queued http connections passed via command line argument: {}",
config.getUInt(MAX_CONNECTIONS_OPTION)
);
max_connections = config.getUInt(MAX_CONNECTIONS_OPTION);
}
if (config.hasProperty(PARALLEL_THREADS_OPTION)) {
SPDLOG_DEBUG(
"Using parallel threads for accepting http connections as passed via command line "
"argument: {}",
config.getUInt(PARALLEL_THREADS_OPTION)
);
parallel_threads = config.getUInt(PARALLEL_THREADS_OPTION);
}
if (config.hasProperty(PORT_OPTION)) {
SPDLOG_DEBUG(
"Using port passed via command line argument: {}", config.getString(PORT_OPTION)
);
port = config.getUInt(PORT_OPTION);
}
}

} // namespace silo_api
6 changes: 3 additions & 3 deletions src/silo_api/runtime_config.test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
#include <gtest/gtest.h>

TEST(RuntimeConfig, shouldReadConfig) {
const auto result =
silo_api::RuntimeConfig::readFromFile("./testBaseData/test_runtime_config.yaml");
silo_api::RuntimeConfig runtime_config;
runtime_config.overwriteFromFile("./testBaseData/test_runtime_config.yaml");

ASSERT_EQ(result.data_directory, std::filesystem::path("test/directory"));
ASSERT_EQ(runtime_config.data_directory, std::filesystem::path("test/directory"));
}

0 comments on commit df8a9e7

Please sign in to comment.