diff --git a/compiler/code-gen/code-generator.h b/compiler/code-gen/code-generator.h index d45cca6073..285aa3aef4 100644 --- a/compiler/code-gen/code-generator.h +++ b/compiler/code-gen/code-generator.h @@ -19,6 +19,7 @@ struct CGContext { std::vector catch_label_used; FunctionPtr parent_func; bool resumable_flag{false}; + bool interruptible_flag{false}; bool namespace_opened{false}; int inside_macro{0}; size_t inside_null_coalesce_fallback{0}; diff --git a/compiler/code-gen/declarations.cpp b/compiler/code-gen/declarations.cpp index 6692a35e23..31a3a97b6e 100644 --- a/compiler/code-gen/declarations.cpp +++ b/compiler/code-gen/declarations.cpp @@ -72,7 +72,11 @@ void FunctionDeclaration::compile(CodeGenerator &W) const { switch (style) { case gen_out_style::tagger: case gen_out_style::cpp: { - FunctionSignatureGenerator(W) << ret_type_gen << " " << FunctionName(function) << "(" << params_gen << ")"; + if (function->is_interruptible) { + FunctionSignatureGenerator(W) << "task_t<" << ret_type_gen << ">" << " " << FunctionName(function) << "(" << params_gen << ")"; + } else { + FunctionSignatureGenerator(W) << ret_type_gen << " " << FunctionName(function) << "(" << params_gen << ")"; + } break; } case gen_out_style::txt: { diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index a8d2d4fedb..8e6398c63d 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -40,25 +40,30 @@ void StaticInit::compile(CodeGenerator &W) const { W << "extern array gen$tl_storers_ht;" << NL; FunctionSignatureGenerator(W) << "void fill_tl_storers_ht()" << SemicolonAndNL() << NL; } - FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN << "return " << RawString(G->settings().php_code_version.get()) << ";" - << NL << END << NL << NL; + if (!G->is_output_mode_k2_component()) { + FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN << "return " << RawString(G->settings().php_code_version.get()) << ";" + << NL << END << NL << NL; + } - FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; - const auto &runtime_opts = G->get_kphp_runtime_opts(); - if (runtime_opts.empty()) { - W << "return nullptr;" << NL; - } else { - W << "*count = " << runtime_opts.size() << ";" << NL; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; - } - W << "static char *argv[] = " << BEGIN; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "arg" << i << "," << NL; + if (!G->is_output_mode_k2_component()) { + FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; + + const auto &runtime_opts = G->get_kphp_runtime_opts(); + if (runtime_opts.empty()) { + W << "return nullptr;" << NL; + } else { + W << "*count = " << runtime_opts.size() << ";" << NL; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; + } + W << "static char *argv[] = " << BEGIN; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "arg" << i << "," << NL; + } + W << END << ";" << NL << "return argv;" << NL; } - W << END << ";" << NL << "return argv;" << NL; + W << END << NL << NL; } - W << END << NL << NL; FunctionSignatureGenerator(W) << ("void init_php_scripts_once_in_master() ") << BEGIN; @@ -102,6 +107,20 @@ void StaticInit::compile(CodeGenerator &W) const { W << END << NL; } +struct RunInterruptedFunction { + FunctionPtr function; + RunInterruptedFunction(FunctionPtr function) : function(function) {} + + void compile(CodeGenerator &W) const { + std::string await_prefix = function->is_interruptible ? "co_await " : ""; + FunctionSignatureGenerator(W) << "task_t " << FunctionName(function) << "$run() " << BEGIN + << await_prefix << FunctionName(function) << "();" << NL + << "co_return;" + << END; + W << NL; + } +}; + struct RunFunction { FunctionPtr function; RunFunction(FunctionPtr function) : function(function) {} @@ -178,8 +197,11 @@ InitScriptsCpp::InitScriptsCpp(SrcFilePtr main_file_id) : void InitScriptsCpp::compile(CodeGenerator &W) const { W << OpenFile("init_php_scripts.cpp", "", false); - W << ExternInclude(G->settings().runtime_headers.get()) << - ExternInclude("server/php-init-scripts.h"); + W << ExternInclude(G->settings().runtime_headers.get()); + if (!G->is_output_mode_k2_component()) { + W << ExternInclude("server/php-init-scripts.h"); + } + W << Include(main_file_id->main_function->header_full_name); @@ -196,10 +218,18 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { return; } - W << RunFunction(main_file_id->main_function) << NL; + if (G->is_output_mode_k2_component()) { + W << RunInterruptedFunction(main_file_id->main_function) << NL; + } else { + W << RunFunction(main_file_id->main_function) << NL; + } W << GlobalsResetFunction(main_file_id->main_function) << NL; - FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + if (G->is_output_mode_k2_component()) { + FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ", task_t&run" ")" << BEGIN; + } else { + FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + } W << "global_vars_allocate(php_globals);" << NL; for (LibPtr lib: G->get_libs()) { @@ -211,9 +241,13 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; - W << "set_script (" - << FunctionName(main_file_id->main_function) << "$run, " - << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; + if (G->is_output_mode_k2_component()) { + W << "run = " << FunctionName(main_file_id->main_function) << "$run();" << NL; + } else { + W << "set_script (" + << FunctionName(main_file_id->main_function) << "$run, " + << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; + } W << END; @@ -239,3 +273,19 @@ void CppMainFile::compile(CodeGenerator &W) const { << END; W << CloseFile(); } + +void ComponentInfoFile::compile(CodeGenerator &W) const { + kphp_assert(G->is_output_mode_k2_component()); + G->settings().get_version(); + auto now = std::chrono::system_clock::now(); + W << OpenFile("image_info.cpp"); + W << ExternInclude(G->settings().runtime_headers.get()); + W << "const ImageInfo *vk_k2_describe() " << BEGIN + << "static ImageInfo imageInfo {\"" << G->settings().k2_component_name.get() << "\"" << "," + << std::chrono::duration_cast(now.time_since_epoch()).count() << "," + << "K2_PLATFORM_HEADER_H_VERSION, " + << "{" << "}};" << NL //todo:k2 add commit hash + << "return &imageInfo;" << NL + << END; + W << CloseFile(); +} diff --git a/compiler/code-gen/files/init-scripts.h b/compiler/code-gen/files/init-scripts.h index 80bef5b327..1c476fbf7c 100644 --- a/compiler/code-gen/files/init-scripts.h +++ b/compiler/code-gen/files/init-scripts.h @@ -23,3 +23,7 @@ struct LibVersionHFile : CodeGenRootCmd { struct CppMainFile : CodeGenRootCmd { void compile(CodeGenerator &W) const final; }; + +struct ComponentInfoFile : CodeGenRootCmd { + void compile(CodeGenerator &W) const final; +}; diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index d75ce37105..71ed0feed1 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -631,6 +631,7 @@ void compile_do(VertexAdaptor root, CodeGenerator &W) { void compile_return(VertexAdaptor root, CodeGenerator &W) { bool resumable_flag = W.get_context().resumable_flag; + bool interruptible_flag = W.get_context().interruptible_flag; if (resumable_flag) { if (root->has_expr()) { W << "RETURN " << MacroBegin{}; @@ -638,7 +639,11 @@ void compile_return(VertexAdaptor root, CodeGenerator &W) { W << "RETURN_VOID " << MacroBegin{}; } } else { - W << "return "; + if (interruptible_flag) { + W << "co_return "; + } else { + W << "return "; + } } if (root->has_expr()) { @@ -841,6 +846,9 @@ void compile_func_call(VertexAdaptor root, CodeGenerator &W, func_ if (mode == func_call_mode::fork_call) { W << FunctionForkName(func); } else { + if (func->is_interruptible) { + W << "(" << "co_await "; + } W << FunctionName(func); } } @@ -865,6 +873,9 @@ void compile_func_call(VertexAdaptor root, CodeGenerator &W, func_ W << JoinValues(args, ", "); W << ")"; + if (func->is_interruptible) { + W << ")"; + } } void compile_func_call_fast(VertexAdaptor root, CodeGenerator &W) { @@ -1448,6 +1459,7 @@ void compile_function(VertexAdaptor func_root, CodeGenerator &W) { W.get_context().parent_func = func; W.get_context().resumable_flag = func->is_resumable; + W.get_context().interruptible_flag = func->is_interruptible; if (func->is_resumable) { compile_function_resumable(func_root, W); diff --git a/compiler/compiler-core.cpp b/compiler/compiler-core.cpp index 014d04ec89..0c1fee7386 100644 --- a/compiler/compiler-core.cpp +++ b/compiler/compiler-core.cpp @@ -126,6 +126,8 @@ void CompilerCore::register_settings(CompilerSettings *settings) { output_mode = OutputMode::cli; } else if (settings->mode.get() == "lib") { output_mode = OutputMode::lib; + } else if (settings->mode.get() == "k2-component") { + output_mode = OutputMode::k2_component; } else { output_mode = OutputMode::server; } diff --git a/compiler/compiler-core.h b/compiler/compiler-core.h index 944f0fec47..e3fd2dd805 100644 --- a/compiler/compiler-core.h +++ b/compiler/compiler-core.h @@ -28,6 +28,7 @@ enum class OutputMode { server, // -M server cli, // -M cli lib, // -M lib + k2_component // -M k2-component }; class CompilerCore { @@ -176,6 +177,10 @@ class CompilerCore { return output_mode == OutputMode::lib; } + bool is_output_mode_k2_component() const { + return output_mode == OutputMode::k2_component; + } + Stats stats; }; diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 67560eeca1..b17917bab5 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -216,6 +216,13 @@ void CompilerSettings::init() { option_as_dir(kphp_src_path); functions_file.value_ = get_full_path(functions_file.get()); runtime_sha256_file.value_ = get_full_path(runtime_sha256_file.get()); + if (link_file.value_.empty()) { + if (mode.get() == "k2-component") { + link_file.value_ = kphp_src_path.get() + "/objs/libkphp-light-runtime.a"; + } else { + link_file.value_ = kphp_src_path.get() + "/objs/libkphp-full-runtime.a"; + } + } link_file.value_ = get_full_path(link_file.get()); if (mode.get() == "lib") { @@ -283,7 +290,7 @@ void CompilerSettings::init() { if (!no_pch.get()) { ss << " -Winvalid-pch -fpch-preprocess"; } - if (dynamic_incremental_linkage.get()) { + if (mode.get() == "k2-component" || dynamic_incremental_linkage.get()) { ss << " -fPIC"; } if (vk::contains(cxx.get(), "clang")) { @@ -386,7 +393,11 @@ void CompilerSettings::init() { option_as_dir(dest_dir); dest_cpp_dir.value_ = dest_dir.get() + "kphp/"; dest_objs_dir.value_ = dest_dir.get() + "objs/"; - binary_path.value_ = dest_dir.get() + mode.get(); + if (mode.get() == "k2-component") { + binary_path.value_ = dest_dir.get() + k2_component_name.get() + ".so"; + } else { + binary_path.value_ = dest_dir.get() + mode.get(); + } performance_analyze_report_path.value_ = dest_dir.get() + "performance_issues.json"; generated_runtime_path.value_ = kphp_src_path.get() + "objs/generated/auto/runtime/"; diff --git a/compiler/compiler-settings.h b/compiler/compiler-settings.h index dc00a23c08..f1cd813bb2 100644 --- a/compiler/compiler-settings.h +++ b/compiler/compiler-settings.h @@ -147,6 +147,7 @@ class CompilerSettings : vk::not_copyable { KphpOption compilation_metrics_file; KphpOption override_kphp_version; KphpOption php_code_version; + KphpOption k2_component_name; KphpOption cxx; KphpOption cxx_toolchain_dir; diff --git a/compiler/data/function-data.h b/compiler/data/function-data.h index bfe1139501..3962e49b62 100644 --- a/compiler/data/function-data.h +++ b/compiler/data/function-data.h @@ -117,6 +117,7 @@ class FunctionData { bool cpp_template_call = false; bool cpp_variadic_call = false; bool is_resumable = false; + bool is_interruptible = false; bool can_be_implicitly_interrupted_by_other_resumable = false; bool is_virtual_method = false; bool is_overridden_method = false; diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 33589f832e..5eadfe6c27 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -215,10 +215,10 @@ int main(int argc, char *argv[]) { 'f', "functions-file", "KPHP_FUNCTIONS", "${KPHP_PATH}/builtin-functions/kphp-full/_functions.txt"); parser.add("File with kphp runtime sha256 hash", settings->runtime_sha256_file, "runtime-sha256", "KPHP_RUNTIME_SHA256", "${KPHP_PATH}/objs/php_lib_version.sha256"); - parser.add("The output binary type: server, cli or lib", settings->mode, - 'M', "mode", "KPHP_MODE", "server", {"server", "cli", "lib"}); + parser.add("The output binary type: server, k2-component, cli or lib", settings->mode, + 'M', "mode", "KPHP_MODE", "server", {"server", "k2-component", "cli", "lib"}); parser.add("A runtime library for building the output binary", settings->link_file, - 'l', "link-with", "KPHP_LINK_FILE", "${KPHP_PATH}/objs/libkphp-full-runtime.a"); + 'l', "link-with", "KPHP_LINK_FILE"); parser.add("Directory where php files will be searched", settings->includes, 'I', "include-dir", "KPHP_INCLUDE_DIR"); parser.add("Destination directory", settings->dest_dir, @@ -293,6 +293,8 @@ int main(int argc, char *argv[]) { "require-functions-typing", "KPHP_REQUIRE_FUNCTIONS_TYPING"); parser.add("Require class typing (1 - @var / default value is mandatory, 0 - auto infer or check if exists)", settings->require_class_typing, "require-class-typing", "KPHP_REQUIRE_CLASS_TYPING"); + parser.add("Define k2 component name. Default is \"KPHP\"", settings->k2_component_name, + "k2-component-name", "KPHP_K2_COMPONENT_NAME", "KPHP"); parser.add_implicit_option("Linker flags", settings->ld_flags); parser.add_implicit_option("Incremental linker flags", settings->incremental_linker_flags); diff --git a/compiler/make/make.cpp b/compiler/make/make.cpp index 988d910585..2728dfd59e 100644 --- a/compiler/make/make.cpp +++ b/compiler/make/make.cpp @@ -22,6 +22,7 @@ #include "compiler/make/make-runner.h" #include "compiler/make/objs-to-bin-target.h" #include "compiler/make/objs-to-obj-target.h" +#include "compiler/make/objs-to-k2-component-target.h" #include "compiler/make/objs-to-static-lib-target.h" #include "compiler/stage.h" #include "compiler/threading/profiler.h" @@ -98,6 +99,10 @@ class MakeSetup { return create_target(new Objs2StaticLibTarget, to_targets(std::move(objs)), lib); } + Target *create_objs2k2_component_target(std::vector objs, File *lib) { + return create_target(new Objs2K2ComponentTarget, to_targets(std::move(objs)), lib); + } + bool make_target(File *bin, const std::string &build_message, int jobs_count) { return make.make_targets(to_targets(bin), build_message, jobs_count); } @@ -177,6 +182,7 @@ static long long get_imported_header_mtime(const std::string &header_path, const return 0; } + // prepare dir kphp_out/objs/pch_{flags} and make a target runtime-headers.h.gch inside it // in production, there will be two pch_ folders: with debug symbols and without them // later on, we'll copy this folder inside /tmp/kphp_pch — that's why if it exists, don't do anything @@ -431,6 +437,8 @@ void run_make() { if (output_mode == OutputMode::lib) { make.create_objs2static_lib_target(objs, &bin_file); + } else if (output_mode == OutputMode::k2_component) { + make.create_objs2k2_component_target(objs, &bin_file); } else { const std::string build_stage{"Compiling"}; AutoProfiler profiler{get_profiler(build_stage)}; diff --git a/compiler/make/objs-to-k2-component-target.h b/compiler/make/objs-to-k2-component-target.h new file mode 100644 index 0000000000..9c8cf75c43 --- /dev/null +++ b/compiler/make/objs-to-k2-component-target.h @@ -0,0 +1,44 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "compiler/compiler-settings.h" +#include "compiler/make/target.h" + +class Objs2K2ComponentTarget : public Target { + static std::string load_all_symbols_pre() { +#if defined(__APPLE__) + return "-Wl,-force_load "; +#else + return "-Wl,--whole-archive "; +#endif + } + + static std::string load_all_symbols_post() { +#if defined(__APPLE__) + return " "; +#else + return " -Wl,--no-whole-archive "; +#endif + } + +public: + std::string get_cmd() final { + std::stringstream ss; + ss << settings->cxx.get() << " -shared -o " << target() << " "; + + for (size_t i = 0; i + 1 < deps.size(); ++i) { + ss << deps[i]->get_name() << " "; + } + + // the last dep is runtime lib + // todo:k2 think about kphp-libraries + assert(deps.size() >= 1 && "There are should be at least one dependency. It's the runtime lib"); + ss << load_all_symbols_pre() << deps.back()->get_name() << load_all_symbols_post(); + return ss.str(); + } +}; diff --git a/compiler/pipes/calc-bad-vars.cpp b/compiler/pipes/calc-bad-vars.cpp index b539b17df4..032dec6f69 100644 --- a/compiler/pipes/calc-bad-vars.cpp +++ b/compiler/pipes/calc-bad-vars.cpp @@ -531,6 +531,22 @@ class CalcBadVars { } } + static void calc_interruptible(const FuncCallGraph &call_graph) { + IdMap into_interruptible(call_graph.n); + IdMap to_parents(call_graph.n); + + for (const auto &func : call_graph.functions) { + if (func->is_interruptible) { + mark(call_graph.rev_graph, into_interruptible, func, to_parents); + } + } + + for (const auto &func : call_graph.functions) { + if (into_interruptible[func]) { + func->is_interruptible = true; + } + } + } static void calc_resumable(const FuncCallGraph &call_graph, const std::vector &dep_data) { for (int i = 0; i < call_graph.n; i++) { @@ -668,6 +684,7 @@ class CalcBadVars { { FuncCallGraph call_graph(std::move(functions), dep_datas); + calc_interruptible(call_graph); calc_resumable(call_graph, dep_datas); generate_bad_vars(call_graph, dep_datas); check_func_colors(call_graph); diff --git a/compiler/pipes/calc-func-dep.cpp b/compiler/pipes/calc-func-dep.cpp index ce7c76049c..b5104b3ecb 100644 --- a/compiler/pipes/calc-func-dep.cpp +++ b/compiler/pipes/calc-func-dep.cpp @@ -55,8 +55,9 @@ VertexPtr CalcFuncDepPass::on_enter_vertex(VertexPtr vertex) { // .dep is used to // 1) build call graph — we actually need only user-defined function // 2) build resumable call graph — to propagate resumable state and to calc resumable chains + // 3) build interruptible call graph - calc interruptible chains // adding all built-in calls won't affect codegen, it will just overhead call graphs and execution time - if (!other_function->is_extern() || other_function->is_resumable || other_function->is_imported_from_static_lib()) { + if (!other_function->is_extern() || other_function->is_resumable || other_function->is_imported_from_static_lib() || other_function->is_interruptible) { data.dep.push_back(other_function); } calls.push_back(other_function); diff --git a/compiler/pipes/code-gen.cpp b/compiler/pipes/code-gen.cpp index 500873ab66..e4db43bf9f 100644 --- a/compiler/pipes/code-gen.cpp +++ b/compiler/pipes/code-gen.cpp @@ -117,7 +117,10 @@ void CodeGenF::on_finish(DataStream> &os) { // TODO: should be done in lib mode also, but in some other way if (!G->is_output_mode_lib()) { - code_gen_start_root_task(os, std::make_unique(vk::singleton::get().flush_forkable_types(), vk::singleton::get().flush_waitable_types())); + if (!G->is_output_mode_k2_component()) { + code_gen_start_root_task(os, std::make_unique(vk::singleton::get().flush_forkable_types(), + vk::singleton::get().flush_waitable_types())); + } code_gen_start_root_task(os, std::make_unique(TypeHintShape::get_all_registered_keys())); code_gen_start_root_task(os, std::make_unique(std::move(all_json_encoders))); code_gen_start_root_task(os, std::make_unique()); @@ -129,9 +132,12 @@ void CodeGenF::on_finish(DataStream> &os) { code_gen_start_root_task(os, std::make_unique()); code_gen_start_root_task(os, std::make_unique()); - if (!G->is_output_mode_lib()) { + if (!G->is_output_mode_lib() && !G->is_output_mode_k2_component()) { code_gen_start_root_task(os, std::make_unique()); } + if (G->is_output_mode_k2_component()) { + code_gen_start_root_task(os, std::make_unique()); + } } void CodeGenF::prepare_generate_function(FunctionPtr func) { diff --git a/compiler/pipes/parse-and-apply-phpdoc.cpp b/compiler/pipes/parse-and-apply-phpdoc.cpp index 12c2eb2070..561dc194f6 100644 --- a/compiler/pipes/parse-and-apply-phpdoc.cpp +++ b/compiler/pipes/parse-and-apply-phpdoc.cpp @@ -281,6 +281,8 @@ class ParseAndApplyPhpDocForFunction { f_->cpp_variadic_call = true; } else if (token == "tl_common_h_dep") { f_->tl_common_h_dep = true; + } else if (token == "interruptible") { + f_->is_interruptible = true; } else { kphp_error(0, fmt_format("Unknown @kphp-extern-func-info {}", token)); }