From ec22ec87b6c4695830735f17ca432902f9eac0ff Mon Sep 17 00:00:00 2001 From: Aleksandr Kirsanov Date: Tue, 18 Jun 2024 18:15:48 +0300 Subject: [PATCH] Codegen globals as a linear memory piece, not as C++ variables (#1011) Before this MR, all constants / globals were codegenerated as separate C++ variables. This MR splits constants and globals into different storages and pipelines and targets to fully avoid mutable state in codegenerated C++ code. The purpose of this approach is to be able to compile a script into .so, load it multiple times, and execute concurrently. --- builtin-functions/_functions.txt | 2 +- common/php-functions.h | 35 +- .../code-gen/const-globals-batched-mem.cpp | 339 ++++++++++++++++++ compiler/code-gen/const-globals-batched-mem.h | 162 +++++++++ compiler/code-gen/declarations.cpp | 31 +- compiler/code-gen/declarations.h | 2 - compiler/code-gen/files/const-vars-init.cpp | 171 +++++++++ compiler/code-gen/files/const-vars-init.h | 21 ++ compiler/code-gen/files/function-header.cpp | 10 +- compiler/code-gen/files/function-source.cpp | 21 +- compiler/code-gen/files/function-source.h | 1 - .../files/global-vars-memory-stats.cpp | 111 ++++++ .../code-gen/files/global-vars-memory-stats.h | 23 ++ compiler/code-gen/files/global-vars-reset.cpp | 98 +++++ compiler/code-gen/files/global-vars-reset.h | 22 ++ .../files/global_vars_memory_stats.cpp | 97 ----- .../code-gen/files/global_vars_memory_stats.h | 21 -- compiler/code-gen/files/init-scripts.cpp | 152 ++++---- compiler/code-gen/files/shape-keys.cpp | 6 +- compiler/code-gen/files/shape-keys.h | 1 - compiler/code-gen/files/vars-cpp.cpp | 204 ----------- compiler/code-gen/files/vars-cpp.h | 28 -- compiler/code-gen/files/vars-reset.cpp | 107 ------ compiler/code-gen/files/vars-reset.h | 25 -- compiler/code-gen/naming.h | 29 +- compiler/code-gen/raw-data.h | 29 +- compiler/code-gen/vertex-compiler.cpp | 72 +++- compiler/compiler-core.cpp | 109 ++++-- compiler/compiler-core.h | 34 +- compiler/compiler-settings.cpp | 18 +- compiler/compiler-settings.h | 4 - compiler/compiler.cmake | 10 +- compiler/data/class-members.cpp | 2 +- compiler/data/function-data.h | 1 + compiler/data/var-data.cpp | 22 +- compiler/data/var-data.h | 12 +- compiler/data/vars-collector.cpp | 55 --- compiler/data/vars-collector.h | 27 -- compiler/kphp2cpp.cpp | 2 - compiler/lambda-utils.cpp | 2 +- compiler/make/make.cpp | 15 +- compiler/name-gen.cpp | 28 -- compiler/name-gen.h | 6 - compiler/pipes/analyzer.cpp | 6 + compiler/pipes/calc-bad-vars.cpp | 2 +- compiler/pipes/calc-empty-functions.cpp | 9 +- compiler/pipes/calc-locations.cpp | 22 +- compiler/pipes/code-gen.cpp | 69 ++-- compiler/pipes/code-gen.h | 1 - compiler/pipes/collect-const-vars.cpp | 34 +- compiler/pipes/convert-sprintf-calls.cpp | 8 +- compiler/pipes/convert-sprintf-calls.h | 2 +- compiler/pipes/final-check.cpp | 32 +- compiler/pipes/gen-tree-postprocess.cpp | 8 +- compiler/pipes/generate-virtual-methods.cpp | 2 +- compiler/pipes/optimization.cpp | 2 +- compiler/pipes/register-variables.cpp | 2 +- .../pipes/remove-empty-function-calls.cpp | 2 +- compiler/threading/hash-table.h | 1 + compiler/vertex-util.cpp | 13 - compiler/vertex-util.h | 1 - .../kphp-vs-php/compiler-cmd-options.md | 4 - runtime/array_functions.cpp | 3 + runtime/interface.cpp | 186 +++++----- runtime/interface.h | 13 +- runtime/memory_usage.cpp | 14 + runtime/memory_usage.h | 10 +- runtime/mixed.cpp | 2 + runtime/php-script-globals.cpp | 44 +++ runtime/php-script-globals.h | 49 +++ runtime/regexp.cpp | 1 + runtime/runtime.cmake | 2 + runtime/string.cpp | 2 + server/php-engine.cpp | 3 +- server/php-init-scripts.cpp | 2 +- server/php-init-scripts.h | 14 +- server/php-runner.cpp | 6 +- tests/cpp/runtime/_runtime-tests-env.cpp | 9 +- tests/phpt/constants/010_arrow_access.php | 3 +- tests/phpt/dl/1043_some_globals.php | 291 +++++++++++++++ tests/python/tests/ffi/test_ffi.py | 2 +- .../php/lib_examples/example1/php/index.php | 7 + tests/python/tests/libs/php/lib_user.php | 2 + 83 files changed, 1910 insertions(+), 1142 deletions(-) create mode 100644 compiler/code-gen/const-globals-batched-mem.cpp create mode 100644 compiler/code-gen/const-globals-batched-mem.h create mode 100644 compiler/code-gen/files/const-vars-init.cpp create mode 100644 compiler/code-gen/files/const-vars-init.h create mode 100644 compiler/code-gen/files/global-vars-memory-stats.cpp create mode 100644 compiler/code-gen/files/global-vars-memory-stats.h create mode 100644 compiler/code-gen/files/global-vars-reset.cpp create mode 100644 compiler/code-gen/files/global-vars-reset.h delete mode 100644 compiler/code-gen/files/global_vars_memory_stats.cpp delete mode 100644 compiler/code-gen/files/global_vars_memory_stats.h delete mode 100644 compiler/code-gen/files/vars-cpp.cpp delete mode 100644 compiler/code-gen/files/vars-cpp.h delete mode 100644 compiler/code-gen/files/vars-reset.cpp delete mode 100644 compiler/code-gen/files/vars-reset.h delete mode 100644 compiler/data/vars-collector.cpp delete mode 100644 compiler/data/vars-collector.h create mode 100644 runtime/memory_usage.cpp create mode 100644 runtime/php-script-globals.cpp create mode 100644 runtime/php-script-globals.h create mode 100644 tests/phpt/dl/1043_some_globals.php diff --git a/builtin-functions/_functions.txt b/builtin-functions/_functions.txt index ef816fff19..54444f053b 100644 --- a/builtin-functions/_functions.txt +++ b/builtin-functions/_functions.txt @@ -202,7 +202,7 @@ function memory_get_allocations() ::: tuple(int, int); function estimate_memory_usage($value ::: any) ::: int; // to enable this function, set KPHP_ENABLE_GLOBAL_VARS_MEMORY_STATS=1 -function get_global_vars_memory_stats($lower_bound ::: int = 0) ::: int[]; +function get_global_vars_memory_stats(int $lower_bound = 0) ::: int[]; function get_net_time() ::: float; function get_script_time() ::: float; diff --git a/common/php-functions.h b/common/php-functions.h index a90de4d647..8a127b712e 100644 --- a/common/php-functions.h +++ b/common/php-functions.h @@ -33,6 +33,16 @@ constexpr int STRLEN_VOID = STRLEN_ERROR; constexpr int STRLEN_FUTURE = STRLEN_ERROR; constexpr int STRLEN_FUTURE_QUEUE = STRLEN_ERROR; +constexpr int SIZEOF_STRING = 8; +constexpr int SIZEOF_ARRAY_ANY = 8; +constexpr int SIZEOF_MIXED = 16; +constexpr int SIZEOF_INSTANCE_ANY = 8; +constexpr int SIZEOF_OPTIONAL = 8; +constexpr int SIZEOF_FUTURE = 8; +constexpr int SIZEOF_FUTURE_QUEUE = 8; +constexpr int SIZEOF_REGEXP = 48; +constexpr int SIZEOF_UNKNOWN = 1; + class ExtraRefCnt { public: enum extra_ref_cnt_value { @@ -232,31 +242,6 @@ bool php_try_to_int(const char *s, size_t l, int64_t *val) { return true; } -//returns len of raw string representation or -1 on error -inline int string_raw_len(int src_len) { - if (src_len < 0 || src_len >= (1 << 30) - 13) { - return -1; - } - - return src_len + 13; -} - -//returns len of raw string representation and writes it to dest or returns -1 on error -inline int string_raw(char *dest, int dest_len, const char *src, int src_len) { - int raw_len = string_raw_len(src_len); - if (raw_len == -1 || raw_len > dest_len) { - return -1; - } - int *dest_int = reinterpret_cast (dest); - dest_int[0] = src_len; - dest_int[1] = src_len; - dest_int[2] = ExtraRefCnt::for_global_const; - memcpy(dest + 3 * sizeof(int), src, src_len); - dest[3 * sizeof(int) + src_len] = '\0'; - - return raw_len; -} - template inline constexpr int three_way_comparison(const T &lhs, const T &rhs) { return lhs < rhs ? -1 : diff --git a/compiler/code-gen/const-globals-batched-mem.cpp b/compiler/code-gen/const-globals-batched-mem.cpp new file mode 100644 index 0000000000..b1f588c3b3 --- /dev/null +++ b/compiler/code-gen/const-globals-batched-mem.cpp @@ -0,0 +1,339 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/const-globals-batched-mem.h" + +#include +#include + +#include "common/php-functions.h" + +#include "compiler/compiler-core.h" +#include "compiler/code-gen/common.h" +#include "compiler/code-gen/includes.h" +#include "compiler/data/var-data.h" +#include "compiler/inferring/public.h" + +// see const-globals-batched-mem.h for detailed comments of what's going on + +namespace { + +ConstantsBatchedMem constants_batched_mem; +GlobalsBatchedMem globals_batched_mem; + +int calc_sizeof_tuple_shape(const TypeData *type); + +[[gnu::always_inline]] inline int calc_sizeof_in_bytes_runtime(const TypeData *type) { + switch (type->get_real_ptype()) { + case tp_int: + return type->use_optional() ? SIZEOF_OPTIONAL + sizeof(int64_t) : sizeof(int64_t); + case tp_float: + return type->use_optional() ? SIZEOF_OPTIONAL + sizeof(double) : sizeof(double); + case tp_string: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_STRING : SIZEOF_STRING; + case tp_array: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_ARRAY_ANY : SIZEOF_ARRAY_ANY; + case tp_regexp: + kphp_assert(!type->use_optional()); + return SIZEOF_REGEXP; + case tp_Class: + kphp_assert(!type->use_optional()); + return SIZEOF_INSTANCE_ANY; + case tp_mixed: + kphp_assert(!type->use_optional()); + return SIZEOF_MIXED; + case tp_bool: + return type->use_optional() ? 2 : 1; + case tp_tuple: + case tp_shape: + return calc_sizeof_tuple_shape(type); + case tp_future: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_FUTURE : SIZEOF_FUTURE; + case tp_future_queue: + return type->use_optional() ? SIZEOF_OPTIONAL + SIZEOF_FUTURE_QUEUE : SIZEOF_FUTURE_QUEUE; + case tp_any: + return SIZEOF_UNKNOWN; + default: + kphp_error(0, fmt_format("Unable to detect sizeof() for type = {}", type->as_human_readable())); + return 0; + } +} + +[[gnu::noinline]] int calc_sizeof_tuple_shape(const TypeData *type) { + kphp_assert(vk::any_of_equal(type->ptype(), tp_tuple, tp_shape)); + + int result = 0; + bool has_align_8bytes = false; + for (auto sub = type->lookup_begin(); sub != type->lookup_end(); ++sub) { + int sub_sizeof = calc_sizeof_in_bytes_runtime(sub->second); + if (sub_sizeof >= 8) { + has_align_8bytes = true; + result = (result + 7) & -8; + } + result += sub_sizeof; + } + if (has_align_8bytes) { + result = (result + 7) & -8; + } + return type->use_optional() + ? has_align_8bytes ? SIZEOF_OPTIONAL + result : 1 + result + : result; +} + +} // namespace + + +void ConstantsBatchedMem::inc_count_by_type(const TypeData *type) { + if (type->use_optional()) { + count_of_type_other++; + return; + } + switch (type->get_real_ptype()) { + case tp_string: + count_of_type_string++; + break; + case tp_regexp: + count_of_type_regexp++; + break; + case tp_array: + count_of_type_array++; + return; + case tp_mixed: + count_of_type_mixed++; + break; + case tp_Class: + count_of_type_instance++; + break; + default: + count_of_type_other++; + } +} + +int ConstantsBatchedMem::detect_constants_batch_count(int n_constants) { + // these values are heuristics (don't use integer division, to avoid changing buckets count frequently) + if (n_constants > 1200000) return 2048; + if (n_constants > 800000) return 1536; + if (n_constants > 500000) return 1024; + if (n_constants > 100000) return 512; + if (n_constants > 10000) return 256; + if (n_constants > 5000) return 128; + if (n_constants > 1000) return 32; + if (n_constants > 500) return 16; + if (n_constants > 100) return 4; + return 1; +} + +const ConstantsBatchedMem &ConstantsBatchedMem::prepare_mem_and_assign_offsets(const std::vector &all_constants) { + ConstantsBatchedMem &mem = constants_batched_mem; + + const int N_BATCHES = detect_constants_batch_count(all_constants.size()); + mem.batches.resize(N_BATCHES); + + for (VarPtr var : all_constants) { + int batch_idx = static_cast(vk::std_hash(var->name) % N_BATCHES); + var->batch_idx = batch_idx; + mem.batches[batch_idx].n_constants++; + if (var->dependency_level > mem.batches[batch_idx].max_dep_level) { + mem.batches[batch_idx].max_dep_level = var->dependency_level; + } + } + + for (int batch_idx = 0; batch_idx < N_BATCHES; ++batch_idx) { + mem.batches[batch_idx].batch_idx = batch_idx; + mem.batches[batch_idx].constants.reserve(mem.batches[batch_idx].n_constants); + } + + for (VarPtr var : all_constants) { + mem.batches[var->batch_idx].constants.emplace_back(var); + } + + for (OneBatchInfo &batch : mem.batches) { + // sort constants by name to make codegen stable + std::sort(batch.constants.begin(), batch.constants.end(), [](VarPtr c1, VarPtr c2) -> bool { + return c1->name.compare(c2->name) < 0; + }); + + for (VarPtr var : batch.constants) { + const TypeData *var_type = tinf::get_type(var); + mem.inc_count_by_type(var_type); // count stat to output it + } + + mem.total_count += batch.constants.size(); + } + + return mem; +} + +void GlobalsBatchedMem::inc_count_by_origin(VarPtr var) { + if (var->is_class_static_var()) { + count_of_static_fields++; + } else if (var->is_function_static_var()) { + count_of_function_statics++; + } else if (vk::string_view{var->name}.starts_with("d$")) { + count_of_nonconst_defines++; + } else if (vk::string_view{var->name}.ends_with("$called")) { + count_of_require_once++; + } else if (!var->is_builtin_runtime) { + count_of_php_global_scope++; + } +} + +int GlobalsBatchedMem::detect_globals_batch_count(int n_globals) { + if (n_globals > 10000) return 256; + if (n_globals > 5000) return 128; + if (n_globals > 1000) return 32; + if (n_globals > 500) return 16; + if (n_globals > 100) return 4; + if (n_globals > 50) return 2; + return 1; +} + +const GlobalsBatchedMem &GlobalsBatchedMem::prepare_mem_and_assign_offsets(const std::vector &all_globals) { + GlobalsBatchedMem &mem = globals_batched_mem; + + const int N_BATCHES = detect_globals_batch_count(all_globals.size()); + mem.batches.resize(N_BATCHES); + + for (VarPtr var : all_globals) { + int batch_idx = static_cast(vk::std_hash(var->name) % N_BATCHES); + var->batch_idx = batch_idx; + mem.batches[batch_idx].n_globals++; + } + + for (int batch_idx = 0; batch_idx < N_BATCHES; ++batch_idx) { + mem.batches[batch_idx].batch_idx = batch_idx; + mem.batches[batch_idx].globals.reserve(mem.batches[batch_idx].n_globals); + } + + for (VarPtr var : all_globals) { + mem.batches[var->batch_idx].globals.emplace_back(var); + } + + for (OneBatchInfo &batch : mem.batches) { + // sort variables by name to make codegen stable + // note, that all_globals contains also function static vars (explicitly added), + // and their names can duplicate or be equal to global vars; + // hence, also sort by holder_func (though global vars don't have holder_func, since there's no point of declaration) + std::sort(batch.globals.begin(), batch.globals.end(), [](VarPtr c1, VarPtr c2) -> bool { + int cmp_name = c1->name.compare(c2->name); + if (cmp_name < 0) { + return true; + } else if (cmp_name > 0) { + return false; + } else if (c1 == c2) { + return false; + } else { + if (!c1->holder_func) return true; + if (!c2->holder_func) return false; + return c1->holder_func->name.compare(c2->holder_func->name) < 0; + } + }); + + int offset = 0; + + for (VarPtr var : batch.globals) { + const TypeData *var_type = tinf::get_type(var); + int cur_sizeof = (calc_sizeof_in_bytes_runtime(var_type) + 7) & -8; // min 8 bytes per variable + + var->offset_in_linear_mem = mem.total_mem_size + offset; // it's continuous + offset += cur_sizeof; + mem.inc_count_by_origin(var); + } + + // leave "spaces" between batches for less incremental re-compilation: + // when PHP code changes (introducing a new global, for example), offsets will be shifted + // only inside one batch, but not throughout the whole project + // (with the exception, when a rounded batch size exceeds next 1KB) + // note, that we don't do this for constants: while globals memory is a single continuous piece, + // constants, on the contrary, are physically independent C++ variables + offset = (offset + 1023) & -1024; + + mem.total_mem_size += offset; + mem.total_count += batch.globals.size(); + } + + return mem; +} + +void ConstantsExternCollector::add_extern_from_var(VarPtr var) { + kphp_assert(var->is_constant()); + extern_constants.insert(var); +} + +void ConstantsExternCollector::add_extern_from_init_val(VertexPtr init_val) { + if (auto var = init_val.try_as()) { + add_extern_from_var(var->var_id); + } + for (VertexPtr child : *init_val) { + add_extern_from_init_val(child); + } +} + +void ConstantsExternCollector::compile(CodeGenerator &W) const { + for (VarPtr c : extern_constants) { + W << "extern " << type_out(tinf::get_type(c)) << " " << c->name << ";" << NL; + } +} + +void ConstantsMemAllocation::compile(CodeGenerator &W) const { + const ConstantsBatchedMem &mem = constants_batched_mem; + + W << "// total_count = " << mem.total_count << NL; + W << "// count(string) = " << mem.count_of_type_string << NL; + W << "// count(regexp) = " << mem.count_of_type_regexp << NL; + W << "// count(array) = " << mem.count_of_type_array << NL; + W << "// count(mixed) = " << mem.count_of_type_mixed << NL; + W << "// count(instance) = " << mem.count_of_type_instance << NL; + W << "// count(other) = " << mem.count_of_type_other << NL; + W << "// n_batches = " << mem.batches.size() << NL; +} + +void GlobalsMemAllocation::compile(CodeGenerator &W) const { + const GlobalsBatchedMem &mem = globals_batched_mem; + + W << "// total_mem_size = " << mem.total_mem_size << NL; + W << "// total_count = " << mem.total_count << NL; + W << "// count(static fields) = " << mem.count_of_static_fields << NL; + W << "// count(function statics) = " << mem.count_of_function_statics << NL; + W << "// count(nonconst defines) = " << mem.count_of_nonconst_defines << NL; + W << "// count(require_once) = " << mem.count_of_require_once << NL; + W << "// count(php global scope) = " << mem.count_of_php_global_scope << NL; + W << "// n_batches = " << mem.batches.size() << NL; + + if (!G->is_output_mode_lib()) { + W << "php_globals.once_alloc_linear_mem(" << mem.total_mem_size << ");" << NL; + } else { + W << "php_globals.once_alloc_linear_mem(\"" << G->settings().static_lib_name.get() << "\", " << mem.total_mem_size << ");" << NL; + } +} + +void PhpMutableGlobalsAssignCurrent::compile(CodeGenerator &W) const { + W << "PhpScriptMutableGlobals &php_globals = PhpScriptMutableGlobals::current();" << NL; +} + +void PhpMutableGlobalsDeclareInResumableClass::compile(CodeGenerator &W) const { + W << "PhpScriptMutableGlobals &php_globals;" << NL; +} + +void PhpMutableGlobalsAssignInResumableConstructor::compile(CodeGenerator &W) const { + W << "php_globals(PhpScriptMutableGlobals::current())"; +} + +void PhpMutableGlobalsRefArgument::compile(CodeGenerator &W) const { + W << "PhpScriptMutableGlobals &php_globals"; +} + +void PhpMutableGlobalsConstRefArgument::compile(CodeGenerator &W) const { + W << "const PhpScriptMutableGlobals &php_globals"; +} + +void GlobalVarInPhpGlobals::compile(CodeGenerator &W) const { + if (global_var->is_builtin_runtime) { + W << "php_globals.get_superglobals().v$" << global_var->name; + } else if (!G->is_output_mode_lib()) { + W << "(*reinterpret_cast<" << type_out(tinf::get_type(global_var)) << "*>(php_globals.mem()+" << global_var->offset_in_linear_mem << "))"; + } else { + W << "(*reinterpret_cast<" << type_out(tinf::get_type(global_var)) << "*>(php_globals.mem_for_lib(\"" << G->settings().static_lib_name.get() << "\")+" << global_var->offset_in_linear_mem << "))"; + } +} diff --git a/compiler/code-gen/const-globals-batched-mem.h b/compiler/code-gen/const-globals-batched-mem.h new file mode 100644 index 0000000000..67970e036e --- /dev/null +++ b/compiler/code-gen/const-globals-batched-mem.h @@ -0,0 +1,162 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "compiler/data/data_ptr.h" +#include "compiler/data/vertex-adaptor.h" + +class TypeData; +class CodeGenerator; + +// Here we put all auto-extracted constants (const strings, const arrays, etc.). +// They are initialized on master process start and therefore accessible from all workers for reading. +// Every const has a special refcount: if PHP code tries to mutate it, it'll be copied to script memory. +// Moreover, every constant has dependency_level (var-data.h). For instance, an array of strings +// should be initialized after all strings have been initialized. +// +// Since their count is huge, they are batched, and initialization is done (codegenerated) per-batch +// (more concrete, all level0 for every batch, then level1, etc.). +// +// When codegen starts, prepare_mem_and_assign_offsets() is called. +// It splits variables into batches and calculates necessary properties used in codegen later. +// Note, that finally each constant is represented as an independent C++ variable +// (whereas mutable globals, see below, are all placed in a single memory piece). +// We tested different ways to use the same approach for constants also. It works, but +// either leads to lots of incremental re-compilation or inconsistent compilation times in vkcom. +// +// See const-vars-init.cpp and collect-const-vars.cpp. +class ConstantsBatchedMem { +public: + struct OneBatchInfo { + int batch_idx; + int n_constants{0}; + std::vector constants; + int max_dep_level{0}; + }; + +private: + friend struct ConstantsMemAllocation; + + int count_of_type_string = 0; + int count_of_type_regexp = 0; + int count_of_type_array = 0; + int count_of_type_mixed = 0; + int count_of_type_instance = 0; + int count_of_type_other = 0; + + int total_count = 0; + + std::vector batches; + + void inc_count_by_type(const TypeData *type); + +public: + static int detect_constants_batch_count(int n_constants); + static const ConstantsBatchedMem &prepare_mem_and_assign_offsets(const std::vector &all_constants); + + const std::vector &get_batches() const { return batches; } + const OneBatchInfo &get_batch(uint64_t batch_hash) const { return batches.at(batch_hash); } +}; + +// While constants are initialized once in master process, mutable globals exists in each script +// (and are initialized on script start, placed in script memory). +// +// Opposed to constants, mutable globals are NOT C++ variables: instead, they all are placed +// in a single linear memory piece (char *), every var has offset (VarData::offset_in_linear_mem), +// and we use (&reinterpret_cast) to access a variable at offset (GlobalVarInPhpGlobals below). +// The purpose of this approach is to avoid mutable state at the moment of code generation, +// so that we could potentially compile a script into .so and load it multiple times. +// +// Note, that for correct offset calculation, the compiler must be aware of sizeof() of any possible type. +// If (earlier) a global inferred a type `std::tuple`, g++ determined its size. +// Now, we need to compute sizes and offsets at the moment of code generation, and to do it +// exactly the same as g++ would. See const-globals-batched-mem.cpp for implementation. +// +// Another thing to point is that we also split globals into batches, but leave "spaces" in linear memory: +// [batch1, ...(nothing, rounded up to 1KB), batch2, ...(nothing), ...] +// It's done to achieve less incremental re-compilation: when PHP code changes introducing a new global, +// offsets will be shifted only inside one batch, but not throughout the whole project. +// +// See globals-vars-reset.cpp and (runtime) php-script-globals.h. +class GlobalsBatchedMem { +public: + struct OneBatchInfo { + int batch_idx; + int n_globals{0}; + std::vector globals; + }; + +private: + friend struct GlobalsMemAllocation; + + int count_of_static_fields = 0; + int count_of_function_statics = 0; + int count_of_nonconst_defines = 0; + int count_of_require_once = 0; + int count_of_php_global_scope = 0; + + int total_count = 0; + int total_mem_size = 0; + + std::vector batches; + + void inc_count_by_origin(VarPtr var); + +public: + static int detect_globals_batch_count(int n_globals); + static const GlobalsBatchedMem &prepare_mem_and_assign_offsets(const std::vector &all_globals); + + const std::vector &get_batches() const { return batches; } + const OneBatchInfo &get_batch(int batch_idx) const { return batches.at(batch_idx); } +}; + +struct ConstantsExternCollector { + void add_extern_from_var(VarPtr var); + void add_extern_from_init_val(VertexPtr init_val); + + void compile(CodeGenerator &W) const; + +private: + std::set extern_constants; +}; + +struct ConstantsMemAllocation { + void compile(CodeGenerator &W) const; +}; + +struct GlobalsMemAllocation { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsAssignCurrent { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsDeclareInResumableClass { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsAssignInResumableConstructor { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsRefArgument { + void compile(CodeGenerator &W) const; +}; + +struct PhpMutableGlobalsConstRefArgument { + void compile(CodeGenerator &W) const; +}; + +struct GlobalVarInPhpGlobals { + VarPtr global_var; + + explicit GlobalVarInPhpGlobals(VarPtr global_var) + : global_var(global_var) {} + + void compile(CodeGenerator &W) const; +}; diff --git a/compiler/code-gen/declarations.cpp b/compiler/code-gen/declarations.cpp index b046e2f750..6692a35e23 100644 --- a/compiler/code-gen/declarations.cpp +++ b/compiler/code-gen/declarations.cpp @@ -7,6 +7,7 @@ #include "common/algorithms/compare.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/files/json-encoder-tags.h" #include "compiler/code-gen/files/tl2cpp/tl2cpp-utils.h" #include "compiler/code-gen/includes.h" @@ -25,10 +26,6 @@ #include "compiler/inferring/type-data.h" #include "compiler/tl-classes.h" -VarDeclaration VarExternDeclaration(VarPtr var) { - return {var, true, false}; -} - VarDeclaration VarPlainDeclaration(VarPtr var) { return {var, false, false}; } @@ -42,10 +39,6 @@ VarDeclaration::VarDeclaration(VarPtr var, bool extern_flag, bool defval_flag) : void VarDeclaration::compile(CodeGenerator &W) const { const TypeData *type = tinf::get_type(var); - if (var->is_builtin_global()) { - W << CloseNamespace(); - } - kphp_assert(type->ptype() != tp_void); W << (extern_flag ? "extern " : "") << TypeName(type) << " " << VarName(var); @@ -64,10 +57,6 @@ void VarDeclaration::compile(CodeGenerator &W) const { "decltype(const_begin(" << VarName(var) << "))" << " " << VarName(var) << name << ";" << NL; } } - - if (var->is_builtin_global()) { - W << OpenNamespace(); - } } FunctionDeclaration::FunctionDeclaration(FunctionPtr function, bool in_header, gen_out_style style) : @@ -441,18 +430,6 @@ ClassDeclaration::ClassDeclaration(ClassPtr klass) : klass(klass) { } -void ClassDeclaration::declare_all_variables(VertexPtr vertex, CodeGenerator &W) const { - if (!vertex) { - return; - } - for (auto child: *vertex) { - declare_all_variables(child, W); - } - if (auto var = vertex.try_as()) { - W << VarExternDeclaration(var->var_id); - } -} - std::unique_ptr ClassDeclaration::detect_if_needs_tl_usings() const { if (tl2cpp::is_php_class_a_tl_constructor(klass) && !tl2cpp::is_php_class_a_tl_array_item(klass)) { const auto &scheme = G->get_tl_classes().get_scheme(); @@ -480,11 +457,13 @@ void ClassDeclaration::compile(CodeGenerator &W) const { tl_dep_usings->compile_dependencies(W); } - klass->members.for_each([&](const ClassMemberInstanceField &f) { + ConstantsExternCollector c_mem_extern; + klass->members.for_each([&c_mem_extern](const ClassMemberInstanceField &f) { if (f.var->init_val) { - declare_all_variables(f.var->init_val, W); + c_mem_extern.add_extern_from_init_val(f.var->init_val); } }); + W << c_mem_extern << NL; auto get_all_interfaces = [klass = this->klass] { auto transform_to_src_name = [](CodeGenerator &W, InterfacePtr i) { W << i->src_name.c_str(); }; diff --git a/compiler/code-gen/declarations.h b/compiler/code-gen/declarations.h index 5a30147c7b..d46adc9cd6 100644 --- a/compiler/code-gen/declarations.h +++ b/compiler/code-gen/declarations.h @@ -26,7 +26,6 @@ struct VarDeclaration { void compile(CodeGenerator &W) const; }; -VarDeclaration VarExternDeclaration(VarPtr var); VarDeclaration VarPlainDeclaration(VarPtr var); struct FunctionDeclaration { @@ -140,7 +139,6 @@ struct ClassDeclaration : CodeGenRootCmd { IncludesCollector compile_front_includes(CodeGenerator &W) const; void compile_back_includes(CodeGenerator &W, IncludesCollector &&front_includes) const; void compile_job_worker_shared_memory_piece_methods(CodeGenerator &W, bool compile_declaration_only = false) const; - void declare_all_variables(VertexPtr v, CodeGenerator &W) const; std::unique_ptr detect_if_needs_tl_usings() const; }; diff --git a/compiler/code-gen/files/const-vars-init.cpp b/compiler/code-gen/files/const-vars-init.cpp new file mode 100644 index 0000000000..51c4f2de0b --- /dev/null +++ b/compiler/code-gen/files/const-vars-init.cpp @@ -0,0 +1,171 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/files/const-vars-init.h" + +#include "common/algorithms/hashes.h" + +#include "compiler/code-gen/const-globals-batched-mem.h" +#include "compiler/code-gen/declarations.h" +#include "compiler/code-gen/namespace.h" +#include "compiler/code-gen/raw-data.h" +#include "compiler/code-gen/vertex-compiler.h" +#include "compiler/data/function-data.h" +#include "compiler/data/src-file.h" +#include "compiler/inferring/public.h" + +struct InitConstVar { + VarPtr var; + explicit InitConstVar(VarPtr var) : var(var) {} + + void compile(CodeGenerator &W) const { + Location save_location = stage::get_location(); + + VertexPtr init_val = var->init_val; + if (init_val->type() == op_conv_regexp) { + const auto &location = init_val->get_location(); + kphp_assert(location.function && location.file); + W << var->name << ".init (" << var->init_val << ", " << RawString(location.function->name) << ", " + << RawString(location.file->relative_file_name + ':' + std::to_string(location.line)) + << ");" << NL; + } else { + W << var->name << " = " << var->init_val << ";" << NL; + } + + stage::set_location(save_location); + } +}; + + +static void compile_raw_array(CodeGenerator &W, const VarPtr &var, int shift) { + if (shift == -1) { + W << InitConstVar(var); + W << var->name << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL << NL; + return; + } + + W << var->name << ".assign_raw((char *) &raw_arrays[" << shift << "]);" << NL << NL; +} + +ConstVarsInit::ConstVarsInit(const ConstantsBatchedMem &all_constants_in_mem) + : all_constants_in_mem(all_constants_in_mem) {} + +void ConstVarsInit::compile_const_init_part(CodeGenerator &W, const ConstantsBatchedMem::OneBatchInfo &batch) { + DepLevelContainer const_raw_array_vars; + DepLevelContainer other_const_vars; + DepLevelContainer const_raw_string_vars; + + IncludesCollector includes; + ConstantsExternCollector c_mem_extern; + for (VarPtr var : batch.constants) { + if (!G->is_output_mode_lib()) { + includes.add_var_signature_depends(var); + includes.add_vertex_depends(var->init_val); + } + c_mem_extern.add_extern_from_init_val(var->init_val); + } + W << includes; + + W << OpenNamespace(); + for (VarPtr var : batch.constants) { + W << type_out(tinf::get_type(var)) << " " << var->name << ";" << NL; + } + W << c_mem_extern; + + for (VarPtr var : batch.constants) { + switch (var->init_val->type()) { + case op_string: + const_raw_string_vars.add(var); + break; + case op_array: + const_raw_array_vars.add(var); + break; + default: + other_const_vars.add(var); + break; + } + } + + std::vector str_values(const_raw_string_vars.size()); + std::transform(const_raw_string_vars.begin(), const_raw_string_vars.end(), + str_values.begin(), + [](VarPtr var) { return var->init_val.as()->str_val; }); + + const std::vector const_string_shifts = compile_raw_data(W, str_values); + const std::vector const_array_shifts = compile_arrays_raw_representation(const_raw_array_vars, W); + const size_t max_dep_level = std::max({const_raw_string_vars.max_dep_level(), const_raw_array_vars.max_dep_level(), other_const_vars.max_dep_level(), 1ul}); + + size_t str_idx = 0; + size_t arr_idx = 0; + for (size_t dep_level = 0; dep_level < max_dep_level; ++dep_level) { + const std::string func_name_i = fmt_format("const_init_level{}_file{}", dep_level, batch.batch_idx); + FunctionSignatureGenerator(W) << NL << "void " << func_name_i << "()" << BEGIN; + + for (VarPtr var : const_raw_string_vars.vars_by_dep_level(dep_level)) { + W << var->name << ".assign_raw (&raw[" << const_string_shifts[str_idx++] << "]);" << NL; + } + + for (VarPtr var : const_raw_array_vars.vars_by_dep_level(dep_level)) { + compile_raw_array(W, var, const_array_shifts[arr_idx++]); + } + + for (VarPtr var: other_const_vars.vars_by_dep_level(dep_level)) { + W << InitConstVar(var); + const TypeData *type_data = var->tinf_node.get_type(); + if (vk::any_of_equal(type_data->ptype(), tp_array, tp_mixed, tp_string)) { + W << var->name; + if (type_data->use_optional()) { + W << ".val()"; + } + W << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL; + } + } + + W << END << NL; + } + + W << CloseNamespace(); +} + +void ConstVarsInit::compile_const_init(CodeGenerator &W, const ConstantsBatchedMem &all_constants_in_mem) { + W << OpenNamespace(); + + W << NL; + + FunctionSignatureGenerator(W) << "void const_vars_init() " << BEGIN; + W << ConstantsMemAllocation() << NL; + + int very_max_dep_level = 0; + for (const auto &batch : all_constants_in_mem.get_batches()) { + very_max_dep_level = std::max(very_max_dep_level, batch.max_dep_level); + } + + for (int dep_level = 0; dep_level <= very_max_dep_level; ++dep_level) { + for (const auto &batch : all_constants_in_mem.get_batches()) { + if (dep_level <= batch.max_dep_level) { + const std::string func_name_i = fmt_format("const_init_level{}_file{}", dep_level, batch.batch_idx); + // function declaration + W << "void " << func_name_i << "();" << NL; + // function call + W << func_name_i << "();" << NL; + } + } + } + W << END; + W << CloseNamespace(); +} + +void ConstVarsInit::compile(CodeGenerator &W) const { + for (const auto &batch : all_constants_in_mem.get_batches()) { + W << OpenFile("c." + std::to_string(batch.batch_idx) + ".cpp", "o_const_init", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_const_init_part(W, batch); + W << CloseFile(); + } + + W << OpenFile("const_vars_init.cpp", "", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_const_init(W, all_constants_in_mem); + W << CloseFile(); +} diff --git a/compiler/code-gen/files/const-vars-init.h b/compiler/code-gen/files/const-vars-init.h new file mode 100644 index 0000000000..585e0a0916 --- /dev/null +++ b/compiler/code-gen/files/const-vars-init.h @@ -0,0 +1,21 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "compiler/code-gen/code-gen-root-cmd.h" +#include "compiler/code-gen/code-generator.h" +#include "compiler/code-gen/const-globals-batched-mem.h" + +struct ConstVarsInit : CodeGenRootCmd { + explicit ConstVarsInit(const ConstantsBatchedMem &all_constants_in_mem); + + void compile(CodeGenerator &W) const final; + + static void compile_const_init_part(CodeGenerator& W, const ConstantsBatchedMem::OneBatchInfo& batch); + static void compile_const_init(CodeGenerator &W, const ConstantsBatchedMem &all_constants_in_mem); + +private: + const ConstantsBatchedMem &all_constants_in_mem; +}; diff --git a/compiler/code-gen/files/function-header.cpp b/compiler/code-gen/files/function-header.cpp index edabe8cde5..8a17edc921 100644 --- a/compiler/code-gen/files/function-header.cpp +++ b/compiler/code-gen/files/function-header.cpp @@ -5,6 +5,7 @@ #include "compiler/code-gen/files/function-header.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/function-source.h" #include "compiler/code-gen/includes.h" @@ -25,9 +26,12 @@ void FunctionH::compile(CodeGenerator &W) const { W << includes; W << OpenNamespace(); - for (auto const_var : function->explicit_header_const_var_ids) { - W << VarExternDeclaration(const_var) << NL; + + ConstantsExternCollector c_mem_extern; + for (VarPtr var : function->explicit_header_const_var_ids) { + c_mem_extern.add_extern_from_var(var); } + W << c_mem_extern << NL; if (function->is_inline) { W << "inline "; @@ -53,9 +57,7 @@ void FunctionH::compile(CodeGenerator &W) const { W << includes; W << OpenNamespace(); - declare_global_vars(function, W); declare_const_vars(function, W); - declare_static_vars(function, W); W << UnlockComments(); W << function->root << NL; W << LockComments(); diff --git a/compiler/code-gen/files/function-source.cpp b/compiler/code-gen/files/function-source.cpp index d605f28c2e..a5afbcd157 100644 --- a/compiler/code-gen/files/function-source.cpp +++ b/compiler/code-gen/files/function-source.cpp @@ -5,6 +5,7 @@ #include "compiler/code-gen/files/function-source.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/includes.h" #include "compiler/code-gen/namespace.h" @@ -16,22 +17,12 @@ FunctionCpp::FunctionCpp(FunctionPtr function) : function(function) { } -void declare_global_vars(FunctionPtr function, CodeGenerator &W) { - for (auto global_var : function->global_var_ids) { - W << VarExternDeclaration(global_var) << NL; - } -} - void declare_const_vars(FunctionPtr function, CodeGenerator &W) { - for (auto const_var : function->explicit_const_var_ids) { - W << VarExternDeclaration(const_var) << NL; - } -} - -void declare_static_vars(FunctionPtr function, CodeGenerator &W) { - for (auto static_var : function->static_var_ids) { - W << VarExternDeclaration(static_var) << NL; + ConstantsExternCollector c_mem_extern; + for (VarPtr var : function->explicit_const_var_ids) { + c_mem_extern.add_extern_from_var(var); } + W << c_mem_extern << NL; } void FunctionCpp::compile(CodeGenerator &W) const { @@ -50,9 +41,7 @@ void FunctionCpp::compile(CodeGenerator &W) const { W << includes; W << OpenNamespace(); - declare_global_vars(function, W); declare_const_vars(function, W); - declare_static_vars(function, W); W << UnlockComments(); W << function->root << NL; diff --git a/compiler/code-gen/files/function-source.h b/compiler/code-gen/files/function-source.h index 72332f56cf..7e7062c9c7 100644 --- a/compiler/code-gen/files/function-source.h +++ b/compiler/code-gen/files/function-source.h @@ -16,4 +16,3 @@ struct FunctionCpp : CodeGenRootCmd { void declare_global_vars(FunctionPtr function, CodeGenerator &W); void declare_const_vars(FunctionPtr function, CodeGenerator &W); -void declare_static_vars(FunctionPtr function, CodeGenerator &W); diff --git a/compiler/code-gen/files/global-vars-memory-stats.cpp b/compiler/code-gen/files/global-vars-memory-stats.cpp new file mode 100644 index 0000000000..6e5b7a62c6 --- /dev/null +++ b/compiler/code-gen/files/global-vars-memory-stats.cpp @@ -0,0 +1,111 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/files/global-vars-memory-stats.h" + +#include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" +#include "compiler/code-gen/declarations.h" +#include "compiler/code-gen/includes.h" +#include "compiler/code-gen/namespace.h" +#include "compiler/code-gen/raw-data.h" +#include "compiler/data/src-file.h" +#include "compiler/inferring/public.h" + +GlobalVarsMemoryStats::GlobalVarsMemoryStats(const std::vector &all_globals) { + for (VarPtr global_var : all_globals) { + bool is_primitive = vk::any_of_equal(tinf::get_type(global_var)->get_real_ptype(), tp_bool, tp_int, tp_float, tp_regexp, tp_any); + if (!is_primitive && !global_var->is_builtin_runtime) { + all_nonprimitive_globals.push_back(global_var); + } + } + // to make codegen stable (here we use operator < of VarPtr, see var-data.cpp) + std::sort(all_nonprimitive_globals.begin(), all_nonprimitive_globals.end()); +} + +void GlobalVarsMemoryStats::compile(CodeGenerator &W) const { + int total_count = static_cast(all_nonprimitive_globals.size()); + int parts_cnt = static_cast(std::ceil(static_cast(total_count) / N_GLOBALS_PER_FILE)); + + W << OpenFile("globals_memory_stats.cpp", "", false) + << ExternInclude(G->settings().runtime_headers.get()) + << OpenNamespace(); + + // we don't take libs into account here (don't call "global memory stats" for every lib), + // since we have to guarantee that libs were compiled with a necessary flag also + // (most likely, not, then C++ compilation will fail) + + FunctionSignatureGenerator(W) << "array " << getter_name_ << "(int64_t lower_bound, " + << PhpMutableGlobalsConstRefArgument() << ")" << BEGIN + << "array result;" << NL + << "result.reserve(" << total_count << ", false);" << NL << NL; + + for (int part_id = 0; part_id < parts_cnt; ++part_id) { + const std::string func_name_i = getter_name_ + std::to_string(part_id); + // function declaration + FunctionSignatureGenerator(W) << "void " << func_name_i << "(int64_t lower_bound, array &result, " << PhpMutableGlobalsConstRefArgument() << ")" << SemicolonAndNL(); + // function call + W << func_name_i << "(lower_bound, result, php_globals);" << NL << NL; + } + + W << "return result;" << NL << END + << CloseNamespace() + << CloseFile(); + + for (int part_id = 0; part_id < parts_cnt; ++part_id) { + int offset = part_id * N_GLOBALS_PER_FILE; + int count = std::min(static_cast(all_nonprimitive_globals.size()) - offset, N_GLOBALS_PER_FILE); + + W << OpenFile("globals_memory_stats." + std::to_string(part_id) + ".cpp", "o_globals_memory_stats", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_getter_part(W, part_id, all_nonprimitive_globals, offset, count); + W << CloseFile(); + } +} + +void GlobalVarsMemoryStats::compile_getter_part(CodeGenerator &W, int part_id, const std::vector &global_vars, int offset, int count) { + IncludesCollector includes; + std::vector var_names; + var_names.reserve(count); + for (int i = 0; i < count; ++i) { + VarPtr global_var = global_vars[offset + i]; + includes.add_var_signature_depends(global_var); + std::string var_name; + if (global_var->is_function_static_var()) { + var_name = global_var->holder_func->name + "::"; + } + var_name.append(global_var->as_human_readable()); + var_names.emplace_back(std::move(var_name)); + } + + W << includes << NL + << OpenNamespace(); + + const auto var_name_shifts = compile_raw_data(W, var_names); + W << NL; + + FunctionSignatureGenerator(W) << "static string get_raw_string(int raw_offset) " << BEGIN; + W << "string str;" << NL + << "str.assign_raw(&raw[raw_offset]);" << NL + << "return str;" << NL + << END << NL << NL; + + FunctionSignatureGenerator(W) << "void " << getter_name_ << part_id << "(int64_t lower_bound, array &result, " + << PhpMutableGlobalsConstRefArgument() << ")" << BEGIN; + + if (count) { + W << "int64_t estimation = 0;" << NL; + } + for (int i = 0; i < count; ++i) { + VarPtr global_var = global_vars[offset + i]; + W << "// " << var_names[i] << NL + << "estimation = f$estimate_memory_usage(" << GlobalVarInPhpGlobals(global_var) << ");" << NL + << "if (estimation > lower_bound) " << BEGIN + << "result.set_value(get_raw_string(" << var_name_shifts[i] << "), estimation);" << NL + << END << NL; + } + + W << END; + W << CloseNamespace(); +} diff --git a/compiler/code-gen/files/global-vars-memory-stats.h b/compiler/code-gen/files/global-vars-memory-stats.h new file mode 100644 index 0000000000..8c324b3fed --- /dev/null +++ b/compiler/code-gen/files/global-vars-memory-stats.h @@ -0,0 +1,23 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "compiler/code-gen/code-gen-root-cmd.h" +#include "compiler/code-gen/code-generator.h" +#include "compiler/data/data_ptr.h" + +struct GlobalVarsMemoryStats : CodeGenRootCmd { + explicit GlobalVarsMemoryStats(const std::vector &all_globals); + + void compile(CodeGenerator &W) const final; + +private: + static void compile_getter_part(CodeGenerator &W, int part_id, const std::vector &global_vars, int offset, int count); + + std::vector all_nonprimitive_globals; + + static constexpr const char *getter_name_ = "globals_memory_stats_impl"; // hardcoded in runtime, see f$get_global_vars_memory_stats() + static constexpr int N_GLOBALS_PER_FILE = 512; +}; diff --git a/compiler/code-gen/files/global-vars-reset.cpp b/compiler/code-gen/files/global-vars-reset.cpp new file mode 100644 index 0000000000..1f1fdf4092 --- /dev/null +++ b/compiler/code-gen/files/global-vars-reset.cpp @@ -0,0 +1,98 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "compiler/code-gen/files/global-vars-reset.h" + +#include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" +#include "compiler/code-gen/declarations.h" +#include "compiler/code-gen/namespace.h" +#include "compiler/code-gen/vertex-compiler.h" +#include "compiler/data/src-file.h" + +GlobalVarsReset::GlobalVarsReset(const GlobalsBatchedMem &all_globals_in_mem) + : all_globals_in_mem(all_globals_in_mem) {} + +void GlobalVarsReset::compile_globals_reset_part(CodeGenerator &W, const GlobalsBatchedMem::OneBatchInfo &batch) { + IncludesCollector includes; + for (VarPtr var : batch.globals) { + includes.add_var_signature_depends(var); + } + W << includes; + W << OpenNamespace(); + + W << NL; + ConstantsExternCollector c_mem_extern; + for (VarPtr var : batch.globals) { + if (var->init_val) { + c_mem_extern.add_extern_from_init_val(var->init_val); + } + } + W << c_mem_extern << NL; + + FunctionSignatureGenerator(W) << "void global_vars_reset_file" << batch.batch_idx << "(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + for (VarPtr var : batch.globals) { + if (var->is_builtin_runtime) { // they are manually reset in runtime sources + continue; + } + + // todo probably, inline hard_reset_var() body, since it uses new(&)? + W << "// " << var->as_human_readable() << NL; + W << "hard_reset_var(" << GlobalVarInPhpGlobals(var); + if (var->init_val) { + W << ", " << var->init_val; + } + W << ");" << NL; + } + + W << END; + W << NL; + W << CloseNamespace(); +} + +void GlobalVarsReset::compile_globals_reset(CodeGenerator &W, const GlobalsBatchedMem &all_globals_in_mem) { + W << OpenNamespace(); + FunctionSignatureGenerator(W) << "void global_vars_reset(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + + for (const auto &batch : all_globals_in_mem.get_batches()) { + const std::string func_name_i = "global_vars_reset_file" + std::to_string(batch.batch_idx); + // function declaration + W << "void " << func_name_i << "(" << PhpMutableGlobalsRefArgument() << ");" << NL; + // function call + W << func_name_i << "(php_globals);" << NL; + } + + W << END; + W << NL; + W << CloseNamespace(); +} + +void GlobalVarsReset::compile_globals_allocate(CodeGenerator &W) { + W << OpenNamespace(); + FunctionSignatureGenerator(W) << "void global_vars_allocate(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + + W << GlobalsMemAllocation(); + + W << END << NL; + W << CloseNamespace(); +} + +void GlobalVarsReset::compile(CodeGenerator &W) const { + for (const auto &batch : all_globals_in_mem.get_batches()) { + W << OpenFile("globals_reset." + std::to_string(batch.batch_idx) + ".cpp", "o_globals_reset", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_globals_reset_part(W, batch); + W << CloseFile(); + } + + W << OpenFile("globals_reset.cpp", "", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_globals_reset(W, all_globals_in_mem); + W << CloseFile(); + + W << OpenFile("globals_allocate.cpp", "", false); + W << ExternInclude(G->settings().runtime_headers.get()); + compile_globals_allocate(W); + W << CloseFile(); +} diff --git a/compiler/code-gen/files/global-vars-reset.h b/compiler/code-gen/files/global-vars-reset.h new file mode 100644 index 0000000000..0d519d8e35 --- /dev/null +++ b/compiler/code-gen/files/global-vars-reset.h @@ -0,0 +1,22 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2020 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "compiler/code-gen/code-gen-root-cmd.h" +#include "compiler/code-gen/code-generator.h" +#include "compiler/code-gen/const-globals-batched-mem.h" + +struct GlobalVarsReset : CodeGenRootCmd { + explicit GlobalVarsReset(const GlobalsBatchedMem &all_globals_in_mem); + + void compile(CodeGenerator &W) const final; + + static void compile_globals_reset_part(CodeGenerator &W, const GlobalsBatchedMem::OneBatchInfo &batch); + static void compile_globals_reset(CodeGenerator &W, const GlobalsBatchedMem &all_globals_in_mem); + static void compile_globals_allocate(CodeGenerator &W); + +private: + const GlobalsBatchedMem &all_globals_in_mem; +}; diff --git a/compiler/code-gen/files/global_vars_memory_stats.cpp b/compiler/code-gen/files/global_vars_memory_stats.cpp deleted file mode 100644 index d49a4e008e..0000000000 --- a/compiler/code-gen/files/global_vars_memory_stats.cpp +++ /dev/null @@ -1,97 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/code-gen/files/global_vars_memory_stats.h" - -#include "compiler/code-gen/common.h" -#include "compiler/code-gen/declarations.h" -#include "compiler/code-gen/includes.h" -#include "compiler/code-gen/namespace.h" -#include "compiler/code-gen/raw-data.h" -#include "compiler/data/src-file.h" -#include "compiler/data/vars-collector.h" -#include "compiler/inferring/public.h" - -GlobalVarsMemoryStats::GlobalVarsMemoryStats(SrcFilePtr main_file) : - main_file_{main_file} { -} - -void GlobalVarsMemoryStats::compile(CodeGenerator &W) const { - VarsCollector vars_collector{32, [](VarPtr global_var) { - return vk::none_of_equal(tinf::get_type(global_var)->get_real_ptype(), tp_bool, tp_int, tp_float, tp_any); - }}; - - vars_collector.collect_global_and_static_vars_from(main_file_->main_function); - - auto global_var_parts = vars_collector.flush(); - size_t global_vars_count = 0; - for (const auto &global_vars : global_var_parts) { - global_vars_count += global_vars.size(); - } - - W << OpenFile(getter_name_ + ".cpp", "", false) - << ExternInclude(G->settings().runtime_headers.get()) - << OpenNamespace(); - - FunctionSignatureGenerator(W) << "array " << getter_name_ << "(int64_t lower_bound) " << BEGIN - << "array result;" << NL - << "result.reserve(" << global_vars_count << ", false);" << NL << NL; - - for (size_t part_id = 0; part_id < global_var_parts.size(); ++part_id) { - W << "void " << getter_name_ << "_" << part_id << "(int64_t lower_bound, array &result);" << NL - << getter_name_ << "_" << part_id << "(lower_bound, result);" << NL << NL; - } - - W << "return result;" << NL << END - << CloseNamespace() - << CloseFile(); - - for (size_t part_id = 0; part_id < global_var_parts.size(); ++part_id) { - compile_getter_part(W, global_var_parts[part_id], part_id); - } -} - -void GlobalVarsMemoryStats::compile_getter_part(CodeGenerator &W, const std::set &global_vars, size_t part_id) const { - W << OpenFile(getter_name_ + "_" + std::to_string(part_id) + ".cpp", "o_" + getter_name_, false) - << ExternInclude(G->settings().runtime_headers.get()); - - IncludesCollector includes; - std::vector var_names; - var_names.reserve(global_vars.size()); - for (const auto &global_var : global_vars) { - includes.add_var_signature_depends(global_var); - std::string var_name; - if (global_var->is_function_static_var()) { - var_name = global_var->holder_func->name + "::"; - } - var_name.append(global_var->as_human_readable()); - var_names.emplace_back(std::move(var_name)); - } - - W << includes << NL - << OpenNamespace(); - - FunctionSignatureGenerator(W) << "static string get_raw_string(int raw_offset) " << BEGIN; - const auto var_name_shifts = compile_raw_data(W, var_names); - W << "string str;" << NL - << "str.assign_raw(&raw[raw_offset]);" << NL - << "return str;" << NL - << END << NL << NL; - - FunctionSignatureGenerator(W) << "void " << getter_name_ << "_" << part_id << "(int64_t lower_bound, array &result) " << BEGIN - << "int64_t estimation = 0;" << NL; - size_t var_num = 0; - for (auto global_var : global_vars) { - W << VarDeclaration(global_var, true, false) - << "estimation = f$estimate_memory_usage(" << VarName(global_var) << ");" << NL - << "if (estimation > lower_bound) " << BEGIN - << "result.set_value(get_raw_string(" << var_name_shifts[var_num++] << "), estimation);" << NL - << END << NL; - } - - W << END; - - W << CloseNamespace() - << CloseFile(); -} diff --git a/compiler/code-gen/files/global_vars_memory_stats.h b/compiler/code-gen/files/global_vars_memory_stats.h deleted file mode 100644 index a6f48d4da3..0000000000 --- a/compiler/code-gen/files/global_vars_memory_stats.h +++ /dev/null @@ -1,21 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include "compiler/code-gen/code-gen-root-cmd.h" -#include "compiler/code-gen/code-generator.h" -#include "compiler/data/data_ptr.h" - -struct GlobalVarsMemoryStats : CodeGenRootCmd { - explicit GlobalVarsMemoryStats(SrcFilePtr main_file); - - void compile(CodeGenerator &W) const final; - -private: - void compile_getter_part(CodeGenerator &W, const std::set &global_vars, size_t part_id) const; - - const std::string getter_name_{"get_global_vars_memory_stats_impl"}; - SrcFilePtr main_file_; -}; diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 2fa40b65d5..a8d2d4fedb 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -5,6 +5,7 @@ #include "compiler/code-gen/files/init-scripts.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/shape-keys.h" #include "compiler/code-gen/includes.h" @@ -18,57 +19,49 @@ struct StaticInit { void compile(CodeGenerator &W) const; }; - void StaticInit::compile(CodeGenerator &W) const { - for (LibPtr lib: G->get_libs()) { + if (G->is_output_mode_lib()) { + return; + } + + // "const vars init" declarations + FunctionSignatureGenerator(W) << "void const_vars_init()" << SemicolonAndNL() << NL; + for (LibPtr lib : G->get_libs()) { if (lib && !lib->is_raw_php()) { W << OpenNamespace(lib->lib_namespace()); - FunctionSignatureGenerator(W) << "void global_init_lib_scripts()" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void const_vars_init()" << SemicolonAndNL(); W << CloseNamespace(); } } - W << OpenNamespace(); - FunctionSignatureGenerator(W) << "void const_vars_init()" << SemicolonAndNL() << NL; - FunctionSignatureGenerator(W) << "void tl_str_const_init()" << SemicolonAndNL(); if (G->get_untyped_rpc_tl_used()) { FunctionSignatureGenerator(W) << "array gen$tl_fetch_wrapper(std::unique_ptr)" << SemicolonAndNL(); W << "extern array gen$tl_storers_ht;" << NL; FunctionSignatureGenerator(W) << "void fill_tl_storers_ht()" << SemicolonAndNL() << NL; } - if (G->settings().is_static_lib_mode()) { - FunctionSignatureGenerator(W) << "void global_init_lib_scripts() " << BEGIN; + FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN << "return " << RawString(G->settings().php_code_version.get()) << ";" + << NL << END << NL << NL; + + FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; + const auto &runtime_opts = G->get_kphp_runtime_opts(); + if (runtime_opts.empty()) { + W << "return nullptr;" << NL; } else { - FunctionSignatureGenerator(W) << ("const char *get_php_scripts_version()") << BEGIN - << "return " << RawString(G->settings().php_code_version.get()) << ";" << NL - << END << NL << NL; - - FunctionSignatureGenerator(W) << ("char **get_runtime_options([[maybe_unused]] int *count)") << BEGIN; - const auto &runtime_opts = G->get_kphp_runtime_opts(); - if (runtime_opts.empty()) { - W << "return nullptr;" << NL; - } else { - W << "*count = " << runtime_opts.size() << ";" << NL; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; - } - W << "static char *argv[] = " << BEGIN; - for (size_t i = 0; i != runtime_opts.size(); ++i) { - W << "arg" << i << "," << NL; - } - W << END << ";" << NL - << "return argv;" << NL; + W << "*count = " << runtime_opts.size() << ";" << NL; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "static char arg" << i << "[] = " << RawString{runtime_opts[i]} << ";" << NL; } - W << END << NL << NL; - - FunctionSignatureGenerator(W) << ("void global_init_php_scripts() ") << BEGIN; - for (LibPtr lib: G->get_libs()) { - if (lib && !lib->is_raw_php()) { - W << lib->lib_namespace() << "::global_init_lib_scripts();" << NL; - } + W << "static char *argv[] = " << BEGIN; + for (size_t i = 0; i != runtime_opts.size(); ++i) { + W << "arg" << i << "," << NL; } + W << END << ";" << NL << "return argv;" << NL; } + W << END << NL << NL; + + FunctionSignatureGenerator(W) << ("void init_php_scripts_once_in_master() ") << BEGIN; + if (!G->settings().tl_schema_file.get().empty()) { W << "tl_str_const_init();" << NL; if (G->get_untyped_rpc_tl_used()) { @@ -77,11 +70,19 @@ void StaticInit::compile(CodeGenerator &W) const { } } W << "const_vars_init();" << NL; + for (LibPtr lib : G->get_libs()) { + if (lib && !lib->is_raw_php()) { + W << lib->lib_namespace() << "::const_vars_init();" << NL; + } + } + W << NL; + FunctionSignatureGenerator(W) << "void " << ShapeKeys::get_function_name() << "()" << SemicolonAndNL(); + W << ShapeKeys::get_function_name() << "();" << NL; const auto &ffi = G->get_ffi_root(); const auto &ffi_shared_libs = ffi.get_shared_libs(); if (!ffi_shared_libs.empty()) { - W << "ffi_env_instance = FFIEnv{" << ffi_shared_libs.size() << ", " << ffi.get_dynamic_symbols_num() << "};" << NL; + W << "ffi_env_instance = FFIEnv{" << ffi_shared_libs.size() << ", " << ffi.get_dynamic_symbols_num() << "};" << NL; W << "ffi_env_instance.funcs.dlopen = dlopen;" << NL; W << "ffi_env_instance.funcs.dlsym = dlsym;" << NL; for (const auto &lib : ffi_shared_libs) { @@ -99,7 +100,6 @@ void StaticInit::compile(CodeGenerator &W) const { } W << END << NL; - W << CloseNamespace(); } struct RunFunction { @@ -116,61 +116,58 @@ struct RunFunction { }; -struct GlobalResetFunction { - FunctionPtr function; - GlobalResetFunction(FunctionPtr function); +struct GlobalsResetFunction { + FunctionPtr main_function; + explicit GlobalsResetFunction(FunctionPtr main_function); void compile(CodeGenerator &W) const; }; -GlobalResetFunction::GlobalResetFunction(FunctionPtr function) : - function(function) { -} +GlobalsResetFunction::GlobalsResetFunction(FunctionPtr main_function) + : main_function(main_function) {} -void GlobalResetFunction::compile(CodeGenerator &W) const { +void GlobalsResetFunction::compile(CodeGenerator &W) const { + // "global vars reset" declarations + FunctionSignatureGenerator(W) << "void global_vars_allocate(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void global_vars_reset(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); + W << NL; for (LibPtr lib: G->get_libs()) { if (lib && !lib->is_raw_php()) { W << OpenNamespace(lib->lib_namespace()); - FunctionSignatureGenerator(W) << "void lib_global_vars_reset()" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void global_vars_allocate(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); + FunctionSignatureGenerator(W) << "void global_vars_reset(" << PhpMutableGlobalsRefArgument() << ")" << SemicolonAndNL(); W << CloseNamespace(); } } - FunctionSignatureGenerator(W) << "void " << FunctionName(function) << "$global_reset() " << BEGIN; - W << "void " << GlobalVarsResetFuncName(function) << ";" << NL; - W << GlobalVarsResetFuncName(function) << ";" << NL; + // "global vars reset" calls + FunctionSignatureGenerator(W) << "void " << FunctionName(main_function) << "$globals_reset(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; + W << "global_vars_reset(php_globals);" << NL; for (LibPtr lib: G->get_libs()) { if (lib && !lib->is_raw_php()) { - W << lib->lib_namespace() << "::lib_global_vars_reset();" << NL; + W << lib->lib_namespace() << "::global_vars_reset(php_globals);" << NL; } } W << END << NL; } -struct LibGlobalVarsReset { - const FunctionPtr &main_function; - LibGlobalVarsReset(const FunctionPtr &main_function); +struct LibRunFunction { + FunctionPtr main_function; + LibRunFunction(FunctionPtr main_function); void compile(CodeGenerator &W) const; }; -LibGlobalVarsReset::LibGlobalVarsReset(const FunctionPtr &main_function) : +LibRunFunction::LibRunFunction(FunctionPtr main_function) : main_function(main_function) { } -void LibGlobalVarsReset::compile(CodeGenerator &W) const { - W << OpenNamespace(); - FunctionSignatureGenerator(W) << "void lib_global_vars_reset() " << BEGIN - << "void " << GlobalVarsResetFuncName(main_function) << ";" << NL - << GlobalVarsResetFuncName(main_function) << ";" << NL - << END << NL << NL; - W << "extern bool v$" << main_function->file_id->get_main_func_run_var_name() << ";" << NL; - W << CloseNamespace(); - +void LibRunFunction::compile(CodeGenerator &W) const { + // "run" functions just calls the main file of a lib + // it's guaranteed that it doesn't contain code except declarations (body_value is empty), + // that's why we shouldn't deal with `if (!called)` global W << StaticLibraryRunGlobal(gen_out_style::cpp) << BEGIN << "using namespace " << G->get_global_namespace() << ";" << NL - << "if (!v$" << main_function->file_id->get_main_func_run_var_name() << ")" << BEGIN << FunctionName(main_function) << "();" << NL - << END << NL << END << NL << NL; } @@ -191,33 +188,32 @@ void InitScriptsCpp::compile(CodeGenerator &W) const { W << ExternInclude("dlfcn.h"); // dlopen, dlsym } - if (!G->settings().is_static_lib_mode()) { - W << NL; - FunctionSignatureGenerator(W) << "void global_init_php_scripts()" << SemicolonAndNL(); - FunctionSignatureGenerator(W) << "void init_php_scripts()" << SemicolonAndNL(); - } - W << NL << StaticInit() << NL; - if (G->settings().is_static_lib_mode()) { - W << LibGlobalVarsReset(main_file_id->main_function); + if (G->is_output_mode_lib()) { + W << LibRunFunction(main_file_id->main_function); W << CloseFile(); return; } W << RunFunction(main_file_id->main_function) << NL; - W << GlobalResetFunction(main_file_id->main_function) << NL; + W << GlobalsResetFunction(main_file_id->main_function) << NL; + + FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() << ")" << BEGIN; - FunctionSignatureGenerator(W) << "void init_php_scripts() " << BEGIN; + W << "global_vars_allocate(php_globals);" << NL; + for (LibPtr lib: G->get_libs()) { + if (lib && !lib->is_raw_php()) { + W << lib->lib_namespace() << "::global_vars_allocate(php_globals);" << NL; + } + } - W << ShapeKeys::get_function_declaration() << ";" << NL; - W << ShapeKeys::get_function_name() << "();" << NL << NL; - W << FunctionName(main_file_id->main_function) << "$global_reset();" << NL; + W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; W << "set_script (" << FunctionName(main_file_id->main_function) << "$run, " - << FunctionName(main_file_id->main_function) << "$global_reset);" << NL; + << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; W << END; @@ -234,7 +230,7 @@ void LibVersionHFile::compile(CodeGenerator &W) const { } void CppMainFile::compile(CodeGenerator &W) const { - kphp_assert(G->settings().is_server_mode() || G->settings().is_cli_mode()); + kphp_assert(G->is_output_mode_server() || G->is_output_mode_cli()); W << OpenFile("main.cpp"); W << ExternInclude("server/php-engine.h") << NL; diff --git a/compiler/code-gen/files/shape-keys.cpp b/compiler/code-gen/files/shape-keys.cpp index 1e72ae5fe1..18428273bf 100644 --- a/compiler/code-gen/files/shape-keys.cpp +++ b/compiler/code-gen/files/shape-keys.cpp @@ -14,15 +14,11 @@ std::string ShapeKeys::get_function_name() noexcept { return "init_shape_demangler"; } -std::string ShapeKeys::get_function_declaration() noexcept { - return "void " + get_function_name() + "()"; -} - void ShapeKeys::compile(CodeGenerator &W) const { W << OpenFile{"_shape_keys.cpp"}; W << ExternInclude{G->settings().runtime_headers.get()}; - FunctionSignatureGenerator{W} << NL << get_function_declaration() << BEGIN; + FunctionSignatureGenerator{W} << "void " << get_function_name() << "()" << BEGIN; W << "std::unordered_map shape_keys_storage{" << NL; for (const auto &[hash, key] : shape_keys_storage_) { diff --git a/compiler/code-gen/files/shape-keys.h b/compiler/code-gen/files/shape-keys.h index ab9fdaf383..585b1c32fb 100644 --- a/compiler/code-gen/files/shape-keys.h +++ b/compiler/code-gen/files/shape-keys.h @@ -18,7 +18,6 @@ struct ShapeKeys : CodeGenRootCmd { void compile(CodeGenerator &W) const final; static std::string get_function_name() noexcept; - static std::string get_function_declaration() noexcept; private: const std::map shape_keys_storage_; diff --git a/compiler/code-gen/files/vars-cpp.cpp b/compiler/code-gen/files/vars-cpp.cpp deleted file mode 100644 index e206c3b9ff..0000000000 --- a/compiler/code-gen/files/vars-cpp.cpp +++ /dev/null @@ -1,204 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/code-gen/files/vars-cpp.h" - -#include "common/algorithms/hashes.h" - -#include "compiler/code-gen/common.h" -#include "compiler/code-gen/declarations.h" -#include "compiler/code-gen/includes.h" -#include "compiler/code-gen/namespace.h" -#include "compiler/code-gen/raw-data.h" -#include "compiler/code-gen/vertex-compiler.h" -#include "compiler/data/src-file.h" -#include "compiler/data/var-data.h" -#include "compiler/stage.h" - -struct InitVar { - VarPtr var; - explicit InitVar(VarPtr var) : var(var) {} - - void compile(CodeGenerator &W) const { - Location save_location = stage::get_location(); - - VertexPtr init_val = var->init_val; - if (init_val->type() == op_conv_regexp) { - const auto &location = init_val->get_location(); - kphp_assert(location.function && location.file); - W << VarName(var) << ".init (" << var->init_val << ", " - << RawString(location.function->name) << ", " - << RawString(location.file->relative_file_name + ':' + std::to_string(location.line)) - << ");" << NL; - } else { - W << VarName(var) << " = " << var->init_val << ";" << NL; - } - - stage::set_location(save_location); - } -}; - - -static void add_dependent_declarations(VertexPtr vertex, std::set &dependent_vars) { - if (!vertex) { - return; - } - for (auto child: *vertex) { - add_dependent_declarations(child, dependent_vars); - } - if (auto var = vertex.try_as()) { - dependent_vars.emplace(var->var_id); - } -} - -void compile_raw_array(CodeGenerator &W, const VarPtr &var, int shift) { - if (shift == -1) { - W << InitVar(var); - W << VarName(var) << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL << NL; - return; - } - - W << VarName(var) << ".assign_raw((char *) &raw_arrays[" << shift << "]);" << NL << NL; -} - -static void compile_vars_part(CodeGenerator &W, const std::vector &vars, size_t part_id) { - std::string file_name = "vars" + std::to_string(part_id) + ".cpp"; - W << OpenFile(file_name, "o_vars_" + std::to_string(part_id / 100), false); - - W << ExternInclude(G->settings().runtime_headers.get()); - - DepLevelContainer const_raw_array_vars; - DepLevelContainer other_const_vars; - DepLevelContainer const_raw_string_vars; - std::set dependent_vars; - - IncludesCollector includes; - for (auto var : vars) { - if (!G->settings().is_static_lib_mode() || !var->is_builtin_global()) { - includes.add_var_signature_depends(var); - includes.add_vertex_depends(var->init_val); - } - } - W << includes; - - W << OpenNamespace(); - for (auto var : vars) { - if (G->settings().is_static_lib_mode() && var->is_builtin_global()) { - continue; - } - - W << VarDeclaration(var); - if (var->is_constant()) { - switch (var->init_val->type()) { - case op_string: - const_raw_string_vars.add(var); - break; - case op_array: - add_dependent_declarations(var->init_val, dependent_vars); - const_raw_array_vars.add(var); - break; - case op_var: - add_dependent_declarations(var->init_val, dependent_vars); - other_const_vars.add(var); - break; - default: - other_const_vars.add(var); - break; - } - } - } - - std::vector extern_depends; - std::set_difference(dependent_vars.begin(), dependent_vars.end(), - vars.begin(), vars.end(), std::back_inserter(extern_depends)); - for (auto var : extern_depends) { - W << VarExternDeclaration(var); - } - - std::vector values(const_raw_string_vars.size()); - std::transform(const_raw_string_vars.begin(), const_raw_string_vars.end(), - values.begin(), - [](const VarPtr &var){ return var->init_val.as()->get_string(); }); - auto const_string_shifts = compile_raw_data(W, values); - - const std::vector const_array_shifts = compile_arrays_raw_representation(const_raw_array_vars, W); - kphp_assert(const_array_shifts.size() == const_raw_array_vars.size()); - - - const size_t max_dep_level = std::max({const_raw_string_vars.max_dep_level(), const_raw_array_vars.max_dep_level(), other_const_vars.max_dep_level()}); - - size_t str_idx = 0; - size_t arr_idx = 0; - for (size_t dep_level = 0; dep_level < max_dep_level; ++dep_level) { - FunctionSignatureGenerator(W) << NL << "void const_vars_init_priority_" << dep_level << "_file_" << part_id << "()" << BEGIN; - - for (const auto &var : const_raw_string_vars.vars_by_dep_level(dep_level)) { - W << VarName(var) << ".assign_raw (&raw[" << const_string_shifts[str_idx++] << "]);" << NL; - } - - for (const auto &var : const_raw_array_vars.vars_by_dep_level(dep_level)) { - compile_raw_array(W, var, const_array_shifts[arr_idx++]); - } - - for (const auto &var: other_const_vars.vars_by_dep_level(dep_level)) { - W << InitVar(var); - const auto *type_data = var->tinf_node.get_type(); - PrimitiveType ptype = type_data->ptype(); - if (vk::any_of_equal(ptype, tp_array, tp_mixed, tp_string)) { - W << VarName(var); - if (type_data->use_optional()) { - W << ".val()"; - } - W << ".set_reference_counter_to(ExtraRefCnt::for_global_const);" << NL; - } - } - - W << END << NL; - } - - W << CloseNamespace(); - W << CloseFile(); -} - - -VarsCpp::VarsCpp(std::vector &&max_dep_levels, size_t parts_cnt) - : max_dep_levels_(std::move(max_dep_levels)) - , parts_cnt_(parts_cnt) { - kphp_assert(1 <= parts_cnt_); - kphp_error(parts_cnt_ <= G->settings().globals_split_count.get(), - fmt_format("Too many globals initialization .cpp files({}) for the specified globals_split_count({})", parts_cnt_, - G->settings().globals_split_count.get())); -} - -void VarsCpp::compile(CodeGenerator &W) const { - W << OpenFile("vars.cpp", "", false); - W << OpenNamespace(); - - FunctionSignatureGenerator(W) << "void const_vars_init() " << BEGIN; - - const int very_max_dep_level = *std::max_element(max_dep_levels_.begin(), max_dep_levels_.end()); - for (int dep_level = 0; dep_level <= very_max_dep_level; ++dep_level) { - for (size_t part_id = 0; part_id < parts_cnt_; ++part_id) { - if (dep_level <= max_dep_levels_[part_id]) { - auto init_fun = fmt_format("const_vars_init_priority_{}_file_{}();", dep_level, part_id); - // function declaration - W << "void " << init_fun << NL; - // function call - W << init_fun << NL; - } - } - } - W << END; - W << CloseNamespace(); - W << CloseFile(); -} - -VarsCppPart::VarsCppPart(std::vector &&vars_of_part, size_t part_id) - : vars_of_part_(std::move(vars_of_part)) - , part_id(part_id) {} - -void VarsCppPart::compile(CodeGenerator &W) const { - std::sort(vars_of_part_.begin(), vars_of_part_.end()); - compile_vars_part(W, vars_of_part_, part_id); -} diff --git a/compiler/code-gen/files/vars-cpp.h b/compiler/code-gen/files/vars-cpp.h deleted file mode 100644 index 6450761001..0000000000 --- a/compiler/code-gen/files/vars-cpp.h +++ /dev/null @@ -1,28 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include - -#include "compiler/code-gen/code-gen-root-cmd.h" -#include "compiler/code-gen/code-generator.h" - -struct VarsCpp : CodeGenRootCmd { - VarsCpp(std::vector &&max_dep_levels, size_t parts_cnt); - void compile(CodeGenerator &W) const final; - -private: - std::vector max_dep_levels_; - size_t parts_cnt_; -}; - -struct VarsCppPart : CodeGenRootCmd { - VarsCppPart(std::vector &&vars_of_part, size_t part_id); - void compile(CodeGenerator &W) const final; - -private: - mutable std::vector vars_of_part_; - size_t part_id; -}; diff --git a/compiler/code-gen/files/vars-reset.cpp b/compiler/code-gen/files/vars-reset.cpp deleted file mode 100644 index 3cd6f28e31..0000000000 --- a/compiler/code-gen/files/vars-reset.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/code-gen/files/vars-reset.h" - -#include "compiler/code-gen/common.h" -#include "compiler/code-gen/declarations.h" -#include "compiler/code-gen/includes.h" -#include "compiler/code-gen/namespace.h" -#include "compiler/code-gen/vertex-compiler.h" -#include "compiler/data/class-data.h" -#include "compiler/data/src-file.h" -#include "compiler/data/vars-collector.h" -#include "compiler/vertex.h" - -GlobalVarsReset::GlobalVarsReset(SrcFilePtr main_file) : - main_file_(main_file) { -} - -void GlobalVarsReset::declare_extern_for_init_val(VertexPtr v, std::set &externed_vars, CodeGenerator &W) { - if (auto var_vertex = v.try_as()) { - VarPtr var = var_vertex->var_id; - if (externed_vars.insert(var).second) { - W << VarExternDeclaration(var); - } - return; - } - for (VertexPtr son : *v) { - declare_extern_for_init_val(son, externed_vars, W); - } -} - -void GlobalVarsReset::compile_part(FunctionPtr func, const std::set &used_vars, int part_i, CodeGenerator &W) { - IncludesCollector includes; - for (auto var : used_vars) { - includes.add_var_signature_depends(var); - } - W << includes; - W << OpenNamespace(); - - std::set externed_vars; - for (auto var : used_vars) { - W << VarExternDeclaration(var); - if (var->init_val) { - declare_extern_for_init_val(var->init_val, externed_vars, W); - } - } - - FunctionSignatureGenerator(W) << "void " << GlobalVarsResetFuncName(func, part_i) << " " << BEGIN; - for (auto var : used_vars) { - if (G->settings().is_static_lib_mode() && var->is_builtin_global()) { - continue; - } - - W << "hard_reset_var(" << VarName(var); - //FIXME: brk and comments - if (var->init_val) { - W << ", " << var->init_val; - } - W << ");" << NL; - } - - W << END; - W << NL; - W << CloseNamespace(); -} - -void GlobalVarsReset::compile_func(FunctionPtr func, int parts_n, CodeGenerator &W) { - W << OpenNamespace(); - FunctionSignatureGenerator(W) << "void " << GlobalVarsResetFuncName(func) << " " << BEGIN; - - for (int i = 0; i < parts_n; i++) { - W << "void " << GlobalVarsResetFuncName(func, i) << ";" << NL; - W << GlobalVarsResetFuncName(func, i) << ";" << NL; - } - - W << END; - W << NL; - W << CloseNamespace(); -} - -void GlobalVarsReset::compile(CodeGenerator &W) const { - FunctionPtr main_func = main_file_->main_function; - - VarsCollector vars_collector{32}; - vars_collector.collect_global_and_static_vars_from(main_func); - auto used_vars = vars_collector.flush(); - - static const std::string vars_reset_src_prefix = "vars_reset."; - std::vector src_names(used_vars.size()); - for (int i = 0; i < used_vars.size(); i++) { - src_names[i] = vars_reset_src_prefix + std::to_string(i) + "." + main_func->src_name; - } - - for (int i = 0; i < used_vars.size(); i++) { - W << OpenFile(src_names[i], "o_vars_reset", false); - W << ExternInclude(G->settings().runtime_headers.get()); - compile_part(main_func, used_vars[i], i, W); - W << CloseFile(); - } - - W << OpenFile(vars_reset_src_prefix + main_func->src_name, "", false); - W << ExternInclude(G->settings().runtime_headers.get()); - compile_func(main_func, used_vars.size(), W); - W << CloseFile(); -} diff --git a/compiler/code-gen/files/vars-reset.h b/compiler/code-gen/files/vars-reset.h deleted file mode 100644 index 23387c9779..0000000000 --- a/compiler/code-gen/files/vars-reset.h +++ /dev/null @@ -1,25 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include "compiler/code-gen/code-gen-root-cmd.h" -#include "compiler/code-gen/code-generator.h" -#include "compiler/data/data_ptr.h" -#include "compiler/data/vertex-adaptor.h" - -struct GlobalVarsReset : CodeGenRootCmd { - explicit GlobalVarsReset(SrcFilePtr main_file); - - void compile(CodeGenerator &W) const final; - - static void compile_part(FunctionPtr func, const std::set &used_vars, int part_i, CodeGenerator &W); - - static void compile_func(FunctionPtr func, int parts_n, CodeGenerator &W); - - static void declare_extern_for_init_val(VertexPtr v, std::set &externed_vars, CodeGenerator &W); - -private: - SrcFilePtr main_file_; -}; diff --git a/compiler/code-gen/naming.h b/compiler/code-gen/naming.h index d4b8c77c8f..e17f9150f1 100644 --- a/compiler/code-gen/naming.h +++ b/compiler/code-gen/naming.h @@ -6,8 +6,6 @@ #include -#include "common/type_traits/list_of_types.h" - #include "compiler/code-gen/common.h" #include "compiler/code-gen/gen-out-style.h" #include "compiler/data/function-data.h" @@ -220,31 +218,8 @@ struct VarName { void compile(CodeGenerator &W) const { if (!name.empty()) { W << name; - return; - } - - if (var->is_function_static_var()) { - W << FunctionName(var->holder_func) << "$"; - } - - W << "v$" << var->name; - } -}; - -struct GlobalVarsResetFuncName { - explicit GlobalVarsResetFuncName(FunctionPtr main_func, int part = -1) : - main_func_(main_func), - part_(part) {} - - void compile(CodeGenerator &W) const { - W << FunctionName(main_func_) << "$global_vars_reset"; - if (part_ >= 0) { - W << std::to_string(part_); + } else { + W << "v$" << var->name; } - W << "()"; } - -private: - const FunctionPtr main_func_; - const int part_{-1}; }; diff --git a/compiler/code-gen/raw-data.h b/compiler/code-gen/raw-data.h index a4ba330460..1c1989708b 100644 --- a/compiler/code-gen/raw-data.h +++ b/compiler/code-gen/raw-data.h @@ -87,9 +87,32 @@ class RawString { std::vector compile_arrays_raw_representation(const DepLevelContainer &const_raw_array_vars, CodeGenerator &W); -template ().begin()), - typename = decltype(std::declval().end())> +//returns len of raw string representation or -1 on error +inline int string_raw_len(int src_len) { + if (src_len < 0 || src_len >= (1 << 30) - 13) { + return -1; + } + + return src_len + 13; +} + +//returns len of raw string representation and writes it to dest or returns -1 on error +inline int string_raw(char *dest, int dest_len, const char *src, int src_len) { + int raw_len = string_raw_len(src_len); + if (raw_len == -1 || raw_len > dest_len) { + return -1; + } + int *dest_int = reinterpret_cast (dest); + dest_int[0] = src_len; + dest_int[1] = src_len; + dest_int[2] = ExtraRefCnt::for_global_const; + memcpy(dest + 3 * sizeof(int), src, src_len); + dest[3 * sizeof(int) + src_len] = '\0'; + + return raw_len; +} + +template std::vector compile_raw_data(CodeGenerator &W, const Container &values) { std::string raw_data; std::vector const_string_shifts(values.size()); diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index 6f50d95b45..d75ce37105 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -11,6 +11,7 @@ #include "common/wrappers/likely.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/json-encoder-tags.h" #include "compiler/code-gen/files/tracing-autogen.h" @@ -1017,7 +1018,9 @@ void compile_foreach_ref_header(VertexAdaptor root, CodeGenerator &W BEGIN; - //save value + // save value: codegen `T &v$name = it.get_value()` + // note, that in global scope `v$name` remains a C++ variable (though other mutable globals are placed in linear mem) + // (there are also compile-time checks that foreach-ref global vars aren't used outside a loop) W << TypeName(tinf::get_type(x)) << " &"; W << x << " = " << it << ".get_value();" << NL; @@ -1345,13 +1348,16 @@ void compile_function_resumable(VertexAdaptor func_root, CodeGenera //MEMBER VARIABLES - for (auto var : func->param_ids) { + for (VarPtr var : func->param_ids) { kphp_error(!var->is_reference, "reference function parametrs are forbidden in resumable mode"); W << VarPlainDeclaration(var); } - for (auto var : func->local_var_ids) { + for (VarPtr var : func->local_var_ids) { W << VarPlainDeclaration(var); // inplace variables are stored as Resumable class fields as well } + if (func->has_global_vars_inside) { + W << PhpMutableGlobalsDeclareInResumableClass(); + } if (func->kphp_tracing) { // append KphpTracingFuncCallGuard as a member variable also ('start()' is called below) TracingAutogen::codegen_runtime_func_guard_declaration(W, func); } @@ -1363,19 +1369,34 @@ void compile_function_resumable(VertexAdaptor func_root, CodeGenera //CONSTRUCTOR FunctionSignatureGenerator(W) << FunctionClassName(func) << "(" << FunctionParams(func) << ")"; - if (!func->param_ids.empty()) { + bool has_members_in_constructor = !func->param_ids.empty() || !func->local_var_ids.empty() || func->has_global_vars_inside; + if (has_members_in_constructor) { + bool any_inited = false; W << " :" << NL << Indent(+2); - W << JoinValues(func->param_ids, ",", join_mode::multiple_lines, - [](CodeGenerator &W, VarPtr var) { - W << VarName(var) << "(" << VarName(var) << ")"; - }); + if (!func->param_ids.empty()) { + W << JoinValues(func->param_ids, ",", join_mode::multiple_lines, + [](CodeGenerator &W, VarPtr var) { + W << VarName(var) << "(" << VarName(var) << ")"; + }); + any_inited = true; + } if (!func->local_var_ids.empty()) { - W << "," << NL; + if (any_inited) { + W << "," << NL; + } + W << JoinValues(func->local_var_ids, ",", join_mode::multiple_lines, + [](CodeGenerator &W, VarPtr var) { + W << VarName(var) << "()"; + }); + any_inited = true; + } + if (func->has_global_vars_inside) { + if (any_inited) { + W << "," << NL; + } + W << PhpMutableGlobalsAssignInResumableConstructor(); + any_inited = true; } - W << JoinValues(func->local_var_ids, ",", join_mode::multiple_lines, - [](CodeGenerator &W, VarPtr var) { - W << VarName(var) << "()"; - }); W << Indent(-2); } W << " " << BEGIN << END << NL; @@ -1434,6 +1455,9 @@ void compile_function(VertexAdaptor func_root, CodeGenerator &W) { } W << FunctionDeclaration(func, false) << " " << BEGIN; + if (func->has_global_vars_inside) { + W << PhpMutableGlobalsAssignCurrent() << NL; + } if (func->kphp_tracing) { TracingAutogen::codegen_runtime_func_guard_declaration(W, func); @@ -1680,9 +1704,9 @@ bool try_compile_append_inplace(VertexAdaptor root, CodeGenerator &W // append all strings directly to the $lhs without creating a temporary // string for the $rhs result; also, grow $lhs accordingly, so it // can fit all the string parts - if (root->lhs()->type() == op_var) { - kphp_assert(tinf::get_type(root->lhs().as())->ptype() == tp_string); - compile_string_build_impl(root->rhs().as(), VarName{root->lhs().as()->var_id}, lhs_type, W); + if (auto as_var = root->lhs().try_as(); as_var && !as_var->var_id->is_in_global_scope()) { + kphp_assert(tinf::get_type(as_var)->ptype() == tp_string); + compile_string_build_impl(root->rhs().as(), VarName{as_var->var_id}, lhs_type, W); return true; } W << "(" << BEGIN; @@ -2131,9 +2155,21 @@ void compile_common_op(VertexPtr root, CodeGenerator &W) { case op_null: W << "Optional{}"; break; - case op_var: - W << VarName(root.as()->var_id); + case op_var: { + VarPtr var_id = root.as()->var_id; + if (var_id->is_constant()) { + // auto-extracted constant variables (const strings, arrays, etc.) in codegen are C++ variables + W << var_id->name; + } else if (var_id->is_in_global_scope() && !var_id->is_foreach_reference) { + // mutable globals, as opposed, are not C++ variables: instead, + // they all are placed in linear memory chunks, see php-script-globals.h + // with the only exception of `foreach (... as &$ref)` in global scope, see compile_foreach_ref_header() + W << GlobalVarInPhpGlobals(var_id); + } else { + W << VarName(var_id); + } break; + } case op_string: compile_string(root.as(), W); break; diff --git a/compiler/compiler-core.cpp b/compiler/compiler-core.cpp index 2be235feb0..014d04ec89 100644 --- a/compiler/compiler-core.cpp +++ b/compiler/compiler-core.cpp @@ -121,6 +121,14 @@ void CompilerCore::finish() { void CompilerCore::register_settings(CompilerSettings *settings) { kphp_assert (settings_ == nullptr); settings_ = settings; + + if (settings->mode.get() == "cli") { + output_mode = OutputMode::cli; + } else if (settings->mode.get() == "lib") { + output_mode = OutputMode::lib; + } else { + output_mode = OutputMode::server; + } } const CompilerSettings &CompilerCore::settings() const { @@ -197,6 +205,11 @@ FFIRoot &CompilerCore::get_ffi_root() { return ffi; } +OutputMode CompilerCore::get_output_mode() const { + return output_mode; +} + + vk::string_view CompilerCore::calc_relative_name(SrcFilePtr file, bool builtin) const { vk::string_view full_file_name = file->file_name; if (full_file_name.starts_with(settings_->base_dir.get())) { @@ -463,59 +476,72 @@ VarPtr CompilerCore::create_var(const std::string &name, VarData::Type type) { return var; } -VarPtr CompilerCore::get_global_var(const std::string &name, VarData::Type type, - VertexPtr init_val, bool *is_new_inserted) { - auto *node = global_vars_ht.at(vk::std_hash(name)); +VarPtr CompilerCore::get_global_var(const std::string &name, VertexPtr init_val) { + auto *node = globals_ht.at(vk::std_hash(name)); + + if (!node->data) { + AutoLocker locker(node); + if (!node->data) { + node->data = create_var(name, VarData::var_global_t); + node->data->init_val = init_val; + node->data->is_builtin_runtime = VarData::does_name_eq_any_builtin_runtime(name); + } + } + + return node->data; +} + +VarPtr CompilerCore::get_constant_var(const std::string &name, VertexPtr init_val, bool *is_new_inserted) { + auto *node = constants_ht.at(vk::std_hash(name)); VarPtr new_var; if (!node->data) { AutoLocker locker(node); if (!node->data) { - new_var = create_var(name, type); + new_var = create_var(name, VarData::var_const_t); new_var->init_val = init_val; node->data = new_var; } } - if (init_val) { + // when a string 'str' meets in several places in php code (same for [1,2,3] and other consts) + // it's created by a thread that first found it, and all others just ref to the same node + // here we make var->init_val->location stable, as it's sometimes used in code generation (locations of regexps, for example) + if (!new_var) { AutoLocker locker(node); - // to avoid of unstable locations, order them - if (node->data->init_val && node->data->init_val->get_location() < init_val->get_location()) { + if (node->data->init_val->get_location() < init_val->get_location()) { std::swap(node->data->init_val, init_val); } } - VarPtr var = node->data; if (is_new_inserted) { *is_new_inserted = static_cast(new_var); } + + VarPtr var = node->data; + // assert that one and the same init_val leads to one and the same var if (!new_var) { + kphp_assert(init_val); + kphp_assert(var->init_val->type() == init_val->type()); kphp_assert_msg(var->name == name, fmt_format("bug in compiler (hash collision) {} {}", var->name, name)); - if (init_val) { - kphp_assert(var->init_val->type() == init_val->type()); - switch (init_val->type()) { - case op_string: - kphp_assert(var->init_val->get_string() == init_val->get_string()); - break; - case op_conv_regexp: { - std::string &new_regexp = init_val.as()->expr().as()->str_val; - std::string &hashed_regexp = var->init_val.as()->expr().as()->str_val; - std::string msg = "hash collision: " + new_regexp + "; " + hashed_regexp; - - kphp_assert_msg(hashed_regexp == new_regexp, msg.c_str()); - break; - } - case op_array: { - std::string new_array_repr = VertexPtrFormatter::to_string(init_val); - std::string hashed_array_repr = VertexPtrFormatter::to_string(var->init_val); - - std::string msg = "hash collision: " + new_array_repr + "; " + hashed_array_repr; - - kphp_assert_msg(new_array_repr == hashed_array_repr, msg.c_str()); - break; - } - default: - break; + + switch (init_val->type()) { + case op_string: + kphp_assert(var->init_val->get_string() == init_val->get_string()); + break; + case op_conv_regexp: { + const std::string &new_regexp = init_val.as()->expr().as()->str_val; + const std::string &hashed_regexp = var->init_val.as()->expr().as()->str_val; + kphp_assert_msg(hashed_regexp == new_regexp, fmt_format("hash collision of regexp: {} vs {}", new_regexp, hashed_regexp)); + break; + } + case op_array: { + std::string new_array_repr = VertexPtrFormatter::to_string(init_val); + std::string hashed_array_repr = VertexPtrFormatter::to_string(var->init_val); + kphp_assert_msg(new_array_repr == hashed_array_repr, fmt_format("hash collision of arrays: {} vs {}", new_array_repr, hashed_array_repr)); + break; } + default: + break; } } return var; @@ -544,13 +570,22 @@ VarPtr CompilerCore::create_local_var(FunctionPtr function, const std::string &n } std::vector CompilerCore::get_global_vars() { - // static class variables are registered as globals, but if they're unused, - // then their types were never calculated; we don't need to export them to vars.cpp - return global_vars_ht.get_all_if([](VarPtr v) { - return v->tinf_node.was_recalc_finished_at_least_once(); + return globals_ht.get_all_if([](VarPtr v) { + // traits' static vars are added at the moment of parsing (class-members.cpp) + // but later never used, and tinf never executed for them + if (v->is_class_static_var() && v->class_id->is_trait()) { + return false; + } + // static vars for classes that are unused, are also present here + // probably, in the future, we'll detect unused globals and don't export them to C++ even as Unknown + return true; }); } +std::vector CompilerCore::get_constants_vars() { + return constants_ht.get_all(); +} + std::vector CompilerCore::get_classes() { return classes_ht.get_all(); } diff --git a/compiler/compiler-core.h b/compiler/compiler-core.h index 4ef7072f5a..944f0fec47 100644 --- a/compiler/compiler-core.h +++ b/compiler/compiler-core.h @@ -8,22 +8,27 @@ /*** Core ***/ //Consists mostly of functions that require synchronization -#include #include #include #include "common/algorithms/hashes.h" +#include "compiler/compiler-settings.h" +#include "compiler/composer.h" #include "compiler/data/data_ptr.h" #include "compiler/data/ffi-data.h" -#include "compiler/compiler-settings.h" +#include "compiler/function-colors.h" #include "compiler/index.h" #include "compiler/stats.h" #include "compiler/threading/data-stream.h" #include "compiler/threading/hash-table.h" #include "compiler/tl-classes.h" -#include "compiler/composer.h" -#include "compiler/function-colors.h" + +enum class OutputMode { + server, // -M server + cli, // -M cli + lib, // -M lib +}; class CompilerCore { private: @@ -33,13 +38,15 @@ class CompilerCore { TSHashTable functions_ht; TSHashTable classes_ht; TSHashTable defines_ht; - TSHashTable global_vars_ht; + TSHashTable constants_ht; // auto-collected constants (const strings / arrays / regexps / pure func calls); are inited once in a master process + TSHashTable globals_ht; // mutable globals (vars in global scope, class static fields); are reset for each php script inside worker processes TSHashTable libs_ht; TSHashTable modulites_ht; TSHashTable composer_json_ht; SrcFilePtr main_file; CompilerSettings *settings_; ComposerAutoloader composer_class_loader; + OutputMode output_mode; FFIRoot ffi; ClassPtr memcache_class; TlClasses tl_classes; @@ -67,6 +74,7 @@ class CompilerCore { SrcDirPtr register_dir(vk::string_view full_dir_name); FFIRoot &get_ffi_root(); + OutputMode get_output_mode() const; void register_main_file(const std::string &file_name, DataStream &os); SrcFilePtr require_file(const std::string &file_name, LibPtr owner_lib, DataStream &os, bool error_if_not_exists = true, bool builtin = false); @@ -103,11 +111,13 @@ class CompilerCore { DefinePtr get_define(std::string_view name); VarPtr create_var(const std::string &name, VarData::Type type); - VarPtr get_global_var(const std::string &name, VarData::Type type, VertexPtr init_val, bool *is_new_inserted = nullptr); + VarPtr get_global_var(const std::string &name, VertexPtr init_val); + VarPtr get_constant_var(const std::string &name, VertexPtr init_val, bool *is_new_inserted = nullptr); VarPtr create_local_var(FunctionPtr function, const std::string &name, VarData::Type type); SrcFilePtr get_main_file() { return main_file; } std::vector get_global_vars(); + std::vector get_constants_vars(); std::vector get_classes(); std::vector get_interfaces(); std::vector get_defines(); @@ -154,6 +164,18 @@ class CompilerCore { return is_functions_txt_parsed; } + bool is_output_mode_server() const { + return output_mode == OutputMode::server; + } + + bool is_output_mode_cli() const { + return output_mode == OutputMode::cli; + } + + bool is_output_mode_lib() const { + return output_mode == OutputMode::lib; + } + Stats stats; }; diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index df948b0d98..67560eeca1 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -204,18 +204,6 @@ void CompilerSettings::option_as_dir(KphpOption &path_option) noexc path_option.value_ = as_dir(path_option.value_); } -bool CompilerSettings::is_static_lib_mode() const { - return mode.get() == "lib"; -} - -bool CompilerSettings::is_server_mode() const { - return mode.get() == "server"; -} - -bool CompilerSettings::is_cli_mode() const { - return mode.get() == "cli"; -} - bool CompilerSettings::is_composer_enabled() const { return !composer_root.get().empty(); } @@ -230,7 +218,7 @@ void CompilerSettings::init() { runtime_sha256_file.value_ = get_full_path(runtime_sha256_file.get()); link_file.value_ = get_full_path(link_file.get()); - if (is_static_lib_mode()) { + if (mode.get() == "lib") { if (!tl_schema_file.get().empty()) { throw std::runtime_error{"Option " + tl_schema_file.get_env_var() + " is forbidden for static lib mode"}; } @@ -264,10 +252,6 @@ void CompilerSettings::init() { throw std::runtime_error{"Option " + threads_count.get_env_var() + " is expected to be <= " + std::to_string(MAX_THREADS_COUNT)}; } - if (globals_split_count.get() == 0) { - throw std::runtime_error{"globals-split-count may not be equal to zero"}; - } - for (std::string &include : includes.value_) { include = as_dir(include); } diff --git a/compiler/compiler-settings.h b/compiler/compiler-settings.h index 3a9e99a277..11342b99e7 100644 --- a/compiler/compiler-settings.h +++ b/compiler/compiler-settings.h @@ -128,7 +128,6 @@ class CompilerSettings : vk::not_copyable { KphpOption no_make; KphpOption jobs_count; KphpOption threads_count; - KphpOption globals_split_count; KphpOption require_functions_typing; KphpOption require_class_typing; @@ -185,9 +184,6 @@ class CompilerSettings : vk::not_copyable { KphpImplicitOption tl_classname_prefix; std::string get_version() const; - bool is_static_lib_mode() const; - bool is_server_mode() const; - bool is_cli_mode() const; bool is_composer_enabled() const; // reports whether composer compatibility mode is on color_settings get_color_settings() const; diff --git a/compiler/compiler.cmake b/compiler/compiler.cmake index 636aaaa6d6..6ae6aadadf 100644 --- a/compiler/compiler.cmake +++ b/compiler/compiler.cmake @@ -46,8 +46,7 @@ prepend(KPHP_COMPILER_DATA_SOURCES data/ src-dir.cpp src-file.cpp var-data.cpp - ffi-data.cpp - vars-collector.cpp) + ffi-data.cpp) prepend(KPHP_COMPILER_INFERRING_SOURCES inferring/ expr-node.cpp @@ -72,11 +71,14 @@ prepend(KPHP_COMPILER_INFERRING_SOURCES inferring/ prepend(KPHP_COMPILER_CODEGEN_SOURCES code-gen/ code-gen-task.cpp code-generator.cpp + const-globals-batched-mem.cpp declarations.cpp files/cmake-lists-txt.cpp + files/const-vars-init.cpp files/function-header.cpp files/function-source.cpp - files/global_vars_memory_stats.cpp + files/global-vars-memory-stats.cpp + files/global-vars-reset.cpp files/init-scripts.cpp files/json-encoder-tags.cpp files/lib-header.cpp @@ -91,8 +93,6 @@ prepend(KPHP_COMPILER_CODEGEN_SOURCES code-gen/ files/shape-keys.cpp files/tracing-autogen.cpp files/type-tagger.cpp - files/vars-cpp.cpp - files/vars-reset.cpp includes.cpp raw-data.cpp vertex-compiler.cpp diff --git a/compiler/data/class-members.cpp b/compiler/data/class-members.cpp index 16d006d7ae..434b62df58 100644 --- a/compiler/data/class-members.cpp +++ b/compiler/data/class-members.cpp @@ -58,7 +58,7 @@ inline ClassMemberStaticField::ClassMemberStaticField(ClassPtr klass, VertexAdap type_hint(type_hint) { std::string global_var_name = replace_backslashes(klass->name) + "$$" + root->get_string(); - var = G->get_global_var(global_var_name, VarData::var_global_t, def_val); + var = G->get_global_var(global_var_name, def_val); root->var_id = var; var->init_val = def_val; var->class_id = klass; diff --git a/compiler/data/function-data.h b/compiler/data/function-data.h index ed11dc03ca..bfe1139501 100644 --- a/compiler/data/function-data.h +++ b/compiler/data/function-data.h @@ -124,6 +124,7 @@ class FunctionData { bool has_lambdas_inside = false; // used for optimization after cloning (not to launch CloneNestedLambdasPass) bool has_var_tags_inside = false; // used for optimization (not to traverse body if no @var inside) bool has_commentTs_inside = false; // used for optimization (not to traverse body if no /*<...>*/ inside) + bool has_global_vars_inside = false; // used for codegeneration; true if function body refs any mutable php globals/superglobals (after cfg pass) bool warn_unused_result = false; bool is_flatten = false; bool is_pure = false; diff --git a/compiler/data/var-data.cpp b/compiler/data/var-data.cpp index 5aebcaaf27..6c7dc36915 100644 --- a/compiler/data/var-data.cpp +++ b/compiler/data/var-data.cpp @@ -32,13 +32,21 @@ const ClassMemberInstanceField *VarData::as_class_instance_field() const { return class_id->members.get_instance_field(name); } -// TODO Dirty HACK, should be removed -bool VarData::does_name_eq_any_builtin_global(const std::string &name) { - static const std::unordered_set names = { - "_SERVER", "_GET", "_POST", "_FILES", "_COOKIE", "_REQUEST", "_ENV", "argc", "argv", - "MC", "MC_True", "config", "Durov", "FullMCTime", "KPHP_MC_WRITE_STAT_PROBABILITY", - "d$PHP_SAPI"}; - return names.find(name) != names.end(); +bool VarData::does_name_eq_any_language_superglobal(const std::string &name) { + // these vars are 'superglobals' in PHP language: they are available in all scopes + static const std::unordered_set superglobal_names = { + "_SERVER", "_GET", "_POST", "_ENV", "_FILES", "_COOKIE", "_REQUEST", + }; + return superglobal_names.find(name) != superglobal_names.end(); +} + +bool VarData::does_name_eq_any_builtin_runtime(const std::string &name) { + // these vars are runtime built-ins, see PhpScriptBuiltInSuperGlobals + static const std::unordered_set runtime_names = { + "_SERVER", "_GET", "_POST", "_ENV", "_FILES", "_COOKIE", "_REQUEST", + "argc", "argv", "d$PHP_SAPI" + }; + return runtime_names.find(name) != runtime_names.end(); } bool operator<(VarPtr a, VarPtr b) { diff --git a/compiler/data/var-data.h b/compiler/data/var-data.h index c6d8572ea0..2da4b9de33 100644 --- a/compiler/data/var-data.h +++ b/compiler/data/var-data.h @@ -44,7 +44,10 @@ class VarData { bool marked_as_const = false; bool is_read_only = true; bool is_foreach_reference = false; - int dependency_level = 0; + bool is_builtin_runtime = false; // $_SERVER, $argv, etc., see PhpScriptBuiltInSuperGlobals in runtime + int dependency_level = 0; // for constants only (c_str$, c_arr$, etc) + int offset_in_linear_mem = -1; // for globals only (offset in g_linear_mem) + int batch_idx = -1; // for constants and globals, a number [0;N), see const-globals-batched-mem.h void set_uninited_flag(bool f); bool get_uninited_flag(); @@ -79,12 +82,9 @@ class VarData { return type_ == var_const_t; } - inline bool is_builtin_global() const { - return type_ == var_global_t && does_name_eq_any_builtin_global(name); - } - const ClassMemberStaticField *as_class_static_field() const; const ClassMemberInstanceField *as_class_instance_field() const; - static bool does_name_eq_any_builtin_global(const std::string &name); + static bool does_name_eq_any_language_superglobal(const std::string &name); + static bool does_name_eq_any_builtin_runtime(const std::string &name); }; diff --git a/compiler/data/vars-collector.cpp b/compiler/data/vars-collector.cpp deleted file mode 100644 index cafde30277..0000000000 --- a/compiler/data/vars-collector.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include "compiler/data/vars-collector.h" - -#include "common/algorithms/hashes.h" - -#include "compiler/data/class-data.h" -#include "compiler/data/function-data.h" -#include "compiler/data/src-file.h" -#include "compiler/data/var-data.h" - -VarsCollector::VarsCollector(size_t parts, std::function vars_checker) : - collected_vars_(parts), - vars_checker_(std::move(vars_checker)) { -} - -void VarsCollector::collect_global_and_static_vars_from(FunctionPtr function) { - if (!visited_functions_.emplace(function).second) { - return; - } - - for (auto dep_function : function->dep) { - collect_global_and_static_vars_from(dep_function); - } - - add_vars(function->global_var_ids.begin(), function->global_var_ids.end()); - add_vars(function->static_var_ids.begin(), function->static_var_ids.end()); -} - -std::vector> VarsCollector::flush() { - visited_functions_.clear(); - - auto last_part = std::remove_if(collected_vars_.begin(), collected_vars_.end(), - [](const std::set &p) { return p.empty(); }); - collected_vars_.erase(last_part, collected_vars_.end()); - return std::move(collected_vars_); -} - -template -void VarsCollector::add_vars(It begin, It end) { - for (; begin != end; begin++) { - VarPtr var_id = *begin; - if (vars_checker_ && !vars_checker_(var_id)) { - continue; - } - const size_t var_hash = var_id->class_id ? - vk::std_hash(var_id->class_id->file_id->main_func_name) : - vk::std_hash(var_id->name); - - const size_t bucket = var_hash % collected_vars_.size(); - collected_vars_[bucket].emplace(var_id); - } -} diff --git a/compiler/data/vars-collector.h b/compiler/data/vars-collector.h deleted file mode 100644 index e97bacf320..0000000000 --- a/compiler/data/vars-collector.h +++ /dev/null @@ -1,27 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2020 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include -#include -#include - -#include "compiler/data/data_ptr.h" - -class VarsCollector { -public: - explicit VarsCollector(size_t parts, std::function vars_checker = {}); - - void collect_global_and_static_vars_from(FunctionPtr function); - std::vector> flush(); - -private: - template - void add_vars(It begin, It end); - - std::unordered_set visited_functions_; - std::vector> collected_vars_; - std::function vars_checker_; -}; diff --git a/compiler/kphp2cpp.cpp b/compiler/kphp2cpp.cpp index 48718d9fd3..94944e16d4 100644 --- a/compiler/kphp2cpp.cpp +++ b/compiler/kphp2cpp.cpp @@ -235,8 +235,6 @@ int main(int argc, char *argv[]) { 'j', "jobs-num", "KPHP_JOBS_COUNT", std::to_string(get_default_threads_count())); parser.add("Threads number for the transpilation", settings->threads_count, 't', "threads-count", "KPHP_THREADS_COUNT", std::to_string(get_default_threads_count())); - parser.add("Count of global variables per dedicated .cpp file. Lowering it could decrease compilation time", settings->globals_split_count, - "globals-split-count", "KPHP_GLOBALS_SPLIT_COUNT", "1536"); parser.add("Builtin tl schema. Incompatible with lib mode", settings->tl_schema_file, 'T', "tl-schema", "KPHP_TL_SCHEMA"); parser.add("Generate storers and fetchers for internal tl functions", settings->gen_tl_internals, diff --git a/compiler/lambda-utils.cpp b/compiler/lambda-utils.cpp index e654213301..093ebac69a 100644 --- a/compiler/lambda-utils.cpp +++ b/compiler/lambda-utils.cpp @@ -439,7 +439,7 @@ void auto_capture_vars_from_body_in_arrow_lambda(FunctionPtr f_lambda) { return var_name != "this" && // $this is captured by another approach, in non-arrow lambdas also var_name.find("::") == std::string::npos && var_name.find("$u") == std::string::npos && // not a superlocal var created in gentree - !VertexUtil::is_superglobal(var_name) && + !VarData::does_name_eq_any_language_superglobal(var_name) && !f_lambda->find_param_by_name(var_name) && !f_lambda->find_use_by_name(var_name); }; diff --git a/compiler/make/make.cpp b/compiler/make/make.cpp index 40206ea103..988d910585 100644 --- a/compiler/make/make.cpp +++ b/compiler/make/make.cpp @@ -388,7 +388,7 @@ static std::forward_list collect_imported_headers() { return imported_headers; } -static std::vector run_pre_make(const CompilerSettings &settings, FILE *make_stats_file, MakeSetup &make, Index &obj_index, File &bin_file) { +static std::vector run_pre_make(OutputMode output_mode, const CompilerSettings &settings, FILE *make_stats_file, MakeSetup &make, Index &obj_index, File &bin_file) { AutoProfiler profiler{get_profiler("Prepare Targets For Build")}; G->del_extra_files(); @@ -405,12 +405,13 @@ static std::vector run_pre_make(const CompilerSettings &settings, FILE * } auto lib_header_dirs = collect_imported_headers(); - return settings.is_static_lib_mode() ? kphp_make_static_lib_target(obj_index, G->get_index(), lib_header_dirs, make) - : kphp_make_target(obj_index, G->get_index(), lib_header_dirs, make); + return output_mode == OutputMode::lib ? kphp_make_static_lib_target(obj_index, G->get_index(), lib_header_dirs, make) + : kphp_make_target(obj_index, G->get_index(), lib_header_dirs, make); } void run_make() { const auto &settings = G->settings(); + OutputMode output_mode = G->get_output_mode(); FILE *make_stats_file = nullptr; if (!settings.stats_file.get().empty()) { make_stats_file = fopen(settings.stats_file.get().c_str(), "w"); @@ -425,10 +426,10 @@ void run_make() { kphp_assert(bin_file.read_stat() >= 0); MakeSetup make{make_stats_file, settings}; - auto objs = run_pre_make(settings, make_stats_file, make, obj_index, bin_file); + auto objs = run_pre_make(output_mode, settings, make_stats_file, make, obj_index, bin_file); stage::die_if_global_errors(); - if (settings.is_static_lib_mode()) { + if (output_mode == OutputMode::lib) { make.create_objs2static_lib_target(objs, &bin_file); } else { const std::string build_stage{"Compiling"}; @@ -440,7 +441,7 @@ void run_make() { } stage::die_if_global_errors(); - const std::string build_stage{settings.is_static_lib_mode() ? "Compiling" : "Linking"}; + const std::string build_stage{output_mode == OutputMode::lib ? "Compiling" : "Linking"}; AutoProfiler profiler{get_profiler(build_stage)}; bool ok = make.make_target(&bin_file, build_stage, settings.jobs_count.get()); @@ -459,7 +460,7 @@ void run_make() { if (!settings.user_binary_path.get().empty()) { hard_link_or_copy(bin_file.path, settings.user_binary_path.get()); } - if (settings.is_static_lib_mode()) { + if (output_mode == OutputMode::lib) { copy_static_lib_to_out_dir(std::move(bin_file)); } } diff --git a/compiler/name-gen.cpp b/compiler/name-gen.cpp index b41a788441..0538a26029 100644 --- a/compiler/name-gen.cpp +++ b/compiler/name-gen.cpp @@ -23,34 +23,6 @@ std::string gen_anonymous_function_name(FunctionPtr function) { return gen_unique_name("lambda", function); } -std::string gen_const_string_name(const std::string &str) { - return fmt_format("const_string$us{:x}", vk::std_hash(str)); -} - -std::string gen_const_regexp_name(const std::string &str) { - return fmt_format("const_regexp$us{:x}", vk::std_hash(str)); -} - -bool is_array_suitable_for_hashing(VertexPtr vertex) { - return vertex->type() == op_array && CheckConst::is_const(vertex); -} - -// checks that inlined as define' value constructor is suitable to be stored as constant var -bool is_object_suitable_for_hashing(VertexPtr vertex) { - return vertex->type() == op_define_val && vertex.as()->value()->type() == op_func_call - && vertex.as()->value()->extra_type == op_ex_constructor_call && vertex->const_type == cnst_const_val; -} - -std::string gen_const_object_name(const VertexAdaptor &def) { - kphp_assert_msg(def->value()->type() == op_func_call, "Internal error: expected op_define_val "); - auto obj_hash = ObjectHash::calc_hash(def); - return fmt_format("const_obj$us{:x}", obj_hash); -} - -std::string gen_const_array_name(const VertexAdaptor &array) { - return fmt_format("const_array$us{:x}", ArrayHash::calc_hash(array)); -} - std::string gen_unique_name(const std::string& prefix, FunctionPtr function) { if (!function) { function = stage::get_function(); diff --git a/compiler/name-gen.h b/compiler/name-gen.h index cf8ad75534..619b53fee9 100644 --- a/compiler/name-gen.h +++ b/compiler/name-gen.h @@ -12,12 +12,6 @@ std::string gen_anonymous_scope_name(FunctionPtr parent_function); std::string gen_anonymous_function_name(FunctionPtr parent_function); std::string gen_unique_name(const std::string& prefix, FunctionPtr function = FunctionPtr{}); -std::string gen_const_string_name(const std::string &str); -std::string gen_const_regexp_name(const std::string &str); -bool is_object_suitable_for_hashing(VertexPtr vertex); -std::string gen_const_object_name(const VertexAdaptor &array); -bool is_array_suitable_for_hashing(VertexPtr vertex); -std::string gen_const_array_name(const VertexAdaptor &array); std::string resolve_uses(FunctionPtr resolve_context, const std::string &class_name); ClassPtr resolve_class_of_arrow_access(FunctionPtr function, VertexPtr lhs, VertexPtr v); diff --git a/compiler/pipes/analyzer.cpp b/compiler/pipes/analyzer.cpp index 6405a7c776..11b60338a0 100644 --- a/compiler/pipes/analyzer.cpp +++ b/compiler/pipes/analyzer.cpp @@ -92,6 +92,12 @@ VertexPtr CommonAnalyzerPass::on_enter_vertex(VertexPtr vertex) { if (var->is_constant()) { run_function_pass(var->init_val, this); } + if (var->is_in_global_scope()) { + // save a flag, that a function's body accesses mutable global / static vars (to codegen `php_globals` variable) + // note, that assigning `has_global_vars_inside = !global_var_ids.empty()` is incorrect: + // for example if a function declares `global $v` but not uses it, or its uses are dropped off after cfg pass + current_function->has_global_vars_inside = true; + } return vertex; } if (vertex->rl_type == val_none) { diff --git a/compiler/pipes/calc-bad-vars.cpp b/compiler/pipes/calc-bad-vars.cpp index 7843108fc3..b539b17df4 100644 --- a/compiler/pipes/calc-bad-vars.cpp +++ b/compiler/pipes/calc-bad-vars.cpp @@ -588,7 +588,7 @@ class CalcBadVars { function->dep = std::move(call_graph.graph[function]); } - if (!G->settings().is_static_lib_mode()) { + if (!G->is_output_mode_lib()) { return; } diff --git a/compiler/pipes/calc-empty-functions.cpp b/compiler/pipes/calc-empty-functions.cpp index 461526a7d1..64314c9a0f 100644 --- a/compiler/pipes/calc-empty-functions.cpp +++ b/compiler/pipes/calc-empty-functions.cpp @@ -4,10 +4,9 @@ #include "compiler/pipes/calc-empty-functions.h" +#include "compiler/compiler-core.h" #include "compiler/data/function-data.h" #include "compiler/data/src-file.h" -#include "compiler/function-pass.h" -#include "compiler/vertex.h" namespace { @@ -49,6 +48,7 @@ FunctionData::body_value get_vertex_body_type(VertexPtr vertex) { case op_seq: return calc_seq_body_type(vertex.as()); case op_empty: + case op_global: return FunctionData::body_value::empty; default: return FunctionData::body_value::non_empty; @@ -89,5 +89,10 @@ FunctionData::body_value calc_function_body_type(FunctionPtr f) { void CalcEmptyFunctions::execute(FunctionPtr f, DataStream &os) { stage::set_function(f); f->body_seq = calc_function_body_type(f); + + if (f->is_main_function() && G->is_output_mode_lib() && G->get_main_file()->main_function == f) { + kphp_error(f->body_seq == FunctionData::body_value::empty, "main php file of a lib mustn't contain code, only declarations"); + } + os << f; } diff --git a/compiler/pipes/calc-locations.cpp b/compiler/pipes/calc-locations.cpp index e53828f384..89d34584c2 100644 --- a/compiler/pipes/calc-locations.cpp +++ b/compiler/pipes/calc-locations.cpp @@ -5,20 +5,24 @@ #include "compiler/pipes/calc-locations.h" #include "compiler/data/class-data.h" +#include "compiler/data/var-data.h" void CalcLocationsPass::on_start() { if (current_function->type == FunctionData::func_class_holder) { - current_function->class_id->members.for_each([](ClassMemberInstanceField &f) { - stage::set_line(f.root->location.line); - f.root->location = stage::get_location(); + current_function->class_id->members.for_each([this](ClassMemberInstanceField &f) { + f.root->location = Location{current_function->file_id, current_function, f.root->location.line}; + if (f.var->init_val) { + f.var->init_val.set_location_recursively(f.root->location); + } }); - current_function->class_id->members.for_each([](ClassMemberStaticField &f) { - stage::set_line(f.root->location.line); - f.root->location = stage::get_location(); + current_function->class_id->members.for_each([this](ClassMemberStaticField &f) { + f.root->location = Location{current_function->file_id, current_function, f.root->location.line}; + if (f.var->init_val) { + f.var->init_val.set_location_recursively(f.root->location); + } }); - current_function->class_id->members.for_each([](ClassMemberConstant &c) { - stage::set_line(c.value->location.line); - c.value.set_location(stage::get_location()); + current_function->class_id->members.for_each([this](ClassMemberConstant &c) { + c.value.set_location_recursively(Location{current_function->file_id, current_function, c.value->location.line}); }); } } diff --git a/compiler/pipes/code-gen.cpp b/compiler/pipes/code-gen.cpp index 254535afd1..500873ab66 100644 --- a/compiler/pipes/code-gen.cpp +++ b/compiler/pipes/code-gen.cpp @@ -4,26 +4,27 @@ #include "compiler/pipes/code-gen.h" -#include "compiler/cpp-dest-dir-initializer.h" #include "compiler/code-gen/code-gen-task.h" #include "compiler/code-gen/code-generator.h" #include "compiler/code-gen/common.h" +#include "compiler/code-gen/const-globals-batched-mem.h" #include "compiler/code-gen/declarations.h" #include "compiler/code-gen/files/cmake-lists-txt.h" +#include "compiler/code-gen/files/const-vars-init.h" #include "compiler/code-gen/files/function-header.h" #include "compiler/code-gen/files/function-source.h" -#include "compiler/code-gen/files/json-encoder-tags.h" -#include "compiler/code-gen/files/global_vars_memory_stats.h" +#include "compiler/code-gen/files/global-vars-memory-stats.h" +#include "compiler/code-gen/files/global-vars-reset.h" #include "compiler/code-gen/files/init-scripts.h" +#include "compiler/code-gen/files/json-encoder-tags.h" #include "compiler/code-gen/files/lib-header.h" -#include "compiler/code-gen/files/tl2cpp/tl2cpp.h" #include "compiler/code-gen/files/shape-keys.h" +#include "compiler/code-gen/files/tl2cpp/tl2cpp.h" #include "compiler/code-gen/files/tracing-autogen.h" #include "compiler/code-gen/files/type-tagger.h" -#include "compiler/code-gen/files/vars-cpp.h" -#include "compiler/code-gen/files/vars-reset.h" #include "compiler/code-gen/raw-data.h" #include "compiler/compiler-core.h" +#include "compiler/cpp-dest-dir-initializer.h" #include "compiler/data/class-data.h" #include "compiler/data/function-data.h" #include "compiler/data/generics-mixins.h" @@ -33,11 +34,6 @@ #include "compiler/pipes/collect-forkable-types.h" #include "compiler/type-hint.h" -size_t CodeGenF::calc_count_of_parts(size_t cnt_global_vars) { - return 1u + cnt_global_vars / G->settings().globals_split_count.get(); -} - - void CodeGenF::execute(FunctionPtr function, DataStream> &unused_os __attribute__ ((unused))) { if (function->does_need_codegen() || function->is_imported_from_static_lib()) { prepare_generate_function(function); @@ -60,6 +56,15 @@ void CodeGenF::on_finish(DataStream> &os) { const std::vector &all_classes = G->get_classes(); std::set all_json_encoders; + std::vector all_globals = G->get_global_vars(); + for (FunctionPtr f : all_functions) { + all_globals.insert(all_globals.end(), f->static_var_ids.begin(), f->static_var_ids.end()); + } + const GlobalsBatchedMem &all_globals_in_mem = GlobalsBatchedMem::prepare_mem_and_assign_offsets(all_globals); + + std::vector all_constants = G->get_constants_vars(); + const ConstantsBatchedMem &all_constants_in_mem = ConstantsBatchedMem::prepare_mem_and_assign_offsets(all_constants); + for (FunctionPtr f : all_functions) { code_gen_start_root_task(os, std::make_unique(f)); code_gen_start_root_task(os, std::make_unique(f)); @@ -90,36 +95,14 @@ void CodeGenF::on_finish(DataStream> &os) { } } - code_gen_start_root_task(os, std::make_unique(G->get_main_file())); - if (G->settings().enable_global_vars_memory_stats.get()) { - code_gen_start_root_task(os, std::make_unique(G->get_main_file())); + if (G->settings().enable_global_vars_memory_stats.get() && !G->is_output_mode_lib()) { + code_gen_start_root_task(os, std::make_unique(all_globals)); } code_gen_start_root_task(os, std::make_unique(G->get_main_file())); + code_gen_start_root_task(os, std::make_unique(all_constants_in_mem)); + code_gen_start_root_task(os, std::make_unique(all_globals_in_mem)); - std::vector vars = G->get_global_vars(); - for (FunctionPtr f : all_functions) { - vars.insert(vars.end(), f->static_var_ids.begin(), f->static_var_ids.end()); - } - size_t parts_cnt = calc_count_of_parts(vars.size()); - - std::vector> vars_batches(parts_cnt); - std::vector max_dep_levels(parts_cnt); - for (VarPtr var : vars) { - vars_batches[vk::std_hash(var->name) % parts_cnt].emplace_back(var); - } - for (size_t part_id = 0; part_id < parts_cnt; ++part_id) { - int max_dep_level{0}; - for (auto var : vars_batches[part_id]) { - if (var->is_constant() && max_dep_level < var->dependency_level) { - max_dep_level = var->dependency_level; - } - } - max_dep_levels[part_id] = max_dep_level; - code_gen_start_root_task(os, std::make_unique(std::move(vars_batches[part_id]), part_id)); - } - code_gen_start_root_task(os, std::make_unique(std::move(max_dep_levels), parts_cnt)); - - if (G->settings().is_static_lib_mode()) { + if (G->is_output_mode_lib()) { std::vector exported_functions; for (FunctionPtr f : all_functions) { if (f->kphp_lib_export) { @@ -133,7 +116,7 @@ void CodeGenF::on_finish(DataStream> &os) { } // TODO: should be done in lib mode also, but in some other way - if (!G->settings().is_static_lib_mode()) { + if (!G->is_output_mode_lib()) { code_gen_start_root_task(os, std::make_unique(vk::singleton::get().flush_forkable_types(), vk::singleton::get().flush_waitable_types())); code_gen_start_root_task(os, std::make_unique(TypeHintShape::get_all_registered_keys())); code_gen_start_root_task(os, std::make_unique(std::move(all_json_encoders))); @@ -146,7 +129,7 @@ void CodeGenF::on_finish(DataStream> &os) { code_gen_start_root_task(os, std::make_unique()); code_gen_start_root_task(os, std::make_unique()); - if (!G->settings().is_static_lib_mode()) { + if (!G->is_output_mode_lib()) { code_gen_start_root_task(os, std::make_unique()); } } @@ -185,9 +168,9 @@ void CodeGenF::prepare_generate_function(FunctionPtr func) { ? func->file_id->owner_lib->headers_dir() + func->header_name : func->subdir + "/" + func->header_name; - std::sort(func->static_var_ids.begin(), func->static_var_ids.end()); - std::sort(func->global_var_ids.begin(), func->global_var_ids.end()); - std::sort(func->local_var_ids.begin(), func->local_var_ids.end()); + std::sort(func->local_var_ids.begin(), func->local_var_ids.end(), [](VarPtr v1, VarPtr v2) { + return v1->name.compare(v2->name) < 0; + }); if (func->kphp_tracing) { TracingAutogen::register_function_marked_kphp_tracing(func); diff --git a/compiler/pipes/code-gen.h b/compiler/pipes/code-gen.h index 58aa425400..7b3f98fae8 100644 --- a/compiler/pipes/code-gen.h +++ b/compiler/pipes/code-gen.h @@ -19,7 +19,6 @@ class CodeGenF final : public SyncPipeF> &unused_os) final; diff --git a/compiler/pipes/collect-const-vars.cpp b/compiler/pipes/collect-const-vars.cpp index 001e734f0b..e0d774d625 100644 --- a/compiler/pipes/collect-const-vars.cpp +++ b/compiler/pipes/collect-const-vars.cpp @@ -7,6 +7,7 @@ #include "compiler/data/src-file.h" #include "compiler/vertex-util.h" #include "compiler/data/var-data.h" +#include "compiler/const-manipulations.h" #include "compiler/compiler-core.h" #include "compiler/name-gen.h" @@ -142,6 +143,34 @@ struct NameGenerator : public VertexVisitor { } return fallback(v); } + +private: + // checks that inlined as define' value constructor is suitable to be stored as constant var + static bool is_object_suitable_for_hashing(VertexPtr vertex) { + return vertex->type() == op_define_val && vertex.as()->value()->type() == op_func_call + && vertex.as()->value()->extra_type == op_ex_constructor_call && vertex->const_type == cnst_const_val; + } + + static bool is_array_suitable_for_hashing(VertexPtr vertex) { + return vertex->type() == op_array && CheckConst::is_const(vertex); + } + + static std::string gen_const_string_name(const std::string &str) { + return fmt_format("c_str${:x}", vk::std_hash(str)); + } + + static std::string gen_const_regexp_name(const std::string &str) { + return fmt_format("c_reg${:x}", vk::std_hash(str)); + } + + static std::string gen_const_object_name(const VertexAdaptor &def) { + kphp_assert_msg(def->value()->type() == op_func_call, "Internal error: expected op_define_val "); + return fmt_format("c_obj${:x}", ObjectHash::calc_hash(def)); + } + + static std::string gen_const_array_name(const VertexAdaptor &array) { + return fmt_format("c_arr${:x}", ArrayHash::calc_hash(array)); + } }; struct ProcessBeforeReplace : public VertexVisitor { @@ -203,7 +232,8 @@ void set_var_dep_level(VarPtr var_id) { VertexPtr CollectConstVarsPass::on_exit_vertex(VertexPtr root) { if (root->const_type == cnst_const_val) { composite_const_depth_ -= static_cast(IsComposite::visit(root)); - if (ShouldStoreOnBottomUp::visit(root)) { + if (ShouldStoreOnBottomUp::visit(root) + && !current_function->is_extern()) { // don't extract constants from extern func default arguments, they are in C++ runtime root = ProcessBeforeReplace::visit(root); root = create_const_variable(root, root->location); } @@ -242,7 +272,7 @@ VertexPtr CollectConstVarsPass::create_const_variable(VertexPtr root, Location l var->extra_type = op_ex_var_const; var->location = loc; - VarPtr var_id = G->get_global_var(name, VarData::var_const_t, VertexUtil::unwrap_inlined_define(root)); + VarPtr var_id = G->get_constant_var(name, VertexUtil::unwrap_inlined_define(root)); set_var_dep_level(var_id); if (composite_const_depth_ > 0) { diff --git a/compiler/pipes/convert-sprintf-calls.cpp b/compiler/pipes/convert-sprintf-calls.cpp index 6bad5fb214..06fe542ac9 100644 --- a/compiler/pipes/convert-sprintf-calls.cpp +++ b/compiler/pipes/convert-sprintf-calls.cpp @@ -140,7 +140,7 @@ VertexPtr ConvertSprintfCallsPass::convert_sprintf_call(VertexAdaptorlocation); vertex_parts.push_back(vertex); if (part.is_specifier()) { @@ -148,10 +148,10 @@ VertexPtr ConvertSprintfCallsPass::convert_sprintf_call(VertexAdaptor::create(vertex_parts); + return VertexAdaptor::create(vertex_parts).set_location(call->location); } -VertexPtr ConvertSprintfCallsPass::convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info) { +VertexPtr ConvertSprintfCallsPass::convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info, const Location &call_location) { if (part.is_specifier()) { VertexPtr element; @@ -180,7 +180,7 @@ VertexPtr ConvertSprintfCallsPass::convert_format_part_to_vertex(const FormatPar return VertexAdaptor::create(convert); } - VertexAdaptor vertex = VertexAdaptor::create(); + VertexAdaptor vertex = VertexAdaptor::create().set_location(call_location); vertex->set_string(part.value); return vertex; } diff --git a/compiler/pipes/convert-sprintf-calls.h b/compiler/pipes/convert-sprintf-calls.h index 55d711bf55..5f14308630 100644 --- a/compiler/pipes/convert-sprintf-calls.h +++ b/compiler/pipes/convert-sprintf-calls.h @@ -37,5 +37,5 @@ class ConvertSprintfCallsPass final : public FunctionPassBase { private: static VertexPtr convert_sprintf_call(VertexAdaptor call); - static VertexPtr convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info); + static VertexPtr convert_format_part_to_vertex(const FormatPart &part, size_t arg_index, const FormatCallInfo &info, const Location &call_location); }; diff --git a/compiler/pipes/final-check.cpp b/compiler/pipes/final-check.cpp index fe8ec17f97..7593c94b0b 100644 --- a/compiler/pipes/final-check.cpp +++ b/compiler/pipes/final-check.cpp @@ -14,7 +14,6 @@ #include "compiler/data/kphp-tracing-tags.h" #include "compiler/data/src-file.h" #include "compiler/data/var-data.h" -#include "compiler/data/vars-collector.h" #include "compiler/vertex-util.h" #include "compiler/type-hint.h" @@ -324,23 +323,12 @@ void check_register_shutdown_functions(VertexAdaptor call) { vk::join(throws, ", "), callback->func_id->get_throws_call_chain())); } -void mark_global_vars_for_memory_stats() { - if (!G->settings().enable_global_vars_memory_stats.get()) { - return; - } - - static std::atomic vars_marked{false}; - if (vars_marked.exchange(true)) { - return; - } - +void mark_global_vars_for_memory_stats(const std::vector &vars_list) { std::unordered_set classes_inside; - VarsCollector vars_collector{0, [&classes_inside](VarPtr variable) { - tinf::get_type(variable)->get_all_class_types_inside(classes_inside); - return false; - }}; - vars_collector.collect_global_and_static_vars_from(G->get_main_file()->main_function); - for (auto klass: classes_inside) { + for (VarPtr var : vars_list) { + tinf::get_type(var)->get_all_class_types_inside(classes_inside); + } + for (ClassPtr klass: classes_inside) { klass->deeply_require_instance_memory_estimate_visitor(); } } @@ -559,7 +547,15 @@ void check_php2c_conv(VertexAdaptor conv) { } // namespace void FinalCheckPass::on_start() { - mark_global_vars_for_memory_stats(); + if (G->settings().enable_global_vars_memory_stats.get()) { + static std::atomic globals_marked{false}; + if (!globals_marked.exchange(true)) { + mark_global_vars_for_memory_stats(G->get_global_vars()); + } + if (!current_function->static_var_ids.empty()) { + mark_global_vars_for_memory_stats(current_function->static_var_ids); + } + } if (current_function->type == FunctionData::func_class_holder) { check_class_immutableness(current_function->class_id); diff --git a/compiler/pipes/gen-tree-postprocess.cpp b/compiler/pipes/gen-tree-postprocess.cpp index ddd9b2a94b..fd1f1c81b4 100644 --- a/compiler/pipes/gen-tree-postprocess.cpp +++ b/compiler/pipes/gen-tree-postprocess.cpp @@ -90,7 +90,7 @@ VertexAdaptor make_require_once_call(SrcFilePtr lib_main_file, Verte } VertexPtr process_require_lib(VertexAdaptor require_lib_call) { - kphp_error_act (!G->settings().is_static_lib_mode(), "require_lib is forbidden to use for compiling libs", return require_lib_call); + kphp_error_act (!G->is_output_mode_lib(), "require_lib is forbidden to use for compiling libs", return require_lib_call); VertexRange args = require_lib_call->args(); kphp_error_act (args.size() == 1, fmt_format("require_lib expected 1 arguments, got {}", args.size()), return require_lib_call); auto lib_name_node = args[0]; @@ -230,9 +230,9 @@ VertexPtr GenTreePostprocessPass::on_enter_vertex(VertexPtr root) { } VertexPtr GenTreePostprocessPass::on_exit_vertex(VertexPtr root) { - if (root->type() == op_var) { - if (VertexUtil::is_superglobal(root->get_string())) { - root->extra_type = op_ex_var_superglobal; + if (auto as_var = root.try_as()) { + if (as_var->str_val[0] == '_' && VarData::does_name_eq_any_language_superglobal(as_var->str_val)) { + as_var->extra_type = op_ex_var_superglobal; } } diff --git a/compiler/pipes/generate-virtual-methods.cpp b/compiler/pipes/generate-virtual-methods.cpp index 32053a863d..681b564473 100644 --- a/compiler/pipes/generate-virtual-methods.cpp +++ b/compiler/pipes/generate-virtual-methods.cpp @@ -423,7 +423,7 @@ void generate_body_of_virtual_method(FunctionPtr virtual_function) { } } if (!cases.empty()) { - auto case_default_warn = generate_critical_error_call(fmt_format("call method({}) on null object", virtual_function->as_human_readable())); + auto case_default_warn = generate_critical_error_call(fmt_format("call method({}) on null object", virtual_function->as_human_readable(false))); cases.emplace_back(VertexAdaptor::create(VertexAdaptor::create(case_default_warn))); } diff --git a/compiler/pipes/optimization.cpp b/compiler/pipes/optimization.cpp index c7ef4756d5..5fb9503ae4 100644 --- a/compiler/pipes/optimization.cpp +++ b/compiler/pipes/optimization.cpp @@ -45,7 +45,7 @@ VarPtr cast_const_array_type(VertexPtr &type_acceptor, const TypeData *required_ ss << type_acceptor->get_string() << "$" << std::hex << vk::std_hash(type_out(required_type)); std::string name = ss.str(); bool is_new = true; - VarPtr var_id = G->get_global_var(name, VarData::var_const_t, type_acceptor, &is_new); + VarPtr var_id = G->get_constant_var(name, type_acceptor, &is_new); var_id->tinf_node.copy_type_from(required_type); // not inside if(is_new) to avoid race conditions when one thread creates and another uses faster if (is_new) { var_id->dependency_level = type_acceptor.as()->var_id->dependency_level + 1; diff --git a/compiler/pipes/register-variables.cpp b/compiler/pipes/register-variables.cpp index cd3d153ce6..30b8f80290 100644 --- a/compiler/pipes/register-variables.cpp +++ b/compiler/pipes/register-variables.cpp @@ -12,7 +12,7 @@ #include "compiler/utils/string-utils.h" VarPtr RegisterVariablesPass::create_global_var(const std::string &name) { - VarPtr var = G->get_global_var(name, VarData::var_global_t, VertexPtr()); + VarPtr var = G->get_global_var(name, VertexPtr()); auto it = registred_vars.insert(make_pair(name, var)); if (it.second == false) { VarPtr old_var = it.first->second; diff --git a/compiler/pipes/remove-empty-function-calls.cpp b/compiler/pipes/remove-empty-function-calls.cpp index cdefe84b0d..6595ed5d0f 100644 --- a/compiler/pipes/remove-empty-function-calls.cpp +++ b/compiler/pipes/remove-empty-function-calls.cpp @@ -46,7 +46,7 @@ VertexPtr RemoveEmptyFunctionCallsPass::on_exit_vertex(VertexPtr v) { // get rid of $called - global variables for empty source files; // namely, detect 'v$src_fooxxx$called = true' assign in such files and remove it, // this allows to avoid further call of register_var() with such global variable - if (!G->settings().is_static_lib_mode() && current_function->is_main_function() && current_function->body_seq == FunctionData::body_value::empty) { + if (!G->is_output_mode_lib() && current_function->is_main_function() && current_function->body_seq == FunctionData::body_value::empty) { auto set = v.as(); auto lhs = set->lhs(); auto rhs = set->rhs(); diff --git a/compiler/threading/hash-table.h b/compiler/threading/hash-table.h index 4fa9189d4b..010be0efe9 100644 --- a/compiler/threading/hash-table.h +++ b/compiler/threading/hash-table.h @@ -64,6 +64,7 @@ class TSHashTable { std::vector get_all() { std::vector res; + res.reserve(used_size); for (int i = 0; i < N; i++) { if (nodes[i].hash != 0) { res.push_back(nodes[i].data); diff --git a/compiler/vertex-util.cpp b/compiler/vertex-util.cpp index 7f6d1606c8..56a01608da 100644 --- a/compiler/vertex-util.cpp +++ b/compiler/vertex-util.cpp @@ -116,19 +116,6 @@ void VertexUtil::func_force_return(VertexAdaptor func, VertexPtr va func->cmd_ref() = VertexAdaptor::create(next); } -bool VertexUtil::is_superglobal(const std::string &s) { - static std::set names = { - "_SERVER", - "_GET", - "_POST", - "_FILES", - "_COOKIE", - "_REQUEST", - "_ENV" - }; - return vk::contains(names, s); -} - bool VertexUtil::is_positive_constexpr_int(VertexPtr v) { auto actual_value = get_actual_value(v).try_as(); return actual_value && parse_int_from_string(actual_value) >= 0; diff --git a/compiler/vertex-util.h b/compiler/vertex-util.h index 24bedc2a5f..c118e7a3a9 100644 --- a/compiler/vertex-util.h +++ b/compiler/vertex-util.h @@ -36,7 +36,6 @@ class VertexUtil { static void func_force_return(VertexAdaptor func, VertexPtr val = {}); - static bool is_superglobal(const std::string &s); static bool is_positive_constexpr_int(VertexPtr v); static bool is_const_int(VertexPtr root); }; diff --git a/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md b/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md index 1fde19e6df..b67550a870 100644 --- a/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md +++ b/docs/kphp-language/kphp-vs-php/compiler-cmd-options.md @@ -75,10 +75,6 @@ Threads number for PHP → C++ codegeneration, default **CPU cores * 2**. Processes number to C++ parallel compilation/linkage, default **CPU cores**. - - -All global variables (const arrays also) are split into chunks of this size, default **1024**. If you have a few but very heavy global vars, lowering this number can decrease compilation time. - A *.tl* file with [TL schema](../../kphp-client/tl-schema-and-rpc/tl-schema-basics.md), default empty. diff --git a/runtime/array_functions.cpp b/runtime/array_functions.cpp index 1b526e1f42..1589dcb5c6 100644 --- a/runtime/array_functions.cpp +++ b/runtime/array_functions.cpp @@ -257,3 +257,6 @@ string implode_string_vector(const string &s, const array &a) { } return result.finish_append(); } + +static_assert(sizeof(array) == SIZEOF_ARRAY_ANY, "sizeof(array) at runtime doesn't match compile-time"); + diff --git a/runtime/interface.cpp b/runtime/interface.cpp index 16c327d864..be036860e7 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -595,7 +595,7 @@ void f$fastcgi_finish_request(int64_t exit_code) { write_safe(1, oub[ob_total_buffer].buffer(), oub[ob_total_buffer].size(), {}); //TODO move to finish_script - free_runtime_environment(); + free_runtime_environment(PhpScriptMutableGlobals::current().get_superglobals()); break; } @@ -866,8 +866,6 @@ bool f$get_magic_quotes_gpc() { return false; } -string v$d$PHP_SAPI __attribute__ ((weak)); - static string php_sapi_name() { switch (query_type) { @@ -890,21 +888,10 @@ static string php_sapi_name() { } string f$php_sapi_name() { - return v$d$PHP_SAPI; + return PhpScriptMutableGlobals::current().get_superglobals().v$d$PHP_SAPI; } -mixed v$_SERVER __attribute__ ((weak)); -mixed v$_GET __attribute__ ((weak)); -mixed v$_POST __attribute__ ((weak)); -mixed v$_FILES __attribute__ ((weak)); -mixed v$_COOKIE __attribute__ ((weak)); -mixed v$_REQUEST __attribute__ ((weak)); -mixed v$_ENV __attribute__ ((weak)); - -mixed v$argc __attribute__ ((weak)); -mixed v$argv __attribute__ ((weak)); - static std::aligned_storage_t), alignof(array)> uploaded_files_storage; static array *uploaded_files = reinterpret_cast *> (&uploaded_files_storage); static long long uploaded_files_last_query_num = -1; @@ -1176,7 +1163,7 @@ class post_reader { } }; -static int parse_multipart_one(post_reader &data, int i) { +static int parse_multipart_one(post_reader &data, int i, mixed &v$_POST, mixed &v$_FILES) { string content_type("text/plain", 10); string name; string filename; @@ -1352,7 +1339,7 @@ static int parse_multipart_one(post_reader &data, int i) { return i; } -static bool parse_multipart(const char *post, int post_len, const string &boundary) { +static bool parse_multipart(const char *post, int post_len, const string &boundary, mixed &v$_POST, mixed &v$_FILES) { static const int MAX_BOUNDARY_LENGTH = 70; if (boundary.empty() || (int)boundary.size() > MAX_BOUNDARY_LENGTH) { @@ -1364,7 +1351,7 @@ static bool parse_multipart(const char *post, int post_len, const string &bounda for (int i = 0; i < post_len; i++) { // fprintf (stderr, "!!!! %d\n", i); - i = parse_multipart_one(data, i); + i = parse_multipart_one(data, i, v$_POST, v$_FILES); // fprintf (stderr, "???? %d\n", i); while (!data.is_boundary(i)) { @@ -1461,16 +1448,16 @@ void arg_add(const char *value) { arg_vars->push_back(string(value)); } -static void reset_superglobals() { +static void reset_superglobals(PhpScriptBuiltInSuperGlobals &superglobals) { dl::enter_critical_section(); - hard_reset_var(v$_SERVER, array()); - hard_reset_var(v$_GET, array()); - hard_reset_var(v$_POST, array()); - hard_reset_var(v$_FILES, array()); - hard_reset_var(v$_COOKIE, array()); - hard_reset_var(v$_REQUEST, array()); - hard_reset_var(v$_ENV, array()); + hard_reset_var(superglobals.v$_SERVER, array()); + hard_reset_var(superglobals.v$_GET, array()); + hard_reset_var(superglobals.v$_POST, array()); + hard_reset_var(superglobals.v$_ENV, array()); + hard_reset_var(superglobals.v$_FILES, array()); + hard_reset_var(superglobals.v$_COOKIE, array()); + hard_reset_var(superglobals.v$_REQUEST, array()); dl::leave_critical_section(); } @@ -1478,7 +1465,7 @@ static void reset_superglobals() { // RFC link: https://tools.ietf.org/html/rfc2617#section-2 // Header example: // Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ== -static void parse_http_authorization_header(const string &header_value) { +static void parse_http_authorization_header(const string &header_value, mixed &v$_SERVER) { array header_parts = explode(' ', header_value); if (header_parts.count() != 2) { return; @@ -1501,7 +1488,7 @@ static void parse_http_authorization_header(const string &header_value) { v$_SERVER.set_value(string("AUTH_TYPE"), auth_scheme); } -static void save_rpc_query_headers(const tl_query_header_t &header) { +static void save_rpc_query_headers(const tl_query_header_t &header, mixed &v$_SERVER) { namespace flag = vk::tl::common::rpc_invoke_req_extra_flags; if (header.actor_id) { @@ -1546,37 +1533,37 @@ static void save_rpc_query_headers(const tl_query_header_t &header) { } } -static void init_superglobals_impl(const http_query_data &http_data, const rpc_query_data &rpc_data, const job_query_data &job_data) { +static void init_superglobals_impl(const http_query_data &http_data, const rpc_query_data &rpc_data, const job_query_data &job_data, PhpScriptBuiltInSuperGlobals &superglobals) { rpc_parse(rpc_data.data.data(), rpc_data.data.size()); - reset_superglobals(); + reset_superglobals(superglobals); if (query_type == QUERY_TYPE_JOB) { - v$_SERVER.set_value(string("JOB_ID"), job_data.job_request->job_id); + superglobals.v$_SERVER.set_value(string("JOB_ID"), job_data.job_request->job_id); init_job_server_interface_lib(job_data); } string uri_str; if (http_data.uri_len) { uri_str.assign(http_data.uri, http_data.uri_len); - v$_SERVER.set_value(string("PHP_SELF"), uri_str); - v$_SERVER.set_value(string("SCRIPT_URL"), uri_str); - v$_SERVER.set_value(string("SCRIPT_NAME"), uri_str); + superglobals.v$_SERVER.set_value(string("PHP_SELF"), uri_str); + superglobals.v$_SERVER.set_value(string("SCRIPT_URL"), uri_str); + superglobals.v$_SERVER.set_value(string("SCRIPT_NAME"), uri_str); } string get_str; if (http_data.get_len) { get_str.assign(http_data.get, http_data.get_len); - f$parse_str(get_str, v$_GET); + f$parse_str(get_str, superglobals.v$_GET); - v$_SERVER.set_value(string("QUERY_STRING"), get_str); + superglobals.v$_SERVER.set_value(string("QUERY_STRING"), get_str); } if (http_data.uri) { if (http_data.get_len) { - v$_SERVER.set_value(string("REQUEST_URI"), (static_SB.clean() << uri_str << '?' << get_str).str()); + superglobals.v$_SERVER.set_value(string("REQUEST_URI"), (static_SB.clean() << uri_str << '?' << get_str).str()); } else { - v$_SERVER.set_value(string("REQUEST_URI"), uri_str); + superglobals.v$_SERVER.set_value(string("REQUEST_URI"), uri_str); } } @@ -1624,13 +1611,13 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q for (int t = 0; t < (int)cookie.count(); t++) { array cur_cookie = explode('=', f$trim(cookie[t]), 2); if ((int)cur_cookie.count() == 2) { - parse_str_set_value(v$_COOKIE, cur_cookie[0], f$urldecode(cur_cookie[1])); + parse_str_set_value(superglobals.v$_COOKIE, cur_cookie[0], f$urldecode(cur_cookie[1])); } } } else if (!strcmp(header_name.c_str(), "host")) { - v$_SERVER.set_value(string("SERVER_NAME"), header_value); + superglobals.v$_SERVER.set_value(string("SERVER_NAME"), header_value); } else if (!strcmp(header_name.c_str(), "authorization")) { - parse_http_authorization_header(header_value); + parse_http_authorization_header(header_value, superglobals.v$_SERVER); } if (!strcmp(header_name.c_str(), "content-type")) { @@ -1659,7 +1646,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q key[2] = 'T'; key[3] = 'P'; key[4] = '_'; - v$_SERVER.set_value(key, header_value); + superglobals.v$_SERVER.set_value(key, header_value); } else { // fprintf (stderr, "%s : %s\n", header_name.c_str(), header_value.c_str()); } @@ -1670,12 +1657,12 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q string HTTP_X_REAL_SCHEME("HTTP_X_REAL_SCHEME", 18); string HTTP_X_REAL_HOST("HTTP_X_REAL_HOST", 16); string HTTP_X_REAL_REQUEST("HTTP_X_REAL_REQUEST", 19); - if (v$_SERVER.isset(HTTP_X_REAL_SCHEME) && v$_SERVER.isset(HTTP_X_REAL_HOST) && v$_SERVER.isset(HTTP_X_REAL_REQUEST)) { - string script_uri(v$_SERVER.get_value(HTTP_X_REAL_SCHEME).to_string()); + if (superglobals.v$_SERVER.isset(HTTP_X_REAL_SCHEME) && superglobals.v$_SERVER.isset(HTTP_X_REAL_HOST) && superglobals.v$_SERVER.isset(HTTP_X_REAL_REQUEST)) { + string script_uri(superglobals.v$_SERVER.get_value(HTTP_X_REAL_SCHEME).to_string()); script_uri.append("://", 3); - script_uri.append(v$_SERVER.get_value(HTTP_X_REAL_HOST).to_string()); - script_uri.append(v$_SERVER.get_value(HTTP_X_REAL_REQUEST).to_string()); - v$_SERVER.set_value(string("SCRIPT_URI"), script_uri); + script_uri.append(superglobals.v$_SERVER.get_value(HTTP_X_REAL_HOST).to_string()); + script_uri.append(superglobals.v$_SERVER.get_value(HTTP_X_REAL_REQUEST).to_string()); + superglobals.v$_SERVER.set_value(string("SCRIPT_URI"), script_uri); } if (http_data.post_len > 0) { @@ -1687,7 +1674,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q raw_post_data.assign(http_data.post, http_data.post_len); dl::leave_critical_section(); - f$parse_str(raw_post_data, v$_POST); + f$parse_str(raw_post_data, superglobals.v$_POST); } } else if (strstr(content_type_lower.c_str(), "multipart/form-data")) { const char *p = strstr(content_type_lower.c_str(), "boundary"); @@ -1703,7 +1690,7 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q end_p--; } // fprintf (stderr, "!%s!\n", p); - is_parsed |= parse_multipart(http_data.post, http_data.post_len, string(p, static_cast(end_p - p))); + is_parsed |= parse_multipart(http_data.post, http_data.post_len, string(p, static_cast(end_p - p)), superglobals.v$_POST, superglobals.v$_FILES); } } } else { @@ -1723,51 +1710,51 @@ static void init_superglobals_impl(const http_query_data &http_data, const rpc_q } } - v$_SERVER.set_value(string("CONTENT_TYPE"), content_type); + superglobals.v$_SERVER.set_value(string("CONTENT_TYPE"), content_type); } double cur_time = microtime(); - v$_SERVER.set_value(string("GATEWAY_INTERFACE"), string("CGI/1.1")); + superglobals.v$_SERVER.set_value(string("GATEWAY_INTERFACE"), string("CGI/1.1")); if (http_data.ip) { - v$_SERVER.set_value(string("REMOTE_ADDR"), f$long2ip(static_cast(http_data.ip))); + superglobals.v$_SERVER.set_value(string("REMOTE_ADDR"), f$long2ip(static_cast(http_data.ip))); } if (http_data.port) { - v$_SERVER.set_value(string("REMOTE_PORT"), static_cast(http_data.port)); + superglobals.v$_SERVER.set_value(string("REMOTE_PORT"), static_cast(http_data.port)); } if (rpc_data.header.qid) { - v$_SERVER.set_value(string("RPC_REQUEST_ID"), f$strval(static_cast(rpc_data.header.qid))); - save_rpc_query_headers(rpc_data.header); - v$_SERVER.set_value(string("RPC_REMOTE_IP"), static_cast(rpc_data.remote_pid.ip)); - v$_SERVER.set_value(string("RPC_REMOTE_PORT"), static_cast(rpc_data.remote_pid.port)); - v$_SERVER.set_value(string("RPC_REMOTE_PID"), static_cast(rpc_data.remote_pid.pid)); - v$_SERVER.set_value(string("RPC_REMOTE_UTIME"), rpc_data.remote_pid.utime); + superglobals.v$_SERVER.set_value(string("RPC_REQUEST_ID"), f$strval(static_cast(rpc_data.header.qid))); + save_rpc_query_headers(rpc_data.header, superglobals.v$_SERVER); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_IP"), static_cast(rpc_data.remote_pid.ip)); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_PORT"), static_cast(rpc_data.remote_pid.port)); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_PID"), static_cast(rpc_data.remote_pid.pid)); + superglobals.v$_SERVER.set_value(string("RPC_REMOTE_UTIME"), rpc_data.remote_pid.utime); } is_head_query = false; if (http_data.request_method_len) { - v$_SERVER.set_value(string("REQUEST_METHOD"), string(http_data.request_method, http_data.request_method_len)); + superglobals.v$_SERVER.set_value(string("REQUEST_METHOD"), string(http_data.request_method, http_data.request_method_len)); if (http_data.request_method_len == 4 && !strncmp(http_data.request_method, "HEAD", http_data.request_method_len)) { is_head_query = true; } } - v$_SERVER.set_value(string("REQUEST_TIME"), int(cur_time)); - v$_SERVER.set_value(string("REQUEST_TIME_FLOAT"), cur_time); - v$_SERVER.set_value(string("SERVER_PORT"), string("80")); - v$_SERVER.set_value(string("SERVER_PROTOCOL"), string("HTTP/1.1")); - v$_SERVER.set_value(string("SERVER_SIGNATURE"), (static_SB.clean() << "Apache/2.2.9 (Debian) PHP/5.2.6-1<second; +} diff --git a/runtime/php-script-globals.h b/runtime/php-script-globals.h new file mode 100644 index 0000000000..028a165c56 --- /dev/null +++ b/runtime/php-script-globals.h @@ -0,0 +1,49 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime/kphp_core.h" + +struct PhpScriptBuiltInSuperGlobals { + // variables below are PHP language superglobals + mixed v$_SERVER; + mixed v$_GET; + mixed v$_POST; + mixed v$_ENV; + mixed v$_FILES; + mixed v$_COOKIE; + mixed v$_REQUEST; + + // variables below are not superglobals of the PHP language, but since they are set by runtime, + // the compiler is also aware about them + mixed v$argc; + mixed v$argv; + string v$d$PHP_SAPI; // define('PHP_SAPI') +}; + +// storage of linear memory used for mutable globals in each script +// on worker start, once_alloc_linear_mem() is called from codegen +// it initializes g_linear_mem, and every mutable global access is codegenerated +// as smth line `(*reinterpret_cast(&php_globals.mem()+offset))` +class PhpScriptMutableGlobals { + char *g_linear_mem{nullptr}; + std::unordered_map libs_linear_mem; + PhpScriptBuiltInSuperGlobals superglobals; + +public: + static PhpScriptMutableGlobals ¤t(); + ~PhpScriptMutableGlobals(); + + void once_alloc_linear_mem(unsigned int n_bytes); + void once_alloc_linear_mem(const char *lib_name, unsigned int n_bytes); + + char *mem() const { return g_linear_mem; } + char *mem_for_lib(const char *lib_name) const; + + PhpScriptBuiltInSuperGlobals &get_superglobals() { return superglobals; } + const PhpScriptBuiltInSuperGlobals &get_superglobals() const { return superglobals; } +}; diff --git a/runtime/regexp.cpp b/runtime/regexp.cpp index 21ba455b56..16669bf205 100644 --- a/runtime/regexp.cpp +++ b/runtime/regexp.cpp @@ -33,6 +33,7 @@ static re2::StringPiece RE2_submatch[MAX_SUBPATTERNS]; int32_t regexp::submatch[3 * MAX_SUBPATTERNS]; pcre_extra regexp::extra; +static_assert(sizeof(regexp) == SIZEOF_REGEXP, "sizeof(regexp) at runtime doesn't match compile-time"); regexp::regexp(const string ®exp_string) { init(regexp_string); diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index edd072a119..62215b7066 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -96,6 +96,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ math_functions.cpp mbstring.cpp memcache.cpp + memory_usage.cpp migration_php8.cpp misc.cpp mixed.cpp @@ -105,6 +106,7 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/ oom_handler.cpp openssl.cpp php_assert.cpp + php-script-globals.cpp profiler.cpp regexp.cpp resumable.cpp diff --git a/runtime/string.cpp b/runtime/string.cpp index 13b570e14f..c405722055 100644 --- a/runtime/string.cpp +++ b/runtime/string.cpp @@ -8,3 +8,5 @@ string::~string() noexcept { destroy(); } + +static_assert(sizeof(string) == SIZEOF_STRING, "sizeof(string) at runtime doesn't match compile-time"); diff --git a/server/php-engine.cpp b/server/php-engine.cpp index 06f8273e90..3c2a1da794 100644 --- a/server/php-engine.cpp +++ b/server/php-engine.cpp @@ -1669,14 +1669,13 @@ void init_all() { StatsHouseManager::get().set_common_tags(); global_init_runtime_libs(); - global_init_php_scripts(); + init_php_scripts_once_in_master(); global_init_script_allocator(); init_handlers(); init_drivers(); - init_php_scripts(); vk::singleton::get().set_idle_worker_status(); worker_id = (int)lrand48(); diff --git a/server/php-init-scripts.cpp b/server/php-init-scripts.cpp index ed9e0f5af4..a275743cce 100644 --- a/server/php-init-scripts.cpp +++ b/server/php-init-scripts.cpp @@ -10,6 +10,6 @@ script_t *get_script() { return main_script; } -void set_script(void (*run)(), void (*clear)()) { +void set_script(void (*run)(), void (*clear)(PhpScriptMutableGlobals &php_globals)) { main_script = new script_t{run, clear}; } diff --git a/server/php-init-scripts.h b/server/php-init-scripts.h index 0e7b51f6b5..2c2193afde 100644 --- a/server/php-init-scripts.h +++ b/server/php-init-scripts.h @@ -6,19 +6,21 @@ #include +class PhpScriptMutableGlobals; + struct script_t { void (*run)(); // this is entrypoint to generated code - void (*clear)(); + void (*clear)(PhpScriptMutableGlobals &php_globals); }; -/// It binds generated code and runtime. Definition is generated by compiler. This function is called at global initialization @see init_all(). -void init_php_scripts() noexcept; -/// It initializes const variables. Definition is generated by compiler. This function is called at global initialization @see init_all(). -void global_init_php_scripts() noexcept; +/// Initializes const variables represented as globals C++ symbols. Definition is generated by compiler. +void init_php_scripts_once_in_master() noexcept; +/// Initializes mutable globals in a single linear memory piece. +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals &php_globals) noexcept; script_t *get_script(); /// It's called from init_php_scripts() and set the entrypoint to generated code -void set_script(void (*run)(), void (*clear)()); +void set_script(void (*run)(), void (*clear)(PhpScriptMutableGlobals &php_globals)); struct script_result { const char *headers; diff --git a/server/php-runner.cpp b/server/php-runner.cpp index 45ce1805c1..24276fd132 100644 --- a/server/php-runner.cpp +++ b/server/php-runner.cpp @@ -359,8 +359,8 @@ void PhpScript::finish() noexcept { void PhpScript::clear() noexcept { assert_state(run_state_t::uncleared); - run_main->clear(); - free_runtime_environment(); + run_main->clear(PhpScriptMutableGlobals::current()); + free_runtime_environment(PhpScriptMutableGlobals::current().get_superglobals()); state = run_state_t::empty; if (use_madvise_dontneed) { if (dl::get_script_memory_stats().real_memory_used > memory_used_to_recreate_script) { @@ -413,7 +413,7 @@ void PhpScript::run() noexcept { in_script_context = true; auto oom_handling_memory_size = static_cast(std::ceil(mem_size * oom_handling_memory_ratio)); auto script_memory_size = mem_size - oom_handling_memory_size; - init_runtime_environment(*data, run_mem, script_memory_size, oom_handling_memory_size); + init_runtime_environment(*data, PhpScriptMutableGlobals::current().get_superglobals(), run_mem, script_memory_size, oom_handling_memory_size); dl::leave_critical_section(); php_assert (dl::in_critical_section == 0); // To ensure that no critical section is left at the end of the initialization check_net_context_errors(); diff --git a/tests/cpp/runtime/_runtime-tests-env.cpp b/tests/cpp/runtime/_runtime-tests-env.cpp index 15228c518b..2bfbc6d37c 100644 --- a/tests/cpp/runtime/_runtime-tests-env.cpp +++ b/tests/cpp/runtime/_runtime-tests-env.cpp @@ -30,7 +30,7 @@ class RuntimeTestsEnvironment final : public testing::Environment { global_init_runtime_libs(); global_init_script_allocator(); - init_runtime_environment(null_query_data{}, script_memory, script_memory_size); + init_runtime_environment(null_query_data{}, PhpScriptMutableGlobals::current().get_superglobals(), script_memory, script_memory_size); php_disable_warnings = true; php_warning_level = 0; } @@ -38,7 +38,7 @@ class RuntimeTestsEnvironment final : public testing::Environment { void TearDown() final { reset_global_vars(); - free_runtime_environment(); + free_runtime_environment(PhpScriptMutableGlobals::current().get_superglobals()); testing::Environment::TearDown(); } @@ -66,10 +66,11 @@ template<> int Storage::tagger>>::get_ template<> int Storage::tagger>::get_tag() noexcept { return 0; } template<> Storage::loader::loader_fun Storage::loader::get_function(int) noexcept { return nullptr; } -void init_php_scripts() noexcept { +void init_php_scripts_once_in_master() noexcept { assert(0 && "this code shouldn't be executed and only for linkage test"); } -void global_init_php_scripts() noexcept { +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals &php_globals) noexcept { + static_cast(php_globals); assert(0 && "this code shouldn't be executed and only for linkage test"); } const char *get_php_scripts_version() noexcept { diff --git a/tests/phpt/constants/010_arrow_access.php b/tests/phpt/constants/010_arrow_access.php index f2c217e8d6..b4d2adfc30 100644 --- a/tests/phpt/constants/010_arrow_access.php +++ b/tests/phpt/constants/010_arrow_access.php @@ -3,10 +3,9 @@ require_once 'kphp_tester_include.php'; class B { - static int $static_int = 0; public int $value; public function __construct(int $x) { - B::$static_int += 1; + // B::$static_int += 1; // accessing globals in constructor in const classes crashes const_init; todo how to detect it in the future? $this->value = $x; } public function getValue() { diff --git a/tests/phpt/dl/1043_some_globals.php b/tests/phpt/dl/1043_some_globals.php new file mode 100644 index 0000000000..731b3629d1 --- /dev/null +++ b/tests/phpt/dl/1043_some_globals.php @@ -0,0 +1,291 @@ +@ok + [], + 'f' => [], + ]; +} + +$targets = getEmptyArrayOfArraysOfClass(); +echo 'count empty arr ', count($targets), "\n"; + +define('NOW', time()); +if (0) { + echo NOW, "\n"; +} + +// const_var +$s_concat = 'asdf' . '3'; +echo $s_concat, "\n"; +// const_var +$a_tuples = [tuple(1, true)]; +echo count($a_tuples); +// const_var float +$float_sin_30 = sin(30); +echo $float_sin_30, "\n"; +// const_var optional +$float_optional_min = min(1.0, null); +echo $float_optional_min, "\n"; +// const_var bool +$g_assigned_const_bool = min(true, false); +var_dump($g_assigned_const_bool); + +function acceptsVariadict(...$args) { + var_dump($args); +} +acceptsVariadict(''. 'str converted to array(1) into variadic'); + +/** @param mixed[] $arr */ +function acceptArrayMixed($arr) { + static $inAcceptArrayMixed = [0]; + + echo count($arr); + preg_match('/asdf+/', $arr[0]); +} + +acceptArrayMixed([1,2,3]); +acceptArrayMixed([4, 5, 6]); + +/** @var tuple(int, tuple(string|false, int|null)) */ +$g_tuple = tuple(1, tuple('str', 10)); +echo $g_tuple[1][0], "\n"; +$g_tuple = tuple(1, tuple(false, null)); +var_dump($g_tuple[1][1]); echo "\n"; + +/** @var tuple(bool) */ +$g_tuple_2 = tuple(true); +var_dump($g_tuple_2[0]); +/** @var tuple(bool,bool) */ +$g_tuple_3 = tuple(true,true); +var_dump($g_tuple_3[1]); +/** @var tuple(bool,int,bool) */ +$g_tuple_4 = tuple(true,0,true); +var_dump($g_tuple_4[2]); +/** @var tuple(bool,bool,bool,bool,bool,bool,bool,bool,bool) */ +$g_tuple_5 = tuple(true,true,true,true,true,true,true,false,true); +var_dump($g_tuple_5[7]); +var_dump($g_tuple_5[8]); + +/** @var shape(x:?int, y:SurveyTarget) */ +$g_shape = shape(['y' => new SurveyTarget]); +echo get_class($g_shape['y']), "\n"; + +function getInt(): int { return 5; } + +/** @var future */ +$g_future = fork(getInt()); +$g_future_result = wait($g_future); + +/** @var ?future_queue */ +$g_future_queue = wait_queue_create([$g_future]); + +function defArgConstants($s1 = 'str', $ar = [1,2,3]) { + $a = [[[[1]]]][0]; + echo $s1, count($ar), "\n"; +} +defArgConstants(); +defArgConstants('str2', [1,2]); + +class WithGlobals { + static public $unused_and_untyped = null; + static public $used_and_untyped = null; + static public int $c1_int = 0; + static public string $c1_string = 'asdf'; + /** @var ?SurveyTarget */ + static public $inst1 = null; + /** @var ?tuple(int, bool, ?SurveyTarget[]) */ + static public $tup1 = null; + /** @var ?tuple(bool) */ + static public $tup2 = null; + /** @var ?tuple(bool,bool,bool,?int,bool,bool,bool,bool) */ + static public $tup3 = null; + /** @var ?tuple(bool,bool,bool,bool,bool,bool,bool,bool,bool) */ + static public $tup4 = null; + /** @var ?shape(single: tuple(bool, mixed, bool)) */ + static public $sh1 = null; + /** @var Exception */ + static public $ex1 = null; + /** @var ?LogicException */ + static public $ex2 = null; + + static function use() { + self::$c1_string .= 'a' . 'b'; + self::$c1_int += 10; + + self::$tup1 = tuple(1, true, null); + self::$tup1 = tuple(1, true, [new SurveyTarget]); + self::$sh1 = shape([ + 'single' => tuple(true, [1, 'str'], false), + ]); + self::$tup3 = tuple(true,true,true,null,true,true,false,true); + self::$tup4 = tuple(true,true,true,true,true,true,true,false,true); + self::$ex1 = new Exception; + self::$ex2 = new LogicException; + + echo self::$used_and_untyped, "\n"; + echo self::$c1_int, "\n"; + echo self::$c1_string, "\n"; + echo self::$sh1['single'][1][1], "\n"; + + var_dump(self::$tup2 === null); + self::$tup2 = tuple(true); + var_dump(self::$tup2[0]); + + var_dump(self::$tup3[6]); + var_dump(self::$tup3[7]); + + var_dump(self::$tup4[7]); + var_dump(self::$tup4[8]); + } +} + +WithGlobals::use(); + +global $g_unknown; + +function accessUnknown() { + global $g_unknown; +} + +accessUnknown(); + +class WithUnusedMethod { + static public int $used_only_in_unreachable = 123; + function unreachableViaCfgMethod() { + echo self::$used_only_in_unreachable, "\n"; + } + function usedMethod() { + throw new Exception; + // this call will be deleted in cfg + $this->unreachableViaCfgMethod(); + } +} + +if (0) (new WithUnusedMethod)->usedMethod(); + +function useSuperglobals() { + $_REQUEST = ['l' => 1]; + echo "count _REQUEST = ", count($_REQUEST), "\n"; + echo "php_sapi = ", PHP_SAPI, "\n"; +} + +useSuperglobals(); + +function toLowerPrint() { + var_dump (mb_strtolower('ABCABC')); +} +toLowerPrint(); + +trait T { + static public int $i = 0; + + public function incAndPrint() { + self::$i++; + echo get_class($this), " = ", self::$i, "\n"; + } +} + +class TInst1 { + use T; +} + +(new TInst1)->incAndPrint(); + +class UnusedClass1 { + static public $field1 = 0; + static private $field2 = null; +} + +class UnusedClass2 { + use T; +} + +$global_arr = [1,2,3]; +foreach ($global_arr as &$global_item_ref) { + $global_item_ref *= 2; +} +unset($global_item_ref); +echo "global_arr = ", implode(',', $global_arr), "\n"; + +class H { function str() { return 'str'; } } +function heredoc(): H { return new H; } + +$html1 = 'div'; +$html2 = 'span'; +if (true) { + $hd = heredoc(); + $html1 .= " title=\"{$hd->str()}\""; + $html2 .= " header=\"{$hd->str()}\""; +} +echo $html1, ' ', $html2, "\n"; + +$htmls = ['i', 'b']; +if (true) { + $hd = heredoc(); + foreach ($htmls as &$htmli) { + $htmli .= " data-txt=\"{$hd->str()}\""; + } +} +echo implode(' ', $htmls), "\n"; + +class SomeAnother22 { + static public string $final_message = ''; + + static function formAndPrint() { + $owner_id = 1; + SomeAnother22::$final_message .= "owner_id = $owner_id"; + echo SomeAnother22::$final_message, "\n"; + } +} + +SomeAnother22::formAndPrint(); + +function declaresGlobalButNotUsesIt() { + global $g_unknown, $g_assigned_const_bool; + static $s_unused = 0; + echo __FUNCTION__, "\n"; + return; + echo $g_assigned_const_bool; + echo $s_unused; +} + +declaresGlobalButNotUsesIt(); + +class UsedInFunctionStaticOnly { + public string $str = ''; +} + +function hasStaticInstance() { + /** @var UsedInFunctionStaticOnly $inst */ + static $inst = null; + if ($inst === null) + $inst = new UsedInFunctionStaticOnly; +} +hasStaticInstance(); + diff --git a/tests/python/tests/ffi/test_ffi.py b/tests/python/tests/ffi/test_ffi.py index 9ce429a2c0..b40e013d8f 100644 --- a/tests/python/tests/ffi/test_ffi.py +++ b/tests/python/tests/ffi/test_ffi.py @@ -65,7 +65,7 @@ def codegen_find(self, patterns): full_name = os.path.join(path, f) if not os.path.isdir(full_name): continue - if f.startswith('o_vars_'): + if f.startswith('o_globals_') or f.startswith('o_const_'): continue for f2 in os.listdir(full_name): with open(os.path.join(full_name, f2), 'r') as cpp_file: diff --git a/tests/python/tests/libs/php/lib_examples/example1/php/index.php b/tests/python/tests/libs/php/lib_examples/example1/php/index.php index b5cc7c3150..e79bdb3562 100644 --- a/tests/python/tests/libs/php/lib_examples/example1/php/index.php +++ b/tests/python/tests/libs/php/lib_examples/example1/php/index.php @@ -1,12 +1,19 @@