From e0c72ad51124a6708cc2b20915701a2dec369925 Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Sun, 5 Nov 2023 20:46:36 -0800 Subject: [PATCH] Make the preprocessor look at builtin includes first Removes references to filesystem include dir, no longer required for embedding or packaging. --- src/build_amalg.py | 4 ++-- src/build_compiler_includes_header.py | 28 +++++++++++---------------- src/dyibicc.h | 1 + src/entry.c | 1 - src/fuzz_entry.c | 1 - src/gen.py | 1 + src/libdyibicc.h | 3 --- src/main.c | 9 ++++----- src/preprocess.c | 23 +++++++++++++++++++++- test/test_helpers_for_update.py | 1 - 10 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/build_amalg.py b/src/build_amalg.py index 6f0cf61..c77326d 100644 --- a/src/build_amalg.py +++ b/src/build_amalg.py @@ -51,6 +51,8 @@ def include_file(f, src): continue if line.startswith('#include "dyn_basic_pdb.h"'): continue + if line.startswith('#include "compincl.h"'): + continue if line.startswith('#include "../include/all/reflect.h"'): continue if line.startswith('#pragma once'): @@ -82,8 +84,6 @@ def main(): f.write('#endif // !X64WIN\n') shutil.copyfile(os.path.join(root_src, 'libdyibicc.h'), os.path.join(out_dir, 'libdyibicc.h')) shutil.copyfile(os.path.join(root_src, '..', 'LICENSE'), os.path.join(out_dir, 'LICENSE')) - shutil.copytree(os.path.join(root_src, '..', 'include'), os.path.join(out_dir, 'include'), - dirs_exist_ok=True) # Smoke test that there's no warnings, etc. Currently only on host platform. os.chdir(out_dir) diff --git a/src/build_compiler_includes_header.py b/src/build_compiler_includes_header.py index 3ed3346..e2cb854 100644 --- a/src/build_compiler_includes_header.py +++ b/src/build_compiler_includes_header.py @@ -25,7 +25,7 @@ def escape(s, encoding='ascii'): def main(): out, inputs = sys.argv[1], sys.argv[2:] with open(out, 'w', newline='\n', encoding='utf-8') as f: - files = {'all':{}, 'linux':{}, 'win':{}} + files = [] for i in inputs: PREFIX = '../../include/' if not i.startswith(PREFIX): @@ -33,27 +33,21 @@ def main(): return 1 path = i[len(PREFIX):] category, name = os.path.split(path) - if files.get(category) is None: - print('Unexpected include category %s' % category) - return 1 with open(i, 'r', encoding='utf-8') as g: - files[category][name] = g.read() - - + files.append((category, name, g.read())) - f.write('#pragma once\n\n' - 'typedef struct CompilerInclude {\n' - ' char* name;\n' + f.write('typedef struct CompilerInclude {\n' + ' char* path;\n' ' int offset;\n' '} CompilerInclude;\n\n') + blob = '' - for c, fc in files.items(): - f.write('static CompilerInclude compiler_includes_%s[] = {\n' % c) - for name, contents in fc.items(): - f.write(' { "%s", %d },\n' % (name, len(blob))) - blob += contents + '\0' - f.write('};\n') - f.write('static const unsigned char compiler_include_blob[%d] = {\n' % len(blob)) + f.write('static CompilerInclude compiler_includes[%d] = {\n' % len(files)) + for cat, name, contents in files: + f.write(' { "__include__/%s/%s", %d },\n' % (cat, name, len(blob))) + blob += contents + '\0' + f.write('};\n') + f.write('static unsigned char compiler_include_blob[%d] = {\n' % len(blob)) f.write('\n'.join(textwrap.wrap(', '.join(str(ord(x)) for x in blob)))) f.write('};\n') diff --git a/src/dyibicc.h b/src/dyibicc.h index 92635d8..1b04b73 100644 --- a/src/dyibicc.h +++ b/src/dyibicc.h @@ -717,6 +717,7 @@ typedef struct CompilerState { HashMap preprocess__pragma_once; HashMap preprocess__container_included; TokenPtrArray preprocess__container_tokens; + HashMap preprocess__builtin_includes_map; int preprocess__include_next_idx; HashMap preprocess__include_path_cache; diff --git a/src/entry.c b/src/entry.c index 651bb95..78b4199 100644 --- a/src/entry.c +++ b/src/entry.c @@ -100,7 +100,6 @@ int main(int argc, char** argv) { DyibiccEnviromentData env_data = { .include_paths = (const char**)include_paths.data, .files = (const char**)input_paths.data, - .dyibicc_include_dir = "./include", .load_file_contents = read_file, .get_function_address = NULL, .output_function = NULL, diff --git a/src/fuzz_entry.c b/src/fuzz_entry.c index 900c096..c860af2 100644 --- a/src/fuzz_entry.c +++ b/src/fuzz_entry.c @@ -25,7 +25,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) { DyibiccEnviromentData env_data = { .include_paths = no_include_paths, .files = one_input_path, - .dyibicc_include_dir = "./include", .load_file_contents = read_file, .get_function_address = NULL, .output_function = NULL, diff --git a/src/gen.py b/src/gen.py index 7db0500..c434c34 100644 --- a/src/gen.py +++ b/src/gen.py @@ -195,6 +195,7 @@ def generate(platform, config, settings, cmdlines, tests, upd_tests, fuzz_tests) EXTRAS_FOR_AMALG = [ '$root/dyibicc.h', + '$root/../include/all/reflect.h', 'compincl.h', '$root/dynasm/dasm_proto.h', '$root/dynasm/dasm_x86.h', diff --git a/src/libdyibicc.h b/src/libdyibicc.h index a6f36a6..5cad0dd 100644 --- a/src/libdyibicc.h +++ b/src/libdyibicc.h @@ -23,9 +23,6 @@ typedef struct DyibiccEnviromentData { // NULL-terminated list of .c files to include in the project. const char** files; - // Path to the compiler's include directory. - const char* dyibicc_include_dir; - // Load the contents of a file by name (typically from disk). // // NOTE/TODO: There is currently a gotcha with this callback. It is used in diff --git a/src/main.c b/src/main.c index 39528e6..729ea89 100644 --- a/src/main.c +++ b/src/main.c @@ -63,8 +63,8 @@ DyibiccContext* dyibicc_set_environment(DyibiccEnviromentData* env_data) { StringArray sys_inc_paths = {0}; #if X64WIN - strarray_push(&sys_inc_paths, format(AL_Temp, "%s/win", env_data->dyibicc_include_dir), AL_Temp); - strarray_push(&sys_inc_paths, format(AL_Temp, "%s/all", env_data->dyibicc_include_dir), AL_Temp); + strarray_push(&sys_inc_paths, "__include__/win", AL_Temp); + strarray_push(&sys_inc_paths, "__include__/all", AL_Temp); #define GET_ENV_VAR(x) \ char* env_##x = getenv(#x); \ @@ -89,9 +89,8 @@ DyibiccContext* dyibicc_set_environment(DyibiccEnviromentData* env_data) { strarray_push(&sys_inc_paths, format(AL_Temp, "%sinclude", env_VcToolsInstallDir), AL_Temp); #else - strarray_push(&sys_inc_paths, format(AL_Temp, "%s/linux", env_data->dyibicc_include_dir), - AL_Temp); - strarray_push(&sys_inc_paths, format(AL_Temp, "%s/all", env_data->dyibicc_include_dir), AL_Temp); + strarray_push(&sys_inc_paths, "__include__/linux", AL_Temp); + strarray_push(&sys_inc_paths, "__include__/all", AL_Temp); strarray_push(&sys_inc_paths, "/usr/local/include", AL_Temp); strarray_push(&sys_inc_paths, "/usr/include/x86_64-linux-gnu", AL_Temp); diff --git a/src/preprocess.c b/src/preprocess.c index 297d917..af35b74 100644 --- a/src/preprocess.c +++ b/src/preprocess.c @@ -623,8 +623,23 @@ static Token* subst(Token* tok, MacroArg* args) { return head.next; } +static bool file_exists_in_builtins(char* path) { + if (C(builtin_includes_map).capacity == 0) { + for (size_t i = 0; i < sizeof(compiler_includes) / sizeof(compiler_includes[0]); ++i) { + hashmap_put(&C(builtin_includes_map), compiler_includes[i].path, + (void*)&compiler_include_blob[compiler_includes[i].offset]); + } + } + + return hashmap_get(&C(builtin_includes_map), path) != NULL; +} + static bool file_exists(char* path) { struct stat st; + + if (file_exists_in_builtins(path)) + return true; + return !stat(path, &st); } @@ -809,7 +824,13 @@ static Token* include_file(Token* tok, char* path, Token* filename_tok) { if (guard_name && hashmap_get(&C(macros), guard_name)) return tok; - Token* tok2 = tokenize_file(path); + Token* tok2; + char* builtin_include_contents = hashmap_get(&C(builtin_includes_map), path); + if (builtin_include_contents) { + tok2 = tokenize_filecontents(path, builtin_include_contents); + } else { + tok2 = tokenize_file(path); + } if (!tok2) error_tok(filename_tok, "%s: cannot open file: %s", path, strerror(errno)); diff --git a/test/test_helpers_for_update.py b/test/test_helpers_for_update.py index 2c464a2..f75c6fd 100644 --- a/test/test_helpers_for_update.py +++ b/test/test_helpers_for_update.py @@ -41,7 +41,6 @@ DyibiccEnviromentData env_data = { .include_paths = (const char**)include_paths, .files = (const char**)input_paths, - .dyibicc_include_dir = "embed/include", .load_file_contents = get_file_by_name, .get_function_address = get_host_helper_func, .output_function = NULL,