Make the preprocessor look at builtin includes first

Removes references to filesystem include dir, no longer required for
embedding or packaging.

src/build_amalg.py

@@ -51,6 +51,8 @@ def include_file(f, src):
                 continue
             if line.startswith('#include "dyn_basic_pdb.h"'):
                 continue
+            if line.startswith('#include "compincl.h"'):
+                continue
             if line.startswith('#include "../include/all/reflect.h"'):
                 continue
             if line.startswith('#pragma once'):
@@ -82,8 +84,6 @@ def main():
         f.write('#endif // !X64WIN\n')
     shutil.copyfile(os.path.join(root_src, 'libdyibicc.h'), os.path.join(out_dir, 'libdyibicc.h'))
     shutil.copyfile(os.path.join(root_src, '..', 'LICENSE'), os.path.join(out_dir, 'LICENSE'))
-    shutil.copytree(os.path.join(root_src, '..', 'include'), os.path.join(out_dir, 'include'),
-                    dirs_exist_ok=True)
 
     # Smoke test that there's no warnings, etc. Currently only on host platform.
     os.chdir(out_dir)

src/build_compiler_includes_header.py

@@ -25,35 +25,29 @@ def escape(s, encoding='ascii'):
 def main():
     out, inputs = sys.argv[1], sys.argv[2:]
     with open(out, 'w', newline='\n', encoding='utf-8') as f:
-        files = {'all':{}, 'linux':{}, 'win':{}}
+        files = []
         for i in inputs:
             PREFIX = '../../include/'
             if not i.startswith(PREFIX):
                 print('Expecting %s as prefix' % PREFIX)
                 return 1
             path = i[len(PREFIX):]
             category, name = os.path.split(path)
-            if files.get(category) is None:
-                print('Unexpected include category %s' % category)
-                return 1
             with open(i, 'r', encoding='utf-8') as g:
-                files[category][name] = g.read()
-
-
+                files.append((category, name, g.read()))
 
-        f.write('#pragma once\n\n'
-                'typedef struct CompilerInclude {\n'
-                '    char* name;\n'
+        f.write('typedef struct CompilerInclude {\n'
+                '    char* path;\n'
                 '    int offset;\n'
                 '} CompilerInclude;\n\n')
+
         blob = ''
-        for c, fc in files.items():
-            f.write('static CompilerInclude compiler_includes_%s[] = {\n' % c)
-            for name, contents in fc.items():
-                f.write('    { "%s", %d },\n' % (name, len(blob)))
-                blob += contents + '\0'
-            f.write('};\n')
-        f.write('static const unsigned char compiler_include_blob[%d] = {\n' % len(blob))
+        f.write('static CompilerInclude compiler_includes[%d] = {\n' % len(files))
+        for cat, name, contents  in files:
+            f.write('    { "__include__/%s/%s", %d },\n' % (cat, name, len(blob)))
+            blob += contents + '\0'
+        f.write('};\n')
+        f.write('static unsigned char compiler_include_blob[%d] = {\n' % len(blob))
         f.write('\n'.join(textwrap.wrap(', '.join(str(ord(x)) for x in blob))))
         f.write('};\n')
 

src/dyibicc.h

@@ -717,6 +717,7 @@ typedef struct CompilerState {
   HashMap preprocess__pragma_once;
   HashMap preprocess__container_included;
   TokenPtrArray preprocess__container_tokens;
+  HashMap preprocess__builtin_includes_map;
 
   int preprocess__include_next_idx;
   HashMap preprocess__include_path_cache;

src/entry.c

@@ -100,7 +100,6 @@ int main(int argc, char** argv) {
   DyibiccEnviromentData env_data = {
       .include_paths = (const char**)include_paths.data,
       .files = (const char**)input_paths.data,
-      .dyibicc_include_dir = "./include",
       .load_file_contents = read_file,
       .get_function_address = NULL,
       .output_function = NULL,

src/fuzz_entry.c

@@ -25,7 +25,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) {
   DyibiccEnviromentData env_data = {
       .include_paths = no_include_paths,
       .files = one_input_path,
-      .dyibicc_include_dir = "./include",
       .load_file_contents = read_file,
       .get_function_address = NULL,
       .output_function = NULL,

src/gen.py

@@ -195,6 +195,7 @@ def generate(platform, config, settings, cmdlines, tests, upd_tests, fuzz_tests)
 
         EXTRAS_FOR_AMALG = [
                 '$root/dyibicc.h',
+                '$root/../include/all/reflect.h',
                 'compincl.h',
                 '$root/dynasm/dasm_proto.h',
                 '$root/dynasm/dasm_x86.h',

src/libdyibicc.h

@@ -23,9 +23,6 @@ typedef struct DyibiccEnviromentData {
   // NULL-terminated list of .c files to include in the project.
   const char** files;
 
-  // Path to the compiler's include directory.
-  const char* dyibicc_include_dir;
-
   // Load the contents of a file by name (typically from disk).
   //
   // NOTE/TODO: There is currently a gotcha with this callback. It is used in

src/main.c

@@ -63,8 +63,8 @@ DyibiccContext* dyibicc_set_environment(DyibiccEnviromentData* env_data) {
 
   StringArray sys_inc_paths = {0};
 #if X64WIN
-  strarray_push(&sys_inc_paths, format(AL_Temp, "%s/win", env_data->dyibicc_include_dir), AL_Temp);
-  strarray_push(&sys_inc_paths, format(AL_Temp, "%s/all", env_data->dyibicc_include_dir), AL_Temp);
+  strarray_push(&sys_inc_paths, "__include__/win", AL_Temp);
+  strarray_push(&sys_inc_paths, "__include__/all", AL_Temp);
 
 #define GET_ENV_VAR(x)                          \
   char* env_##x = getenv(#x);                   \
@@ -89,9 +89,8 @@ DyibiccContext* dyibicc_set_environment(DyibiccEnviromentData* env_data) {
   strarray_push(&sys_inc_paths, format(AL_Temp, "%sinclude", env_VcToolsInstallDir), AL_Temp);
 
 #else
-  strarray_push(&sys_inc_paths, format(AL_Temp, "%s/linux", env_data->dyibicc_include_dir),
-                AL_Temp);
-  strarray_push(&sys_inc_paths, format(AL_Temp, "%s/all", env_data->dyibicc_include_dir), AL_Temp);
+  strarray_push(&sys_inc_paths, "__include__/linux", AL_Temp);
+  strarray_push(&sys_inc_paths, "__include__/all", AL_Temp);
 
   strarray_push(&sys_inc_paths, "/usr/local/include", AL_Temp);
   strarray_push(&sys_inc_paths, "/usr/include/x86_64-linux-gnu", AL_Temp);

src/preprocess.c

@@ -623,8 +623,23 @@ static Token* subst(Token* tok, MacroArg* args) {
   return head.next;
 }
 
+static bool file_exists_in_builtins(char* path) {
+  if (C(builtin_includes_map).capacity == 0) {
+    for (size_t i = 0; i < sizeof(compiler_includes) / sizeof(compiler_includes[0]); ++i) {
+      hashmap_put(&C(builtin_includes_map), compiler_includes[i].path,
+                  (void*)&compiler_include_blob[compiler_includes[i].offset]);
+    }
+  }
+
+  return hashmap_get(&C(builtin_includes_map), path) != NULL;
+}
+
 static bool file_exists(char* path) {
   struct stat st;
+
+  if (file_exists_in_builtins(path))
+    return true;
+
   return !stat(path, &st);
 }
 
@@ -809,7 +824,13 @@ static Token* include_file(Token* tok, char* path, Token* filename_tok) {
   if (guard_name && hashmap_get(&C(macros), guard_name))
     return tok;
 
-  Token* tok2 = tokenize_file(path);
+  Token* tok2;
+  char* builtin_include_contents = hashmap_get(&C(builtin_includes_map), path);
+  if (builtin_include_contents) {
+    tok2 = tokenize_filecontents(path, builtin_include_contents);
+  } else {
+    tok2 = tokenize_file(path);
+  }
   if (!tok2)
     error_tok(filename_tok, "%s: cannot open file: %s", path, strerror(errno));
 

test/test_helpers_for_update.py

@@ -41,7 +41,6 @@
   DyibiccEnviromentData env_data = {
       .include_paths = (const char**)include_paths,
       .files = (const char**)input_paths,
-      .dyibicc_include_dir = "embed/include",
       .load_file_contents = get_file_by_name,
       .get_function_address = get_host_helper_func,
       .output_function = NULL,