Merge pull request #21700 from ziglang/cli-lib-dirs

move linker input file parsing to the frontend

CMakeLists.txt

@@ -522,6 +522,7 @@ set(ZIG_STAGE2_SOURCES
     src/Sema.zig
     src/Sema/bitcast.zig
     src/Sema/comptime_ptr_access.zig
+    src/ThreadSafeQueue.zig
     src/Type.zig
     src/Value.zig
     src/Zcu.zig
@@ -601,7 +602,6 @@ set(ZIG_STAGE2_SOURCES
     src/link/Elf/Archive.zig
     src/link/Elf/Atom.zig
     src/link/Elf/AtomList.zig
-    src/link/Elf/LdScript.zig
     src/link/Elf/LinkerDefined.zig
     src/link/Elf/Merge.zig
     src/link/Elf/Object.zig
@@ -615,6 +615,7 @@ set(ZIG_STAGE2_SOURCES
     src/link/Elf/relocatable.zig
     src/link/Elf/relocation.zig
     src/link/Elf/synthetic_sections.zig
+    src/link/LdScript.zig
     src/link/MachO.zig
     src/link/MachO/Archive.zig
     src/link/MachO/Atom.zig

lib/c.zig

@@ -38,8 +38,6 @@ comptime {
         @export(&strncpy, .{ .name = "strncpy", .linkage = .strong });
         @export(&strcat, .{ .name = "strcat", .linkage = .strong });
         @export(&strncat, .{ .name = "strncat", .linkage = .strong });
-    } else if (is_msvc) {
-        @export(&_fltused, .{ .name = "_fltused", .linkage = .strong });
     }
 }
 
@@ -62,8 +60,6 @@ fn wasm_start() callconv(.C) void {
     _ = main(0, undefined);
 }
 
-var _fltused: c_int = 1;
-
 fn strcpy(dest: [*:0]u8, src: [*:0]const u8) callconv(.C) [*:0]u8 {
     var i: usize = 0;
     while (src[i] != 0) : (i += 1) {

lib/compiler/build_runner.zig

@@ -280,6 +280,10 @@ pub fn main() !void {
                 builder.enable_darling = true;
             } else if (mem.eql(u8, arg, "-fno-darling")) {
                 builder.enable_darling = false;
+            } else if (mem.eql(u8, arg, "-fallow-so-scripts")) {
+                graph.allow_so_scripts = true;
+            } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) {
+                graph.allow_so_scripts = false;
             } else if (mem.eql(u8, arg, "-freference-trace")) {
                 builder.reference_trace = 256;
             } else if (mem.startsWith(u8, arg, "-freference-trace=")) {
@@ -1341,6 +1345,8 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
         \\Advanced Options:
         \\  -freference-trace[=num]      How many lines of reference trace should be shown per compile error
         \\  -fno-reference-trace         Disable reference trace
+        \\  -fallow-so-scripts           Allows .so files to be GNU ld scripts
+        \\  -fno-allow-so-scripts        (default) .so files must be ELF files
         \\  --build-file [file]          Override path to build.zig
         \\  --cache-dir [path]           Override path to local Zig cache directory
         \\  --global-cache-dir [path]    Override path to global Zig cache directory

lib/compiler_rt.zig

@@ -1,6 +1,7 @@
 const builtin = @import("builtin");
+const common = @import("compiler_rt/common.zig");
 
-pub const panic = @import("compiler_rt/common.zig").panic;
+pub const panic = common.panic;
 
 comptime {
     // Integer routines
@@ -236,4 +237,10 @@ comptime {
         _ = @import("compiler_rt/bcmp.zig");
         _ = @import("compiler_rt/ssp.zig");
     }
+
+    if (!builtin.link_libc and builtin.abi == .msvc) {
+        @export(&_fltused, .{ .name = "_fltused", .linkage = common.linkage, .visibility = common.visibility });
+    }
 }
+
+var _fltused: c_int = 1;

lib/std/Build.zig

@@ -123,6 +123,7 @@ pub const Graph = struct {
     incremental: ?bool = null,
     random_seed: u32 = 0,
     dependency_cache: InitializedDepMap = .empty,
+    allow_so_scripts: ?bool = null,
 };
 
 const AvailableDeps = []const struct { []const u8, []const u8 };

lib/std/Build/Cache.zig

@@ -142,6 +142,9 @@ pub const hasher_init: Hasher = Hasher.init(&[_]u8{
 pub const File = struct {
     prefixed_path: PrefixedPath,
     max_file_size: ?usize,
+    /// Populated if the user calls `addOpenedFile`.
+    /// The handle is not owned here.
+    handle: ?fs.File,
     stat: Stat,
     bin_digest: BinDigest,
     contents: ?[]const u8,
@@ -173,6 +176,11 @@ pub const File = struct {
         const new = new_max_size orelse return;
         file.max_file_size = if (file.max_file_size) |old| @max(old, new) else new;
     }
+
+    pub fn updateHandle(file: *File, new_handle: ?fs.File) void {
+        const handle = new_handle orelse return;
+        file.handle = handle;
+    }
 };
 
 pub const HashHelper = struct {
@@ -363,15 +371,20 @@ pub const Manifest = struct {
     /// var file_contents = cache_hash.files.keys()[file_index].contents.?;
     /// ```
     pub fn addFilePath(m: *Manifest, file_path: Path, max_file_size: ?usize) !usize {
+        return addOpenedFile(m, file_path, null, max_file_size);
+    }
+
+    /// Same as `addFilePath` except the file has already been opened.
+    pub fn addOpenedFile(m: *Manifest, path: Path, handle: ?fs.File, max_file_size: ?usize) !usize {
         const gpa = m.cache.gpa;
         try m.files.ensureUnusedCapacity(gpa, 1);
         const resolved_path = try fs.path.resolve(gpa, &.{
-            file_path.root_dir.path orelse ".",
-            file_path.subPathOrDot(),
+            path.root_dir.path orelse ".",
+            path.subPathOrDot(),
         });
         errdefer gpa.free(resolved_path);
         const prefixed_path = try m.cache.findPrefixResolved(resolved_path);
-        return addFileInner(m, prefixed_path, max_file_size);
+        return addFileInner(m, prefixed_path, handle, max_file_size);
     }
 
     /// Deprecated; use `addFilePath`.
@@ -383,13 +396,14 @@ pub const Manifest = struct {
         const prefixed_path = try self.cache.findPrefix(file_path);
         errdefer gpa.free(prefixed_path.sub_path);
 
-        return addFileInner(self, prefixed_path, max_file_size);
+        return addFileInner(self, prefixed_path, null, max_file_size);
     }
 
-    fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, max_file_size: ?usize) !usize {
+    fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, handle: ?fs.File, max_file_size: ?usize) usize {
         const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{});
         if (gop.found_existing) {
             gop.key_ptr.updateMaxSize(max_file_size);
+            gop.key_ptr.updateHandle(handle);
             return gop.index;
         }
         gop.key_ptr.* = .{
@@ -398,6 +412,7 @@ pub const Manifest = struct {
             .max_file_size = max_file_size,
             .stat = undefined,
             .bin_digest = undefined,
+            .handle = handle,
         };
 
         self.hash.add(prefixed_path.prefix);
@@ -565,6 +580,7 @@ pub const Manifest = struct {
                             },
                             .contents = null,
                             .max_file_size = null,
+                            .handle = null,
                             .stat = .{
                                 .size = stat_size,
                                 .inode = stat_inode,
@@ -708,12 +724,19 @@ pub const Manifest = struct {
     }
 
     fn populateFileHash(self: *Manifest, ch_file: *File) !void {
-        const pp = ch_file.prefixed_path;
-        const dir = self.cache.prefixes()[pp.prefix].handle;
-        const file = try dir.openFile(pp.sub_path, .{});
-        defer file.close();
+        if (ch_file.handle) |handle| {
+            return populateFileHashHandle(self, ch_file, handle);
+        } else {
+            const pp = ch_file.prefixed_path;
+            const dir = self.cache.prefixes()[pp.prefix].handle;
+            const handle = try dir.openFile(pp.sub_path, .{});
+            defer handle.close();
+            return populateFileHashHandle(self, ch_file, handle);
+        }
+    }
 
-        const actual_stat = try file.stat();
+    fn populateFileHashHandle(self: *Manifest, ch_file: *File, handle: fs.File) !void {
+        const actual_stat = try handle.stat();
         ch_file.stat = .{
             .size = actual_stat.size,
             .mtime = actual_stat.mtime,
@@ -739,8 +762,7 @@ pub const Manifest = struct {
             var hasher = hasher_init;
             var off: usize = 0;
             while (true) {
-                // give me everything you've got, captain
-                const bytes_read = try file.read(contents[off..]);
+                const bytes_read = try handle.pread(contents[off..], off);
                 if (bytes_read == 0) break;
                 hasher.update(contents[off..][0..bytes_read]);
                 off += bytes_read;
@@ -749,7 +771,7 @@ pub const Manifest = struct {
 
             ch_file.contents = contents;
         } else {
-            try hashFile(file, &ch_file.bin_digest);
+            try hashFile(handle, &ch_file.bin_digest);
         }
 
         self.hash.hasher.update(&ch_file.bin_digest);
@@ -813,6 +835,7 @@ pub const Manifest = struct {
         gop.key_ptr.* = .{
             .prefixed_path = prefixed_path,
             .max_file_size = null,
+            .handle = null,
             .stat = undefined,
             .bin_digest = undefined,
             .contents = null,
@@ -851,6 +874,7 @@ pub const Manifest = struct {
         new_file.* = .{
             .prefixed_path = prefixed_path,
             .max_file_size = null,
+            .handle = null,
             .stat = stat,
             .bin_digest = undefined,
             .contents = null,
@@ -1067,6 +1091,7 @@ pub const Manifest = struct {
             gop.key_ptr.* = .{
                 .prefixed_path = prefixed_path,
                 .max_file_size = file.max_file_size,
+                .handle = file.handle,
                 .stat = file.stat,
                 .bin_digest = file.bin_digest,
                 .contents = null,
@@ -1103,14 +1128,14 @@ pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void
 
 fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
     var buf: [1024]u8 = undefined;
-
     var hasher = hasher_init;
+    var off: u64 = 0;
     while (true) {
-        const bytes_read = try file.read(&buf);
+        const bytes_read = try file.pread(&buf, off);
         if (bytes_read == 0) break;
         hasher.update(buf[0..bytes_read]);
+        off += bytes_read;
     }
-
     hasher.final(bin_digest);
 }
 

lib/std/Build/Step/Compile.zig

@@ -186,6 +186,15 @@ want_lto: ?bool = null,
 use_llvm: ?bool,
 use_lld: ?bool,
 
+/// Corresponds to the `-fallow-so-scripts` / `-fno-allow-so-scripts` CLI
+/// flags, overriding the global user setting provided to the `zig build`
+/// command.
+///
+/// The compiler defaults this value to off so that users whose system shared
+/// libraries are all ELF files don't have to pay the cost of checking every
+/// file to find out if it is a text file instead.
+allow_so_scripts: ?bool = null,
+
 /// This is an advanced setting that can change the intent of this Compile step.
 /// If this value is non-null, it means that this Compile step exists to
 /// check for compile errors and return *success* if they match, and failure
@@ -236,6 +245,7 @@ pub const ExpectedCompileErrors = union(enum) {
     contains: []const u8,
     exact: []const []const u8,
     starts_with: []const u8,
+    stderr_contains: []const u8,
 };
 
 pub const Entry = union(enum) {
@@ -1035,6 +1045,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
     if (b.reference_trace) |some| {
         try zig_args.append(try std.fmt.allocPrint(arena, "-freference-trace={d}", .{some}));
     }
+    try addFlag(&zig_args, "allow-so-scripts", compile.allow_so_scripts orelse b.graph.allow_so_scripts);
 
     try addFlag(&zig_args, "llvm", compile.use_llvm);
     try addFlag(&zig_args, "lld", compile.use_lld);
@@ -1945,39 +1956,62 @@ fn checkCompileErrors(compile: *Compile) !void {
 
     const arena = compile.step.owner.allocator;
 
-    var actual_stderr_list = std.ArrayList(u8).init(arena);
+    var actual_errors_list = std.ArrayList(u8).init(arena);
     try actual_eb.renderToWriter(.{
         .ttyconf = .no_color,
         .include_reference_trace = false,
         .include_source_line = false,
-    }, actual_stderr_list.writer());
-    const actual_stderr = try actual_stderr_list.toOwnedSlice();
+    }, actual_errors_list.writer());
+    const actual_errors = try actual_errors_list.toOwnedSlice();
 
     // Render the expected lines into a string that we can compare verbatim.
     var expected_generated = std.ArrayList(u8).init(arena);
     const expect_errors = compile.expect_errors.?;
 
-    var actual_line_it = mem.splitScalar(u8, actual_stderr, '\n');
+    var actual_line_it = mem.splitScalar(u8, actual_errors, '\n');
 
     // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile
     switch (expect_errors) {
         .starts_with => |expect_starts_with| {
-            if (std.mem.startsWith(u8, actual_stderr, expect_starts_with)) return;
+            if (std.mem.startsWith(u8, actual_errors, expect_starts_with)) return;
             return compile.step.fail(
                 \\
                 \\========= should start with: ============
                 \\{s}
                 \\========= but not found: ================
                 \\{s}
                 \\=========================================
-            , .{ expect_starts_with, actual_stderr });
+            , .{ expect_starts_with, actual_errors });
         },
         .contains => |expect_line| {
             while (actual_line_it.next()) |actual_line| {
                 if (!matchCompileError(actual_line, expect_line)) continue;
                 return;
             }
 
+            return compile.step.fail(
+                \\
+                \\========= should contain: ===============
+                \\{s}
+                \\========= but not found: ================
+                \\{s}
+                \\=========================================
+            , .{ expect_line, actual_errors });
+        },
+        .stderr_contains => |expect_line| {
+            const actual_stderr: []const u8 = if (compile.step.result_error_msgs.items.len > 0)
+                compile.step.result_error_msgs.items[0]
+            else
+                &.{};
+            compile.step.result_error_msgs.clearRetainingCapacity();
+
+            var stderr_line_it = mem.splitScalar(u8, actual_stderr, '\n');
+
+            while (stderr_line_it.next()) |actual_line| {
+                if (!matchCompileError(actual_line, expect_line)) continue;
+                return;
+            }
+
             return compile.step.fail(
                 \\
                 \\========= should contain: ===============
@@ -2003,7 +2037,7 @@ fn checkCompileErrors(compile: *Compile) !void {
                 try expected_generated.append('\n');
             }
 
-            if (mem.eql(u8, expected_generated.items, actual_stderr)) return;
+            if (mem.eql(u8, expected_generated.items, actual_errors)) return;
 
             return compile.step.fail(
                 \\
@@ -2012,7 +2046,7 @@ fn checkCompileErrors(compile: *Compile) !void {
                 \\========= but found: ====================
                 \\{s}
                 \\=========================================
-            , .{ expected_generated.items, actual_stderr });
+            , .{ expected_generated.items, actual_errors });
         },
     }
 }

lib/std/Thread/WaitGroup.zig

@@ -14,6 +14,11 @@ pub fn start(self: *WaitGroup) void {
     assert((state / one_pending) < (std.math.maxInt(usize) / one_pending));
 }
 
+pub fn startMany(self: *WaitGroup, n: usize) void {
+    const state = self.state.fetchAdd(one_pending * n, .monotonic);
+    assert((state / one_pending) < (std.math.maxInt(usize) / one_pending));
+}
+
 pub fn finish(self: *WaitGroup) void {
     const state = self.state.fetchSub(one_pending, .acq_rel);
     assert((state / one_pending) > 0);