Wrap LLD into python

.readthedocs.yaml

@@ -9,6 +9,8 @@ build:
   tools:
     python: "3.11"
   apt_packages:
+    - lld-15
+    - liblld-15-dev
     - llvm-15
   jobs:
     post_checkout:

buildscripts/incremental/setup_conda_environment.cmd

@@ -21,4 +21,7 @@ if "%LLVM%"=="16" (
 )
 
 call conda install -y -q -c %LLVMDEV_CHANNEL% llvmdev="%LLVM%" libxml2
+if "%LLVM%"=="16" (
+  call conda install -y -q -c %LLVMDEV_CHANNEL% lld="16"
+)
 if %errorlevel% neq 0 exit /b %errorlevel%

buildscripts/incremental/setup_conda_environment.sh

@@ -28,15 +28,20 @@ set -v
 
 # Install llvmdev (separate channel, for now)
 if [ "$LLVM" == "16" ]; then
-    $CONDA_INSTALL -c conda-forge llvmdev="16"
+    # also install lld, not required when using the numba channel because the numba
+    #  channel includes lld.
+    $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16"
 else
     $CONDA_INSTALL -c numba llvmdev="15.*"
 fi
 
+$CONDA_INSTALL -c numba libxml2
+
 # Install the compiler toolchain, for osx, bootstrapping needed
 # which happens in build.sh
 if [[ $(uname) == Linux ]]; then
-$CONDA_INSTALL gcc_linux-64 gxx_linux-64
+# $CONDA_INSTALL gcc_linux-64 gxx_linux-64
+$CONDA_INSTALL -c conda-forge libstdcxx-ng=12
 fi
 
 # Install dependencies for code coverage (codecov.io)

docs/source/user-guide/binding/index.rst

@@ -32,5 +32,6 @@ implement Numba_'s JIT compiler.
    analysis-utilities
    pass_timings
    misc
+   lld
    examples
 

docs/source/user-guide/binding/lld.rst

@@ -0,0 +1,63 @@
+================
+Linking with LLD
+================
+
+.. currentmodule:: llvmlite.binding
+
+The LLD linker is automatically built into llvmlite to provide easy-to-use, cross-platform linking.
+
+Known limitations
+=================
+
+Currently, only ``lld::elf::link`` is used and thus COFF and MACHO object files will **not** link.
+
+The following llvm drivers are usable from :func:`lld_main`
+
+* ld.lld   (Linux)
+* ld64.lld (macos/darwin)
+* lld-link (windows)
+* wasm-ld  (Web Assembly)
+
+Functions
+=========
+
+* .. function:: lld_main(args)
+
+    * ``args`` is a list of strings passed to the lld linker as arguments
+    * returns the output of the specific lld command being run. If there is an error, an :exc:`Exception` will be thrown.
+
+    example: ``binding.lld_main(["ld.lld", "--help"])``
+
+
+* .. function:: lld_linux(output_file, objects, args=[])
+
+    * ``objects`` is a list of object files' names as **strings**.
+    * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command)
+    * returns the output of lld_main for the given arguments
+
+    Links given object files into an executable using :func:`lld_main()`
+
+
+* .. function:: lld_windows(output_file, objects, args=[])
+
+    Link for Windows target
+
+
+* .. function:: lld_macos(output_file, objects, args=[])
+
+    Link for Macos target
+
+
+* .. function:: lld_wasm(output_file, objects, args=[])
+
+    Link for wasm target
+
+
+* .. function:: lld_auto(output_file, objects, args=[], add_extension=True)
+
+    * ``add_extension`` should this function automatically add **.exe** and **.app** for windows and macos targets
+    * returns the output of lld_main for the given arguments
+
+    Automatically determines which function to use given the host operating system.
+    This will only use the ``lld_linux``, ``lld_windows``, and ``lld_macos`` functions.
+    Throws :exc:`Exception` if the host operating system isn't ``Darwin``, ``Windows``, or ``Linux``

docs/source/user-guide/binding/target-information.rst

@@ -141,7 +141,7 @@ Classes
 
    * .. method:: create_target_machine(cpu='', features='', \
           opt=2, reloc='default', codemodel='jitdefault', \
-          abiname='')
+          abiname='', force_elf=False)
 
         Create a new :class:`TargetMachine` instance for this
         target and with the given options:
@@ -153,6 +153,7 @@ Classes
         * *reloc* is the relocation model.
         * *codemodel* is the code model.
         * *abiname* is the name of the ABI.
+        * *force_elf* is an option that forces the target to use the ELF object format instead of the target's object format.
 
         The defaults for reloc and codemodel are appropriate for
         JIT compilation.

ffi/CMakeLists.txt

@@ -18,9 +18,12 @@ set(CMAKE_CXX_STANDARD 17)
 include(GNUInstallDirs)
 
 find_package(LLVM REQUIRED CONFIG)
+find_package(LLD REQUIRED CONFIG)
 
 message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
 message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+message(STATUS "Found LLD ${LLD_PACKAGE_VERSION}")
+message(STATUS "Using LLDConfig.cmake in: ${LLD_DIR}")
 
 # Set your project compile flags.
 # E.g. if using the C++ header files
@@ -41,12 +44,14 @@ else()
     message(STATUS "SVML not found")
 endif()
 
+add_definitions(-DLLVM_ENABLE_PROJECTS="lld")
 
 # Define our shared library
 add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp
             module.cpp value.cpp executionengine.cpp transforms.cpp type.cpp
             passmanagers.cpp targets.cpp dylib.cpp linker.cpp object_file.cpp
-            custom_passes.cpp orcjit.cpp memorymanager.cpp newpassmanagers.cpp)
+            custom_passes.cpp orcjit.cpp memorymanager.cpp lld.cpp
+            newpassmanagers.cpp)
 
 # Find the libraries that correspond to the LLVM components
 # that we wish to use.
@@ -55,9 +60,9 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp
 # Bug reported upstream at: https://bugs.llvm.org/show_bug.cgi?id=47003
 # BROKEN: llvm_map_components_to_libnames(llvm_libs all)
 if ($ENV{LLVMLITE_SHARED})
-    set(llvm_libs LLVM)
+    set(llvm_libs $ENV{lld_libs} LLVM)
 else()
-    set(llvm_libs ${LLVM_AVAILABLE_LIBS})
+    set(llvm_libs ${LLVM_AVAILABLE_LIBS} $ENV{lld_libs})
 endif()
 
 # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears

ffi/Makefile.freebsd

@@ -12,7 +12,7 @@ INCLUDE = core.h
 SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \
 	executionengine.cpp transforms.cpp passmanagers.cpp type.cpp targets.cpp \
 	dylib.cpp linker.cpp object_file.cpp orcjit.cpp custom_passes.cpp \
-	memorymanager.cpp newpassmanagers.cpp
+	memorymanager.cpp lld.cpp newpassmanagers.cpp
 OUTPUT = libllvmlite.so
 
 all: $(OUTPUT)

ffi/Makefile.linux

@@ -13,7 +13,8 @@ LIBS = $(LLVM_LIBS)
 INCLUDE = core.h
 OBJ = assembly.o bitcode.o core.o initfini.o module.o value.o \
 	  executionengine.o transforms.o passmanagers.o targets.o type.o dylib.o \
-	  linker.o object_file.o custom_passes.o orcjit.o memorymanager.o newpassmanagers.o
+	  linker.o object_file.o custom_passes.o orcjit.o memorymanager.o lld.o \
+    newpassmanagers.o
 OUTPUT = libllvmlite.so
 
 all: $(OUTPUT)

ffi/Makefile.osx

@@ -9,7 +9,7 @@ INCLUDE = core.h
 SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \
 	  executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp type.cpp \
 	  dylib.cpp linker.cpp object_file.cpp custom_passes.cpp orcjit.cpp \
-	  memorymanager.cpp newpassmanagers.cpp
+	  memorymanager.cpp lld.cpp newpassmanagers.cpp
 OUTPUT = libllvmlite.dylib
 MACOSX_DEPLOYMENT_TARGET ?= 10.9
 

ffi/build.py

@@ -21,6 +21,10 @@
 
 is_64bit = sys.maxsize >= 2**32
 
+lld_libs = ["lldCommon", "lldELF", "lldWasm"]
+cxx_lld_libs = ' '.join([f"-l{lib}" for lib in lld_libs])
+cmake_lld_libs = ';'.join(lld_libs)
+
 
 def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None):
     old_dir = os.getcwd()
@@ -32,6 +36,7 @@ def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None):
     args.append(cmake_dir)
     try:
         os.chdir(build_dir)
+        os.environ['lld_libs'] = cmake_lld_libs
         print('Running:', ' '.join(args))
         subprocess.check_call(args)
     finally:
@@ -77,7 +82,6 @@ def find_windows_generator():
         generators.append(
             (env_generator, env_arch, env_toolkit)
         )
-
     generators.extend([
         # use VS2019 to match how llvmdev is built
         ('Visual Studio 16 2019', ('x64' if is_64bit else 'Win32'), 'v142'),
@@ -104,7 +108,6 @@ def main_windows():
     config = 'Release'
     if not os.path.exists(build_dir):
         os.mkdir(build_dir)
-    # Run configuration step
     try_cmake(here_dir, build_dir, *generator)
     subprocess.check_call(['cmake', '--build', build_dir, '--config', config])
     shutil.copy(os.path.join(build_dir, config, 'llvmlite.dll'), target_dir)
@@ -119,6 +122,7 @@ def main_posix_cmake(kind, library_ext):
     subprocess.check_call(['cmake', '--build', build_dir, '--config', config])
     shutil.copy(os.path.join(build_dir, 'libllvmlite' + library_ext), target_dir)
 
+
 def main_posix(kind, library_ext):
     if os.environ.get("LLVMLITE_USE_CMAKE", "0") == "1":
         return main_posix_cmake(kind, library_ext)
@@ -183,7 +187,8 @@ def main_posix(kind, library_ext):
     # Get LLVM information for building
     libs = run_llvm_config(llvm_config, "--system-libs --libs all".split())
     # Normalize whitespace (trim newlines)
-    os.environ['LLVM_LIBS'] = ' '.join(libs.split())
+    os.environ['LLVM_LIBS'] = \
+        f'{cxx_lld_libs} ' + ' '.join(libs.split())
 
     cxxflags = run_llvm_config(llvm_config, ["--cxxflags"])
     # on OSX cxxflags has null bytes at the end of the string, remove them
@@ -203,6 +208,7 @@ def main_posix(kind, library_ext):
 
     ldflags = run_llvm_config(llvm_config, ["--ldflags"])
     os.environ['LLVM_LDFLAGS'] = ldflags.strip()
+
     # static link libstdc++ for portability
     if int(os.environ.get('LLVMLITE_CXX_STATIC_LINK', 0)):
         os.environ['CXX_STATIC_LINK'] = "-static-libstdc++"
@@ -216,7 +222,6 @@ def main_posix(kind, library_ext):
     subprocess.check_call(['make', '-f', makefile] + makeopts)
     shutil.copy('libllvmlite' + library_ext, target_dir)
 
-
 def main():
     if sys.platform == 'win32':
         main_windows()

ffi/lld.cpp

@@ -0,0 +1,21 @@
+#include "core.h"
+#include "lld/Common/Driver.h"
+#include "llvm/Support/raw_ostream.h"
+
+extern "C" {
+
+API_EXPORT(bool)
+LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) {
+    // InitLLVM X(Argc, Argv);
+    std::string command_output;
+    llvm::raw_string_ostream command_stream(command_output);
+    std::vector<const char *> Args(Argv, Argv + Argc);
+
+    bool linker_output =
+        !lld::elf::link(Args, command_stream, command_stream, false, false);
+
+    *outstr = LLVMPY_CreateString(command_output.c_str());
+    return linker_output;
+}
+
+} // end extern "C"

llvmlite/binding/__init__.py

@@ -17,3 +17,4 @@
 from .object_file import *
 from .context import *
 from .orcjit import *
+from .lld import *

llvmlite/binding/lld.py

@@ -0,0 +1,84 @@
+from ctypes import c_bool, POINTER, c_char_p, c_int
+import platform
+from typing import List
+
+from llvmlite.binding import ffi
+
+
+ffi.lib.LLVMPY_lld_main.restype = c_bool
+ffi.lib.LLVMPY_lld_main.argtypes = [c_int, POINTER(c_char_p), POINTER(c_char_p)]
+
+
+def lld_main(lld_args) -> str:
+    """
+    Calls LLD - the LLVM linker.
+
+    The function calls the `lld::elf::link()` with the `lld_args` (list of
+    strings) as arguments.
+
+    Examples
+    ~~~~~~~~
+
+    Shows help:
+
+    >>> from llvmlite.binding import lld_main
+    >>> lld_main(["ld.lld", "--help"])
+    """
+    args = (c_char_p * len(lld_args))()
+    for i, arg in enumerate(lld_args):
+        args[i] = arg.encode()
+    with ffi.OutputString() as outstr:
+        r = ffi.lib.LLVMPY_lld_main(len(lld_args), args, outstr)
+        if r:
+            raise Exception("lld_main() failed, error code: %d\
+                            \nCommand Output: %s" % (r, str(outstr)))
+        return str(outstr)
+
+
+def lld_runner(command: str):
+    def wrapped(output: str, objects: List[str], args: List[str] = []) -> str:
+        '''
+        runs the command:
+        "{platform's lld command} -o {output-file} {*input_files} {*args}"
+        output: output file as a str
+        object: a list of input .o files as strings
+        args: additional arguments for the command
+        '''
+        return lld_main([command, '-o', output, *objects, *args])
+    return wrapped
+
+
+lld_windows = lld_runner("lld-link")
+lld_macos = lld_runner("ld64.lld")
+lld_linux = lld_runner("ld.lld")
+lld_wasm = lld_runner("wasm-ld")
+
+
+def lld_auto(output: str, objects: List[str],
+             args: List[str] = [],
+             add_extension=True) -> str:
+    '''
+    Automatically determines which lld function
+    to run based on the hosts system.
+
+    Does not use `lld_wasm()`
+
+    add_extension: adds `.exe` and other file endings automatically
+    '''
+    system = platform.system()
+
+    if system == "Linux":
+        # has no standard file extension
+        return lld_linux(output, objects, args)
+    elif system == "Windows":
+        name = output + ('.exe' * add_extension)
+        return lld_windows(name, objects, args)
+    elif system == "Darwin": # Macos
+        name = output + ('.app' * add_extension)
+        return lld_macos(name, objects, args)
+    else:
+        msg = (f'''Invalid host system name: {system}
+                If you are sure a driver exists for this system,
+                please use the `lld_main()` function
+                '''.strip())
+        raise RuntimeError(msg)