Add support for importing symbols from static library

docs/source/user-guide/binding/execution-engine.rst

@@ -79,6 +79,13 @@ The ExecutionEngine class
             or a object file instance. Object file instance is not usable after this
             call.
 
+   * .. method:: add_archive(archive_file)
+
+        Add the symbols from the specified static archive file to the execution
+        engine. It is a fatal error in LLVM if the *archive_file* does not exist.
+
+        * *archive_file* str: a path to the static archive file
+
    * .. method:: set_object_cache(notify_func=None, getbuffer_func=None)
 
         Set the object cache callbacks for this engine.

ffi/executionengine.cpp

@@ -7,10 +7,12 @@
 #include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/ObjectCache.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/Memory.h"
-
+#include "llvm/Support/MemoryBuffer.h"
 #include <cstdio>
 #include <memory>
 
@@ -167,6 +169,42 @@ LLVMPY_MCJITAddObjectFile(LLVMExecutionEngineRef EE, LLVMObjectFileRef ObjF) {
         {std::move(binary_tuple.first), std::move(binary_tuple.second)});
 }
 
+API_EXPORT(int)
+LLVMPY_MCJITAddArchive(LLVMExecutionEngineRef EE, const char *ArchiveName,
+                       const char **OutError) {
+    using namespace llvm;
+    using namespace llvm::object;
+    auto engine = unwrap(EE);
+
+    ErrorOr<std::unique_ptr<MemoryBuffer>> ArBufOrErr =
+        MemoryBuffer::getFile(ArchiveName);
+
+    std::error_code EC = ArBufOrErr.getError();
+    if (EC) {
+        *OutError = LLVMPY_CreateString(EC.message().c_str());
+        return 1;
+    }
+
+    Expected<std::unique_ptr<object::Archive>> ArchiveOrError =
+        object::Archive::create(ArBufOrErr.get()->getMemBufferRef());
+
+    if (!ArchiveOrError) {
+        auto takeErr = ArchiveOrError.takeError();
+        std::string archiveErrStr =
+            "Unable to load archive: " + std::string(ArchiveName);
+        *OutError = LLVMPY_CreateString(archiveErrStr.c_str());
+        return 1;
+    }
+
+    std::unique_ptr<MemoryBuffer> &ArBuf = ArBufOrErr.get();
+    std::unique_ptr<object::Archive> &Ar = ArchiveOrError.get();
+    object::OwningBinary<object::Archive> owningBinaryArchive(std::move(Ar),
+                                                              std::move(ArBuf));
+    engine->addArchive(std::move(owningBinaryArchive));
+    *OutError = LLVMPY_CreateString("");
+    return 0;
+}
+
 //
 // Object cache
 //

llvmlite/binding/executionengine.py

@@ -155,6 +155,18 @@ def add_object_file(self, obj_file):
 
         ffi.lib.LLVMPY_MCJITAddObjectFile(self, obj_file)
 
+    def add_archive(self, archive_file):
+        """
+        Add archive file to the jit. archive_file is a string
+        representing the file system path to the archive file.
+        """
+
+        with ffi.OutputString() as outerr:
+            res = ffi.lib.LLVMPY_MCJITAddArchive(self, archive_file.encode(),
+                                                 outerr)
+            if res:
+                raise RuntimeError(str(outerr))
+
     def set_object_cache(self, notify_func=None, getbuffer_func=None):
         """
         Set the object cache "notifyObjectCompiled" and "getBuffer"
@@ -286,6 +298,14 @@ def _dispose(self):
     ffi.LLVMObjectFileRef
 ]
 
+ffi.lib.LLVMPY_MCJITAddArchive.argtypes = [
+    ffi.LLVMExecutionEngineRef,
+    c_char_p,
+    POINTER(c_char_p)
+]
+
+ffi.lib.LLVMPY_MCJITAddArchive.restype = c_int
+
 
 class _ObjectCacheData(Structure):
     _fields_ = [

llvmlite/tests/a.c

@@ -0,0 +1,5 @@
+
+int __multiply_accumulate(int a, int b, int c)
+{
+  return (a * b) + c;
+}

llvmlite/tests/b.c

@@ -0,0 +1,4 @@
+int __multiply_subtract(int a, int b, int c)
+{
+  return (a * b) - c;
+}

llvmlite/tests/test_binding.py

@@ -10,12 +10,58 @@
 import sys
 import unittest
 from contextlib import contextmanager
-from tempfile import mkstemp
+from tempfile import mkstemp, TemporaryDirectory, mkdtemp
+from pathlib import Path
 
 from llvmlite import ir
 from llvmlite import binding as llvm
 from llvmlite.binding import ffi
 from llvmlite.tests import TestCase
+import llvmlite.tests
+from setuptools._distutils.ccompiler import new_compiler
+from setuptools._distutils.sysconfig import customize_compiler
+import functools
+
+
+@contextmanager
+def _gentmpfile(suffix):
+    # windows locks the tempfile so use a tempdir + file, see
+    # https://github.com/numba/numba/issues/3304
+    try:
+        tmpdir = mkdtemp()
+        ntf = open(os.path.join(tmpdir, "temp%s" % suffix), 'wt')
+        yield ntf
+    finally:
+        try:
+            ntf.close()
+            os.remove(ntf)
+        except Exception:
+            pass
+        else:
+            os.rmdir(tmpdir)
+
+
+@functools.lru_cache(maxsize=1)
+def external_compiler_works():
+    """
+    Returns True if the "external compiler" bound in setuptools._distutil
+    is present and working, False otherwise.
+    """
+    compiler = new_compiler()
+    customize_compiler(compiler)
+    for suffix in ['.c', '.cxx']:
+        try:
+            with _gentmpfile(suffix) as ntf:
+                simple_c = "int main(void) { return 0; }"
+                ntf.write(simple_c)
+                ntf.flush()
+                ntf.close()
+                # *output_dir* is set to avoid the compiler putting temp files
+                # in the current directory.
+                compiler.compile([ntf.name], output_dir=Path(ntf.name).anchor)
+        except Exception: # likely CompileError or file system issue
+            return False
+    return True
 
 
 # arvm7l needs extra ABI symbols to link successfully
@@ -1935,6 +1981,60 @@ def test_get_section_content(self):
                 self.assertEqual(s.data().hex(), issue_632_text)
 
 
+class TestArchiveFile(BaseTest):
+    mod_archive_asm = """
+        ;ModuleID = <string>
+        target triple = "{triple}"
+
+        declare i32 @__multiply_accumulate(i32 %0, i32 %1, i32 %2)
+        declare i32 @__multiply_subtract(i32 %0, i32 %1, i32 %2)
+    """
+
+    @unittest.skipUnless(sys.platform.startswith('linux'),
+                         "Linux-specific test")
+    def test_add_archive(self):
+        if not external_compiler_works():
+            self.skipTest()
+
+        tmpdir = TemporaryDirectory()
+        try:
+            target_machine = self.target_machine(jit=False)
+
+            jit = llvm.create_mcjit_compiler(self.module(self.mod_archive_asm),
+                                             target_machine)
+
+            # Create compiler with default options
+            c = new_compiler()
+            customize_compiler(c)
+            workdir = tmpdir.name
+            srcdir = os.path.dirname(llvmlite.tests.__file__) + "/"
+
+            # Compile into .o files
+            file1 = srcdir + "a.c"
+            file2 = srcdir + "b.c"
+
+            objects = c.compile([file1, file2], output_dir=workdir)
+            library_name = "foo"
+
+            c.create_static_lib(objects, library_name, output_dir=workdir)
+
+            static_library_name = workdir + "/" + "lib" + library_name + ".a"
+            jit.add_archive(static_library_name)
+
+            mac_func = CFUNCTYPE(c_int, c_int, c_int, c_int)(
+                jit.get_function_address("__multiply_accumulate"))
+
+            msub_func = CFUNCTYPE(c_int, c_int, c_int, c_int)(
+                jit.get_function_address("__multiply_subtract"))
+
+            self.assertEqual(mac_func(10, 10, 20), 120)
+            self.assertEqual(msub_func(10, 10, 20), 80)
+        finally:
+            # Manually invoke cleanup to remove tmpdir, objects
+            # and static library created during test
+            tmpdir.cleanup()
+
+
 class TestTimePasses(BaseTest):
     def test_reporting(self):
         mp = llvm.create_module_pass_manager()

setup.py

@@ -100,6 +100,7 @@ def run(self):
         from llvmlite.utils import get_library_files
         self.distribution.package_data = {
             "llvmlite.binding": get_library_files(),
+            "llvmlite.tests": ["*.c"]
         }
 
 
@@ -110,6 +111,7 @@ def run(self):
         from llvmlite.utils import get_library_files
         self.distribution.package_data = {
             "llvmlite.binding": get_library_files(),
+            "llvmlite.tests": ["*.c"]
         }
         install.run(self)
 
@@ -155,6 +157,7 @@ def run(self):
             build_library_files(self.dry_run)
             self.distribution.package_data.update({
                 "llvmlite.binding": get_library_files(),
+                "llvmlite.tests": ["*.c"]
             })
             # Run wheel build command
             bdist_wheel.run(self)