From f5b9f4ea19ff1578af49106070ddc8485dcd7d96 Mon Sep 17 00:00:00 2001 From: gerioldman Date: Wed, 28 Aug 2024 02:57:56 +0200 Subject: [PATCH] First draft version of Tasking MIL linking with b_lto and prelinking --- mesonbuild/backend/backends.py | 11 +++-- mesonbuild/backend/ninjabackend.py | 62 +++++++------------------- mesonbuild/backend/vs2010backend.py | 4 +- mesonbuild/build.py | 5 ++- mesonbuild/compilers/compilers.py | 17 ++++--- mesonbuild/compilers/mixins/apple.py | 5 ++- mesonbuild/compilers/mixins/elbrus.py | 5 ++- mesonbuild/compilers/mixins/gnu.py | 5 ++- mesonbuild/compilers/mixins/tasking.py | 37 ++++++++++++--- mesonbuild/linkers/linkers.py | 3 ++ 10 files changed, 83 insertions(+), 71 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index e35660b69a51..09ece046d72d 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -25,6 +25,7 @@ from .. import mesonlib from .. import mlog from ..compilers import LANGUAGES_USING_LDFLAGS, detect +from ..utils.universal import get_compiler_for_source from ..mesonlib import ( File, MachineChoice, MesonException, OrderedSet, ExecutableSerialisation, classify_unity_sources, @@ -836,7 +837,7 @@ def canonicalize_filename(fname: str) -> str: fname = fname.replace(ch, '_') return hashed + fname - def object_filename_from_source(self, target: build.BuildTarget, source: 'FileOrString', targetdir: T.Optional[str] = None) -> str: + def object_filename_from_source(self, target: build.BuildTarget, compiler: Compiler, source: 'FileOrString', targetdir: T.Optional[str] = None) -> str: assert isinstance(source, mesonlib.File) if isinstance(target, build.CompileTarget): return target.sources_map[source] @@ -867,7 +868,10 @@ def object_filename_from_source(self, target: build.BuildTarget, source: 'FileOr gen_source = os.path.relpath(os.path.join(build_dir, rel_src), os.path.join(self.environment.get_source_dir(), target.get_subdir())) machine = self.environment.machines[target.for_machine] - ret = self.canonicalize_filename(gen_source) + '.' + machine.get_object_suffix() + object_suffix = machine.get_object_suffix() + object_suffix_override = compiler.get_object_suffix_override(target, source) + object_suffix = object_suffix if object_suffix_override is None else object_suffix_override + ret = self.canonicalize_filename(gen_source) + '.' + object_suffix if targetdir is not None: return os.path.join(targetdir, ret) return ret @@ -924,7 +928,8 @@ def _determine_ext_objs(self, extobj: 'build.ExtractedObjects', proj_dir_to_buil sources.append(_src) for osrc in sources: - objname = self.object_filename_from_source(extobj.target, osrc, targetdir) + compiler = get_compiler_for_source(extobj.target.compilers.values(), osrc) + objname = self.object_filename_from_source(extobj.target, compiler, osrc, targetdir) objpath = os.path.join(proj_dir_to_build_root, objname) result.append(objpath) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index a3f9d8e4e71a..d9ae02d72d4c 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -1080,19 +1080,11 @@ def generate_target(self, target) -> None: # Skip the link stage for this special type of target return linker, stdlib_args = self.determine_linker_and_stdlib_args(target) - # For prelinking and TASKING mil linking there needs to be an additional link target and the object list is modified + if not isinstance(target, build.StaticLibrary): final_obj_list = obj_list elif target.prelink: final_obj_list = self.generate_prelink(target, obj_list) - elif 'c' in target.compilers: - key = OptionKey('b_tasking_mil_link') - if key not in target.get_options() or key not in target.compilers['c'].base_options: - final_obj_list = obj_list - elif target.get_option(key): - final_obj_list = self.generate_mil_link(target, obj_list) - else: - final_obj_list = obj_list else: final_obj_list = obj_list elem = self.generate_link(target, outname, final_obj_list, linker, pch_objects, stdlib_args=stdlib_args) @@ -2613,9 +2605,8 @@ def generate_compile_rules(self) -> None: for langname, compiler in clist.items(): if compiler.get_id() == 'clang': self.generate_llvm_ir_compile_rule(compiler) - if OptionKey('b_tasking_mil_link') in compiler.base_options: + if compiler.get_id() in {'cctc', 'ccarm', 'cc51', 'ccmcs', 'ccpcp'}: self.generate_tasking_mil_compile_rules(compiler) - self.generate_tasking_mil_link_rules(compiler) self.generate_compile_rule_for(langname, compiler) self.generate_pch_rule_for(langname, compiler) for mode in compiler.get_modes(): @@ -3057,12 +3048,7 @@ def generate_single_compile(self, target: build.BuildTarget, src, raise AssertionError(f'BUG: broken generated source file handling for {src!r}') else: raise InvalidArguments(f'Invalid source type: {src!r}') - obj_basename = self.object_filename_from_source(target, src) - # If mil linking is enabled for the target, then compilation output has to be MIL files instead of object files - if compiler.get_language() == 'c': - key = OptionKey('b_tasking_mil_link') - if key in compiler.base_options and target.get_option(key) and src.rsplit('.', 1)[1] in compilers.lang_suffixes['c']: - obj_basename = f'{os.path.splitext(obj_basename)[0]}.mil' + obj_basename = self.object_filename_from_source(target, compiler, src) rel_obj = os.path.join(self.get_target_private_dir(target), obj_basename) dep_file = compiler.depfile_for_object(rel_obj) @@ -3086,9 +3072,12 @@ def generate_single_compile(self, target: build.BuildTarget, src, # If TASKING compiler family is used and MIL linking is enabled for the target, # then compilation rule name is a special one to output MIL files # instead of object files for .c files - key = OptionKey('b_tasking_mil_link') - if key in compiler.base_options and target.get_option(key) and src.rsplit('.', 1)[1] in compilers.lang_suffixes['c']: - compiler_name = self.get_compiler_rule_name('tasking_mil_compile', compiler.for_machine) + key = OptionKey('b_lto') + if compiler.get_id() in {'cctc', 'ccarm', 'cc51', 'ccmcs', 'ccpcp'}: + if ((isinstance(target, build.StaticLibrary) and target.prelink) or target.get_option(key)) and src.rsplit('.', 1)[1] in compilers.lang_suffixes['c']: + compiler_name = self.get_compiler_rule_name('tasking_mil_compile', compiler.for_machine) + else: + compiler_name = self.compiler_to_rule_name(compiler) else: compiler_name = self.compiler_to_rule_name(compiler) extra_deps = [] @@ -3467,36 +3456,19 @@ def generate_prelink(self, target, obj_list): prelinker = target.get_prelinker() cmd = prelinker.exelist[:] - cmd += prelinker.get_prelink_args(prelink_name, obj_list) + obj_list, args = prelinker.get_prelink_args(target, prelink_name, obj_list) + cmd += args + if prelinker.get_prelink_append_compile_args(): + compile_args = self._generate_single_compile_base_args(target, prelinker) + compile_args += self._generate_single_compile_target_args(target, prelinker) + compile_args = compile_args.compiler.compiler_args(compile_args) + cmd += compile_args.to_native() cmd = self.replace_paths(target, cmd) elem.add_item('COMMAND', cmd) elem.add_item('description', f'Prelinking {prelink_name}.') self.add_build(elem) - return [prelink_name] - - def generate_mil_link(self, target: build.StaticLibrary, obj_list: T.List[str]) -> T.List[str]: - assert isinstance(target, build.StaticLibrary) - - mil_linked_name = os.path.join(self.get_target_private_dir(target), target.name + '-mil_link.o') - mil_link_list = [] - obj_file_list = [] - for obj in obj_list: - if obj.endswith('.mil'): - mil_link_list.append(obj) - else: - obj_file_list.append(obj) - obj_file_list.append(mil_linked_name) - compiler = get_compiler_for_source(target.compilers.values(), mil_link_list[0][:-3] + '.c') - commands = self._generate_single_compile_base_args(target, compiler) - commands += self._generate_single_compile_target_args(target, compiler) - commands = commands.compiler.compiler_args(commands) - - elem = NinjaBuildElement(self.all_outputs, [mil_linked_name], self.get_compiler_rule_name('tasking_mil_link', compiler.for_machine), mil_link_list) - elem.add_item('ARGS', commands) - self.add_build(elem) - - return obj_file_list + return obj_list def generate_link(self, target: build.BuildTarget, outname, obj_list, linker: T.Union['Compiler', 'StaticLinker'], extra_args=None, stdlib_args=None): extra_args = extra_args if extra_args is not None else [] diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index a12963cdee06..d9019b3e7332 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -1725,7 +1725,7 @@ def path_normalize_add(path, lis): self.add_preprocessor_defines(lang, inc_cl, file_defines) self.add_include_dirs(lang, inc_cl, file_inc_dirs) ET.SubElement(inc_cl, 'ObjectFileName').text = "$(IntDir)" + \ - self.object_filename_from_source(target, s) + self.object_filename_from_source(target, compiler, s) for s in gen_src: if path_normalize_add(s, previous_sources): inc_cl = ET.SubElement(inc_src, 'CLCompile', Include=s) @@ -1739,7 +1739,7 @@ def path_normalize_add(path, lis): self.add_include_dirs(lang, inc_cl, file_inc_dirs) s = File.from_built_file(target.get_subdir(), s) ET.SubElement(inc_cl, 'ObjectFileName').text = "$(IntDir)" + \ - self.object_filename_from_source(target, s) + self.object_filename_from_source(target, compiler, s) for lang, headers in pch_sources.items(): impl = headers[1] if impl and path_normalize_add(impl, previous_sources): diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 6dacd206ba44..3b9adccdc5ec 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -2160,7 +2160,10 @@ def post_init(self) -> None: elif self.rust_crate_type == 'staticlib': self.suffix = 'a' else: - self.suffix = 'a' + if 'c' in self.compilers and self.compilers['c'].get_id() in {'cctc', 'ccarm', 'cc51', 'ccmcs', 'ccpcp'}: + self.suffix = 'ma' if self.options.get_value('b_lto') and not self.prelink else 'a' + else: + self.suffix = 'a' self.filename = self.prefix + self.name + '.' + self.suffix self.outputs[0] = self.filename diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 39d1ee7054aa..cd422f449484 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -247,7 +247,6 @@ def init_option(self, name: OptionKey) -> options._U: OptionKey('b_bitcode'): BaseOption(options.UserBooleanOption, 'Generate and embed bitcode (only macOS/iOS/tvOS)', False), OptionKey('b_vscrt'): BaseOption(options.UserComboOption, 'VS run-time library type to use.', 'from_buildtype', choices=MSCRT_VALS + ['from_buildtype', 'static_from_buildtype']), - OptionKey('b_tasking_mil_link'): BaseOption(options.UserBooleanOption, 'Use TASKING compiler families MIL linking feature', False), } base_options = {key: base_opt.init_option(key) for key, base_opt in BASE_OPTIONS.items()} @@ -429,8 +428,8 @@ class CompileResult(HoldableObject): output_name: T.Optional[str] = field(default=None, init=False) cached: bool = field(default=False, init=False) - class Compiler(HoldableObject, metaclass=abc.ABCMeta): + # Libraries to ignore in find_library() since they are provided by the # compiler or the C library. Currently only used for MSVC. ignore_libs: T.List[str] = [] @@ -1325,9 +1324,13 @@ def get_feature_args(self, kwargs: DFeatures, build_to_src: str) -> T.List[str]: # TODO: using a TypeDict here would improve this raise EnvironmentException(f'{self.id} does not implement get_feature_args') - def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: + def get_prelink_args(self, target: BuildTarget, prelink_name: str, obj_list: T.List[str]) -> T.Tuple[T.List[str], T.List[str]]: raise EnvironmentException(f'{self.id} does not know how to do prelinking.') + def get_prelink_append_compile_args(self) -> bool: + """Controls whether compile args have to be used for prelinking or not""" + return False + def rsp_file_syntax(self) -> 'RSPFileSyntax': """The format of the RSP file that this compiler supports. @@ -1351,12 +1354,8 @@ def get_preprocessor(self) -> Compiler: def form_compileropt_key(self, basename: str) -> OptionKey: return OptionKey(f'{self.language}_{basename}', machine=self.for_machine) - def get_tasking_mil_link_args(self, option_enabled: bool) -> T.List[str]: - """ - Argument for enabling TASKING's MIL link feature, - for most compilers, this will return nothing. - """ - return [] + def get_object_suffix_override(self, target: BuildTarget, source: mesonlib.File) -> T.Optional[str]: + return None def get_global_options(lang: str, comp: T.Type[Compiler], diff --git a/mesonbuild/compilers/mixins/apple.py b/mesonbuild/compilers/mixins/apple.py index fc93d38a5673..ca1b50a37475 100644 --- a/mesonbuild/compilers/mixins/apple.py +++ b/mesonbuild/compilers/mixins/apple.py @@ -9,6 +9,7 @@ from ...mesonlib import MesonException if T.TYPE_CHECKING: + from ...build import BuildTarget from ..._typing import ImmutableListProtocol from ...environment import Environment from ..compilers import Compiler @@ -56,6 +57,6 @@ def openmp_link_flags(self, env: Environment) -> T.List[str]: raise MesonException("Couldn't find libomp") return self.__BASE_OMP_FLAGS + link - def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: + def get_prelink_args(self, target: BuildTarget, prelink_name: str, obj_list: T.List[str]) -> T.Tuple[T.List[str], T.List[str]]: # The objects are prelinked through the compiler, which injects -lSystem - return ['-nostdlib', '-r', '-o', prelink_name] + obj_list + return [prelink_name], ['-nostdlib', '-r', '-o', prelink_name] + obj_list diff --git a/mesonbuild/compilers/mixins/elbrus.py b/mesonbuild/compilers/mixins/elbrus.py index 66f419cf02d8..464756e1b57c 100644 --- a/mesonbuild/compilers/mixins/elbrus.py +++ b/mesonbuild/compilers/mixins/elbrus.py @@ -15,6 +15,7 @@ from .gnu import gnu_optimization_args from ...mesonlib import Popen_safe from ...options import OptionKey +from ...build import BuildTarget if T.TYPE_CHECKING: from ...environment import Environment @@ -76,8 +77,8 @@ def get_default_include_dirs(self) -> T.List[str]: def get_optimization_args(self, optimization_level: str) -> T.List[str]: return gnu_optimization_args[optimization_level] - def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: - return ['-r', '-nodefaultlibs', '-nostartfiles', '-o', prelink_name] + obj_list + def get_prelink_args(self, target: BuildTarget, prelink_name: str, obj_list: T.List[str]) -> T.Tuple[T.List[str], T.List[str]]: + return [prelink_name], ['-r', '-nodefaultlibs', '-nostartfiles', '-o', prelink_name] + obj_list def get_pch_suffix(self) -> str: # Actually it's not supported for now, but probably will be supported in future diff --git a/mesonbuild/compilers/mixins/gnu.py b/mesonbuild/compilers/mixins/gnu.py index 62f55543a0a7..ebacbeb9e6d8 100644 --- a/mesonbuild/compilers/mixins/gnu.py +++ b/mesonbuild/compilers/mixins/gnu.py @@ -17,6 +17,7 @@ from ... import mesonlib from ... import mlog from ...options import OptionKey +from ...build import BuildTarget from mesonbuild.compilers.compilers import CompileCheckMode if T.TYPE_CHECKING: @@ -607,8 +608,8 @@ def get_has_func_attribute_extra_args(self, name: str) -> T.List[str]: # error. return ['-Werror=attributes'] - def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: - return ['-r', '-o', prelink_name] + obj_list + def get_prelink_args(self, target: BuildTarget, prelink_name: str, obj_list: T.List[str]) -> T.Tuple[T.List[str], T.List[str]]: + return [prelink_name], ['-r', '-o', prelink_name] + obj_list def get_lto_compile_args(self, *, threads: int = 0, mode: str = 'default') -> T.List[str]: if threads == 0: diff --git a/mesonbuild/compilers/mixins/tasking.py b/mesonbuild/compilers/mixins/tasking.py index 7675dbc75329..78db6e8a15a4 100644 --- a/mesonbuild/compilers/mixins/tasking.py +++ b/mesonbuild/compilers/mixins/tasking.py @@ -7,8 +7,10 @@ import os import typing as T -from ...mesonlib import EnvironmentException +from ...mesonlib import EnvironmentException, File from ...options import OptionKey +from ...build import BuildTarget, StaticLibrary +from ...compilers import lang_suffixes if T.TYPE_CHECKING: from ...compilers.compilers import Compiler @@ -56,7 +58,7 @@ def __init__(self) -> None: self.base_options = { OptionKey(o) for o in [ - 'b_tasking_mil_link', + 'b_lto', 'b_staticpic', 'b_ndebug' ] @@ -112,6 +114,21 @@ def get_optimization_args(self, optimization_level: str) -> T.List[str]: def get_no_optimization_args(self) -> T.List[str]: return ['-O0'] + def get_prelink_args(self, target: BuildTarget, prelink_name: str, obj_list: T.List[str]) -> T.Tuple[T.List[str], T.List[str]]: + mil_link_list = [] + obj_file_list = [] + for obj in obj_list: + if obj.endswith('.mil'): + mil_link_list.append(obj) + else: + obj_file_list.append(obj) + obj_file_list.append(prelink_name) + + return obj_file_list, ['--mil-link', '-o', prelink_name, '-c'] + mil_link_list + + def get_prelink_append_compile_args(self) -> bool: + return True + def compute_parameters_with_absolute_paths(self, parameter_list: T.List[str], build_dir: str) -> T.List[str]: for idx, i in enumerate(parameter_list): if i[:2] == '-I' or i[:2] == '-L': @@ -119,8 +136,18 @@ def compute_parameters_with_absolute_paths(self, parameter_list: T.List[str], bu return parameter_list - def get_tasking_mil_link_args(self, option_enabled: bool) -> T.List[str]: - return ['--mil-link'] if option_enabled else [] - def get_preprocess_only_args(self) -> T.List[str]: return ['-E'] + + # This is required for MIL linking, if b_lto is enabled on the target + def get_object_suffix_override(self, target: BuildTarget, source: File) -> T.Optional[str]: + if target.get_option(OptionKey('b_lto')) or (isinstance(target, StaticLibrary) and target.prelink): + if not source.rsplit('.', 1)[1] in lang_suffixes['c']: + if isinstance(target, StaticLibrary) and not target.prelink: + raise EnvironmentException('Tried using MIL linking for a static library with a assembly file. This can only be done if the static library is prelinked or disable \'b_lto\'.') + else: + return None + else: + return 'mil' + else: + return None diff --git a/mesonbuild/linkers/linkers.py b/mesonbuild/linkers/linkers.py index 588b66369878..74696292b67e 100644 --- a/mesonbuild/linkers/linkers.py +++ b/mesonbuild/linkers/linkers.py @@ -1721,6 +1721,9 @@ def get_search_args(self, dirname: str) -> T.List[str]: def get_output_args(self, outputname: str) -> T.List[str]: return ['-o', outputname] + def get_lto_args(self) -> T.List[str]: + return ['--mil-link'] + def rsp_file_syntax(self) -> RSPFileSyntax: return RSPFileSyntax.TASKING