From c71736a3be11947c242e899d42befc5aecf037da Mon Sep 17 00:00:00 2001 From: Rob Moran Date: Mon, 21 Aug 2017 19:38:45 +0100 Subject: [PATCH 01/54] output folder WiP --- yotta/build.py | 17 ++++++++++++++++- yotta/debug.py | 7 ++++++- yotta/lib/cmakegen.py | 2 +- yotta/options/config.py | 2 +- yotta/start.py | 7 ++++++- yotta/test_subcommand.py | 6 +++--- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index 38d95e83..90dd7956 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -16,6 +16,8 @@ from yotta.lib import cmakegen # Target, , represents an installed target, internal from yotta.lib import target +# settings, , load and save settings, internal +from yotta.lib import settings # install, , install subcommand, internal from yotta import install # --config option, , , internal @@ -27,6 +29,11 @@ def addOptions(parser, add_build_targets=True): action='store_true', default=False, help='Only generate CMakeLists, don\'t run CMake or build' ) + parser.add_argument('-o', '--output-folder', dest='output_folder', + default=None, + help='Specify a build output folder instead of ./build/', + metavar='path/to/output/folder' + ) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) parser.add_argument('-d', '--debug-build', dest='release_build', action='store_false', default=True) # the target class adds its own build-system specific options. In the @@ -43,6 +50,11 @@ def addOptions(parser, add_build_targets=True): ) def execCommand(args, following_args): + if args.output_folder: + settings.setProperty('build', 'folder', os.path.abspath(args.output_folder), True) + elif settings.getProperty('build', 'folder'): + settings.setProperty('build', 'folder', '', True) + status = installAndBuild(args, following_args) return status['status'] @@ -101,7 +113,10 @@ def installAndBuild(args, following_args): # version specs), which it will display install_status = install.execCommand(args, []) - builddir = os.path.join(cwd, 'build', target.getName()) + if settings.getProperty('build', 'folder'): + builddir = settings.getProperty('build', 'folder') + else: + builddir = os.path.join(cwd, 'build', target.getName()) all_deps = c.getDependenciesRecursive( target = target, diff --git a/yotta/debug.py b/yotta/debug.py index a84d617c..ac156730 100644 --- a/yotta/debug.py +++ b/yotta/debug.py @@ -9,6 +9,8 @@ # validate, , validate things, internal from yotta.lib import validate +# settings, , load and save settings, internal +from yotta.lib import settings # --config option, , , internal from yotta import options @@ -33,7 +35,10 @@ def execCommand(args, following_args): logging.error(error) return 1 - builddir = os.path.join(cwd, 'build', target.getName()) + if settings.getProperty('build', 'folder'): + builddir = settings.getProperty('build', 'folder') + else: + builddir = os.path.join(cwd, 'build', target.getName()) if args.program is None: if c.isApplication(): diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 0f750d44..66c9d897 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -112,7 +112,7 @@ def generateRecursive(self, component, all_components, builddir=None, modbuilddi if builddir is None: builddir = self.buildroot if modbuilddir is None: - modbuilddir = os.path.join(builddir, 'ym') + modbuilddir = os.path.join(builddir, 'modules') if processed_components is None: processed_components = dict() diff --git a/yotta/options/config.py b/yotta/options/config.py index c8e3ab28..3d79b86c 100644 --- a/yotta/options/config.py +++ b/yotta/options/config.py @@ -26,7 +26,7 @@ def __call__(self, parser, namespace, values, option_string=None): def addTo(parser): parser.add_argument( - '--config', default=None, dest='config', help= + '-c', '--config', default=None, dest='config', help= "Specify the path to a JSON configuration file to extend the build "+ "configuration provided by the target. This is most useful for "+ "ensuring test coverage of the ways that different targets will "+ diff --git a/yotta/start.py b/yotta/start.py index a12e1d6d..cd487b9e 100644 --- a/yotta/start.py +++ b/yotta/start.py @@ -9,6 +9,8 @@ # validate, , validate things, internal from yotta.lib import validate +# settings, , load and save settings, internal +from yotta.lib import settings # --config option, , , internal from yotta import options @@ -37,7 +39,10 @@ def execCommand(args, following_args): logging.error(error) return 1 - builddir = os.path.join(cwd, 'build', target.getName()) + if settings.getProperty('build', 'folder'): + builddir = settings.getProperty('build', 'folder') + else: + builddir = os.path.join(cwd, 'build', target.getName()) if args.program is None: # if no program was specified, default to the name of the executable diff --git a/yotta/test_subcommand.py b/yotta/test_subcommand.py index 34a8b4b9..025558f7 100644 --- a/yotta/test_subcommand.py +++ b/yotta/test_subcommand.py @@ -51,7 +51,7 @@ def findCTests(builddir, recurse_yotta_modules=False): add_test_re = re.compile('add_test\\(([^" ]*)\s*"(.*)"\\)', flags=re.IGNORECASE) for root, dirs, files in os.walk(builddir, topdown=True): if not recurse_yotta_modules: - dirs = [d for d in dirs if d != 'ym'] + dirs = [d for d in dirs if d != 'modules'] if 'CTestTestfile.cmake' in files: with open(os.path.join(root, 'CTestTestfile.cmake'), 'r') as ctestf: dir_tests = [] @@ -73,7 +73,7 @@ def moduleFromDirname(build_subdir, all_modules, toplevel_module): modtop = True submod = False module = toplevel_module - # /ym//ym//somedir/somedir --> submod2 + # /modules//modules//somedir/somedir --> submod2 for part in fsutils.fullySplitPath(build_subdir): if submod: if part in all_modules: @@ -81,7 +81,7 @@ def moduleFromDirname(build_subdir, all_modules, toplevel_module): modtop = True submod = False else: - if part == 'ym' and modtop: + if part == 'modules' and modtop: submod = True else: submod = False From 37a8d3a664ff03466c9c9d4295991d52991c6cea Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 23 Aug 2017 14:34:32 +0100 Subject: [PATCH 02/54] gitignore idea --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3e142a47..e596b33b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ dist git-access-testing github-access-testing *~ +*.idea From 79d208fa4f038c805f56f6904f2ae2b48c964780 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 23 Aug 2017 15:59:08 +0100 Subject: [PATCH 03/54] common path manipulation tools --- yotta/build.py | 18 ++++-------------- yotta/clean.py | 23 ++++++++++++++++++----- yotta/debug.py | 8 ++++---- yotta/lib/paths.py | 41 +++++++++++++++++++++++++++++++++++++++++ yotta/start.py | 8 ++++---- 5 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 yotta/lib/paths.py diff --git a/yotta/build.py b/yotta/build.py index 90dd7956..4980f559 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -18,6 +18,8 @@ from yotta.lib import target # settings, , load and save settings, internal from yotta.lib import settings +# paths +from yotta.lib import paths # install, , install subcommand, internal from yotta import install # --config option, , , internal @@ -29,11 +31,7 @@ def addOptions(parser, add_build_targets=True): action='store_true', default=False, help='Only generate CMakeLists, don\'t run CMake or build' ) - parser.add_argument('-o', '--output-folder', dest='output_folder', - default=None, - help='Specify a build output folder instead of ./build/', - metavar='path/to/output/folder' - ) + paths.add_parser_argument(parser) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) parser.add_argument('-d', '--debug-build', dest='release_build', action='store_false', default=True) # the target class adds its own build-system specific options. In the @@ -50,11 +48,6 @@ def addOptions(parser, add_build_targets=True): ) def execCommand(args, following_args): - if args.output_folder: - settings.setProperty('build', 'folder', os.path.abspath(args.output_folder), True) - elif settings.getProperty('build', 'folder'): - settings.setProperty('build', 'folder', '', True) - status = installAndBuild(args, following_args) return status['status'] @@ -113,10 +106,7 @@ def installAndBuild(args, following_args): # version specs), which it will display install_status = install.execCommand(args, []) - if settings.getProperty('build', 'folder'): - builddir = settings.getProperty('build', 'folder') - else: - builddir = os.path.join(cwd, 'build', target.getName()) + builddir = paths.get_configured_output_path(args, target) all_deps = c.getDependenciesRecursive( target = target, diff --git a/yotta/clean.py b/yotta/clean.py index 6d3ca2a5..0a1df9e5 100644 --- a/yotta/clean.py +++ b/yotta/clean.py @@ -12,13 +12,26 @@ # fsutils, , misc filesystem utils, internal from yotta.lib import fsutils +# paths +from yotta.lib import paths + + def addOptions(parser): - pass + paths.add_parser_argument(parser) + def execCommand(args, following_args): - c = validate.currentDirectoryModule() - if not c: - return 1 + paths_to_remove = [] + + current = validate.currentDirectoryModule() + if current: + paths_to_remove.append(os.path.join(current.path, 'build')) - fsutils.rmRf(os.path.join(c.path, 'build')) + specific = paths.get_configured_output_path(args) + if specific: + paths_to_remove.append(specific) + for path in paths_to_remove: + if os.path.exists(path) and not validate.directoryModule(path): + print('removing: %s' % path) + fsutils.rmRf(path) diff --git a/yotta/debug.py b/yotta/debug.py index ac156730..170bc320 100644 --- a/yotta/debug.py +++ b/yotta/debug.py @@ -11,6 +11,8 @@ from yotta.lib import validate # settings, , load and save settings, internal from yotta.lib import settings +# paths +from yotta.lib import paths # --config option, , , internal from yotta import options @@ -20,6 +22,7 @@ def addOptions(parser): parser.add_argument('program', default=None, nargs='?', help='name of the program to be debugged' ) + paths.add_parser_argument(parser) def execCommand(args, following_args): @@ -35,10 +38,7 @@ def execCommand(args, following_args): logging.error(error) return 1 - if settings.getProperty('build', 'folder'): - builddir = settings.getProperty('build', 'folder') - else: - builddir = os.path.join(cwd, 'build', target.getName()) + builddir = paths.get_configured_output_path(args, target) if args.program is None: if c.isApplication(): diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py new file mode 100644 index 00000000..31f3f926 --- /dev/null +++ b/yotta/lib/paths.py @@ -0,0 +1,41 @@ +# Copyright 2014-2017 ARM Limited +# +# Licensed under the Apache License, Version 2.0 +# See LICENSE file for details. + +# standard library modules, , , +import os + +# settings, , load and save settings, internal +from yotta.lib import settings + +# this module to provide tools for path manipulation and caching + +BUILD_OUTPUT_KEY = 'folder' +PARSER_OUTPUT_FOLDER_KWARGS = dict + + +def add_parser_argument(parser): + parser.add_argument( + '-o', + '--output-folder', + dest='output_folder', + default=None, + help='Specify a build output folder instead of ./build/', + metavar='path/to/output/folder' + ) + + +def get_configured_output_path(args, target=None): + '''common method for setting/loading/defaulting a build output path''' + # load from command line or local config + path = args.output_folder or settings.getProperty('build', BUILD_OUTPUT_KEY) + + # else organise directories by target name in cwd + if not path and target: + path = os.path.join(os.getcwd(), 'build', target.getName()) + + path = os.path.abspath(path) + # could skip this if path==current ... is performance super important here? + settings.setProperty('build', BUILD_OUTPUT_KEY, path, True) + return path diff --git a/yotta/start.py b/yotta/start.py index cd487b9e..001e43bc 100644 --- a/yotta/start.py +++ b/yotta/start.py @@ -11,6 +11,8 @@ from yotta.lib import validate # settings, , load and save settings, internal from yotta.lib import settings +# paths +from yotta.lib import paths # --config option, , , internal from yotta import options @@ -20,6 +22,7 @@ def addOptions(parser): parser.add_argument('program', default=None, nargs='?', help='name of the program to be started' ) + paths.add_parser_argument(parser) def execCommand(args, following_args): @@ -39,10 +42,7 @@ def execCommand(args, following_args): logging.error(error) return 1 - if settings.getProperty('build', 'folder'): - builddir = settings.getProperty('build', 'folder') - else: - builddir = os.path.join(cwd, 'build', target.getName()) + builddir = paths.get_configured_output_path(args, target) if args.program is None: # if no program was specified, default to the name of the executable From b36dcb726a4323f47443339db3d3ec66794c0f2d Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 23 Aug 2017 16:29:16 +0100 Subject: [PATCH 04/54] use the generator rather than assertions --- yotta/lib/cmakegen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 66c9d897..663b77aa 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -107,7 +107,8 @@ def generateRecursive(self, component, all_components, builddir=None, modbuilddi for error in gen.generateRecursive(...): print(error) ''' - assert(self.configured) + if not self.configured: + yield 'Not configured' if builddir is None: builddir = self.buildroot From 97b8d0fab8aa2d7394007d5a670100c3182d52a7 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 23 Aug 2017 17:14:33 +0100 Subject: [PATCH 05/54] unused variables --- yotta/lib/cmakegen.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 663b77aa..a014b844 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -67,10 +67,6 @@ def configure(self, component, all_dependencies): ''' Ensure all config-time files have been generated. Return a dictionary of generated items. ''' - r = {} - - builddir = self.buildroot - # only dependencies which are actually valid can contribute to the # config data (which includes the versions of all dependencies in its # build info) if the dependencies aren't available we can't tell what @@ -81,11 +77,11 @@ def configure(self, component, all_dependencies): self.set_toplevel_definitions = '' if self.build_info_include_file is None: - self.build_info_include_file, build_info_definitions = self.getBuildInfo(component.path, builddir) + self.build_info_include_file, build_info_definitions = self.getBuildInfo(component.path, self.buildroot) self.set_toplevel_definitions += build_info_definitions if self.config_include_file is None: - self.config_include_file, config_definitions, self.config_json_file = self._getConfigData(available_dependencies, component, builddir, self.build_info_include_file) + self.config_include_file, config_definitions, self.config_json_file = self._getConfigData(available_dependencies, component, self.buildroot, self.build_info_include_file) self.set_toplevel_definitions += config_definitions self.configured = True From e6ed1f64bb237d0cc6e276a8c2f9c14322434319 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 24 Aug 2017 12:15:18 +0100 Subject: [PATCH 06/54] relative paths for CMakeLists --- yotta/lib/cmakegen.py | 16 ++++++++++------ yotta/lib/templates/base_CMakeLists.txt | 12 ++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index a014b844..de9faa07 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -6,6 +6,7 @@ # standard library modules, , , import os import logging +import functools import re import itertools from collections import defaultdict @@ -57,6 +58,7 @@ def __init__(self, directory, target): self.config_json_file = None self.build_info_include_file = None self.build_uuid = None + self.relative = None def _writeFile(self, path, contents): dirname = os.path.dirname(path) @@ -73,6 +75,11 @@ def configure(self, component, all_dependencies): # version they are. Anything missing here should always be a test # dependency that isn't going to be used, otherwise the yotta build # command will fail before we get here + + # a single-input filter function to convert paths to be relative to the buildroot + self.relative = functools.partial(os.path.relpath, start=self.buildroot) + jinja_environment.filters['relative'] = self.relative + available_dependencies = OrderedDict((k, v) for k, v in all_dependencies.items() if v) self.set_toplevel_definitions = '' @@ -337,7 +344,7 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade # make the path to the build-info header available both to CMake and # in the preprocessor: - full_build_info_header_path = replaceBackslashes(os.path.abspath(build_info_header_path)) + full_build_info_header_path = replaceBackslashes(self.relative(build_info_header_path)) logger.debug('build info header include path: "%s"', full_build_info_header_path) definitions.append(('YOTTA_BUILD_INFO_HEADER', '"'+full_build_info_header_path+'"')) @@ -380,7 +387,7 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade # out for gcc-compatible compilers only: config_include_file = os.path.join(builddir, 'yotta_config.h') config_json_file = os.path.join(builddir, 'yotta_config.json') - set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"%s\")\n' % replaceBackslashes(os.path.abspath(config_json_file)) + set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"%s\")\n' % replaceBackslashes(self.relative(config_json_file)) self._writeFile( config_include_file, @@ -609,16 +616,13 @@ def generate( # generate the top-level CMakeLists.txt template = jinja_environment.get_template('base_CMakeLists.txt') - - relpath = os.path.relpath(builddir, self.buildroot) - file_contents = template.render({ #pylint: disable=no-member "toplevel": toplevel, "target_name": self.target.getName(), "set_definitions": self.set_toplevel_definitions, "toolchain_file": toolchain_file_path, "component": component, - "relpath": relpath, + "build_dir": builddir, "include_root_dirs": include_root_dirs, "include_sys_dirs": include_sys_dirs, "include_other_dirs": include_other_dirs, diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index 6a53a41d..c15d88ce 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -17,7 +17,7 @@ add_custom_target(all_tests) cmake_policy(SET CMP0017 OLD) # toolchain file for {{ target_name }} -set(CMAKE_TOOLCHAIN_FILE "{{ toolchain_file | replaceBackslashes }}") +set(CMAKE_TOOLCHAIN_FILE "{{ toolchain_file | relative | replaceBackslashes }}") # provide function for post-processing executables function (yotta_postprocess_target target_type_ target_name_) @@ -54,7 +54,7 @@ if(NOT DEFINED YOTTA_FORCE_INCLUDE_FLAG) set(YOTTA_FORCE_INCLUDE_FLAG "-include") endif() endif() -add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"{{ config_include_file | replaceBackslashes }}\"") +add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"{{ config_include_file | relative | replaceBackslashes }}\"") {% endif %} # include root directories of all components we depend on (directly and @@ -101,7 +101,7 @@ set(YOTTA_MODULE_NAME {{ component.getName() }}) # delegate to an existing CMakeLists.txt: add_subdirectory( "{{ delegate_to | replaceBackslashes }}" - "{{ delegate_build_dir | replaceBackslashes }}" + "{{ delegate_build_dir | relative | replaceBackslashes }}" ) {% else %} # recurse into subdirectories for this component, using the two-argument @@ -109,8 +109,8 @@ add_subdirectory( # tree, not the working directory {% for srcdir, workingdir in add_own_subdirs %} add_subdirectory( - "{{ srcdir | replaceBackslashes }}" - "${CMAKE_BINARY_DIR}/{{ relpath | replaceBackslashes }}/{{ workingdir | replaceBackslashes }}" + "{{ srcdir | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ build_dir | relative | replaceBackslashes }}/{{ workingdir | replaceBackslashes }}" ) {% endfor %} {% endif %} @@ -127,6 +127,6 @@ target_compile_definitions({{ component.getName() }} PRIVATE "-DYOTTA_MODULE_NAM {% if cmake_includes %} # include .cmake files provided by the target: {% for f in cmake_includes %} -include("{{ f | replaceBackslashes }}") +include("{{ f | relative | replaceBackslashes }}") {% endfor %} {% endif %} From 1826a31f73b637fefca7f78c32d756d6421c7e70 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 24 Aug 2017 13:31:34 +0100 Subject: [PATCH 07/54] protect paths code from argparser Namespace missing keys --- yotta/lib/paths.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index 31f3f926..7adcff8a 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -28,8 +28,10 @@ def add_parser_argument(parser): def get_configured_output_path(args, target=None): '''common method for setting/loading/defaulting a build output path''' + args = vars(args) + # load from command line or local config - path = args.output_folder or settings.getProperty('build', BUILD_OUTPUT_KEY) + path = args.get('output_folder') or settings.getProperty('build', BUILD_OUTPUT_KEY) # else organise directories by target name in cwd if not path and target: From abfeeb259bcb0f6d79acb282fb08ffa401008162 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 24 Aug 2017 13:47:16 +0100 Subject: [PATCH 08/54] tidy up installAndBuild --- yotta/build.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index 4980f559..70c4b955 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -61,16 +61,16 @@ def installAndBuild(args, following_args): If status: is nonzero there was some sort of error. Other properties are optional, and may not be set if that step was not attempted. ''' - build_status = generate_status = install_status = 0 + build_status = generate_status = 0 + args_dict = vars(args) if not hasattr(args, 'build_targets'): - vars(args)['build_targets'] = [] + args_dict['build_targets'] = [] if 'test' in args.build_targets: logging.error('Cannot build "test". Use "yotta test" to run tests.') return {'status':1} - cwd = os.getcwd() c = validate.currentDirectoryModule() if not c: return {'status':1} @@ -87,20 +87,20 @@ def installAndBuild(args, following_args): # run the install command before building, we need to add some options the # install command expects to be present to do this: - vars(args)['component'] = None - vars(args)['act_globally'] = False + args_dict['component'] = None + args_dict['act_globally'] = False if not hasattr(args, 'install_test_deps'): if 'all_tests' in args.build_targets: - vars(args)['install_test_deps'] = 'all' + args_dict['install_test_deps'] = 'all' elif not len(args.build_targets): - vars(args)['install_test_deps'] = 'own' + args_dict['install_test_deps'] = 'own' else: # If the named build targets include tests from other modules, we # need to install the deps for those modules. To do this we need to # be able to tell which module a library belongs to, which is not # straightforward (especially if there is custom cmake involved). # That's why this is 'all', and not 'none'. - vars(args)['install_test_deps'] = 'all' + args_dict['install_test_deps'] = 'all' # install may exit non-zero for non-fatal errors (such as incompatible # version specs), which it will display @@ -144,7 +144,7 @@ def installAndBuild(args, following_args): # run pre-build scripts for all components: runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) - if (not hasattr(args, 'generate_only')) or (not args.generate_only): + if not args_dict.get('generate_only'): error = target.build( builddir, c, args, release_build=args.release_build, build_args=following_args, targets=args.build_targets From 403d6aa28c4bb735682abdde0dcc4e3b67f1e663 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 24 Aug 2017 15:04:36 +0100 Subject: [PATCH 09/54] explicit flag for performing a full export (?) --- yotta/build.py | 12 ++++++++++++ yotta/clean.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/yotta/build.py b/yotta/build.py index 70c4b955..c501c980 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -6,6 +6,7 @@ # standard library modules, , , import os import logging +import shutil # validate, , validate things, internal @@ -20,6 +21,7 @@ from yotta.lib import settings # paths from yotta.lib import paths +from yotta.lib import fsutils # install, , install subcommand, internal from yotta import install # --config option, , , internal @@ -31,6 +33,10 @@ def addOptions(parser, add_build_targets=True): action='store_true', default=False, help='Only generate CMakeLists, don\'t run CMake or build' ) + parser.add_argument('-s', '--source-export', dest='source_export', + action='store_true', default=False, + help='Copy the source directories into the build output directory' + ) paths.add_parser_argument(parser) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) parser.add_argument('-d', '--debug-build', dest='release_build', action='store_false', default=True) @@ -144,6 +150,12 @@ def installAndBuild(args, following_args): # run pre-build scripts for all components: runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) + if args.source_export: + source_export_path = os.path.join(builddir, 'source') + logging.info('copying source to %s', source_export_path) + fsutils.rmRf(source_export_path) + shutil.copytree(src='source', dst=source_export_path) + if not args_dict.get('generate_only'): error = target.build( builddir, c, args, release_build=args.release_build, diff --git a/yotta/clean.py b/yotta/clean.py index 0a1df9e5..801ccf52 100644 --- a/yotta/clean.py +++ b/yotta/clean.py @@ -5,6 +5,7 @@ # standard library modules, , , import os +import logging # validate, , validate things, internal from yotta.lib import validate @@ -33,5 +34,5 @@ def execCommand(args, following_args): for path in paths_to_remove: if os.path.exists(path) and not validate.directoryModule(path): - print('removing: %s' % path) + logging.info('removing: %s' % path) fsutils.rmRf(path) From e5e503ce7a8b7b62102e1939e771c03d9d9128a0 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 25 Aug 2017 12:12:39 +0100 Subject: [PATCH 10/54] print debug traceback on exception --- yotta/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yotta/main.py b/yotta/main.py index 9bb56fed..5a14540d 100644 --- a/yotta/main.py +++ b/yotta/main.py @@ -246,7 +246,9 @@ def onParserAdded(parser): logging.warning('interrupted') status = -1 except Exception as e: + import traceback logging.error(e) + logging.debug(traceback.format_exc()) status = -1 sys.exit(status or 0) From d317d59d7c11f90b3a07af3aad745765f95149a5 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 25 Aug 2017 13:25:09 +0100 Subject: [PATCH 11/54] 'export mode' added (but broken) 1. use common variable for module & target caches 2. change cache directories if exporting 3. export entire built directory to destination 4. missing relative 'include_directories' fixed --- yotta/build.py | 62 ++++++++++++++++++++----- yotta/init.py | 3 +- yotta/lib/cmakegen.py | 23 +++++---- yotta/lib/component.py | 7 ++- yotta/lib/folders.py | 10 ++-- yotta/lib/paths.py | 34 +++++++++----- yotta/lib/templates/base_CMakeLists.txt | 17 ++++--- yotta/link.py | 7 ++- yotta/link_target.py | 7 ++- 9 files changed, 120 insertions(+), 50 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index c501c980..7ecd4343 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -33,9 +33,12 @@ def addOptions(parser, add_build_targets=True): action='store_true', default=False, help='Only generate CMakeLists, don\'t run CMake or build' ) - parser.add_argument('-s', '--source-export', dest='source_export', - action='store_true', default=False, - help='Copy the source directories into the build output directory' + parser.add_argument('-x', '--export', dest='export', + default=False, const=True, nargs='?', + help=( + 'Export mode. If flag is set, generates builds without reference to Yotta.' + '[optional] Specify an output path to receive the clean build' + ) ) paths.add_parser_argument(parser) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) @@ -67,11 +70,28 @@ def installAndBuild(args, following_args): If status: is nonzero there was some sort of error. Other properties are optional, and may not be set if that step was not attempted. ''' - build_status = generate_status = 0 + build_status = 0 + generate_status = 0 + error = None args_dict = vars(args) - if not hasattr(args, 'build_targets'): - args_dict['build_targets'] = [] + if args.export: + logging.warn('overriding yotta namespaces') + new_modules = settings.getProperty('build', 'modules_directory_name') or 'modules' + new_targets = settings.getProperty('build', 'targets_directory_name') or 'targets' + # to save downloading the cached files, copy the old cache + try: + if not os.path.exists(new_modules): + shutil.copytree('yotta_modules', new_modules) + shutil.copytree('yotta_targets', new_targets) + except Exception as e: + logging.error(e) + logging.error('failed to use existing cache; rebuilding') + + paths.Modules_Folder = new_modules + paths.Targets_Folder = new_targets + + args_dict.setdefault('build_targets', []) if 'test' in args.build_targets: logging.error('Cannot build "test". Use "yotta test" to run tests.') @@ -150,12 +170,6 @@ def installAndBuild(args, following_args): # run pre-build scripts for all components: runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) - if args.source_export: - source_export_path = os.path.join(builddir, 'source') - logging.info('copying source to %s', source_export_path) - fsutils.rmRf(source_export_path) - shutil.copytree(src='source', dst=source_export_path) - if not args_dict.get('generate_only'): error = target.build( builddir, c, args, release_build=args.release_build, @@ -176,6 +190,30 @@ def installAndBuild(args, following_args): "`yotta install` for details." ) + if not error and args.export and args.export is not True: + # these export exclusions apply at any level of the directory structure + excludes = { + '.yotta.json', + '.module.json', + 'module.json', + '.yotta_origin.json', + 'target.json', + 'yotta_modules', + 'yotta_targets' + } + export_to = os.path.abspath(args.export) + logging.info('exporting built project to %s', export_to) + check = os.path.join(export_to, 'source') + if os.path.exists(export_to) and os.listdir(export_to) and not os.path.exists(check): + # as the export can be any path we do a sanity check before calling 'rmRf' + raise Exception( + 'Aborting export: directory is not empty and does not look like a previous export' + ' (expecting: %s)' + % (check) + ) + fsutils.rmRf(export_to) + shutil.copytree(src='.', dst=export_to, ignore=lambda a, b: excludes) + return { 'status': build_status or generate_status or install_status, 'missing_status': missing, diff --git a/yotta/init.py b/yotta/init.py index 3d9f78dd..784d734e 100644 --- a/yotta/init.py +++ b/yotta/init.py @@ -11,6 +11,7 @@ # Component, , represents an installed component, internal from yotta.lib import component +from yotta.lib import paths # version, , represent versions and specifications, internal from yotta.lib import version # validate, , validate various things, internal @@ -67,7 +68,7 @@ def yesNo(string): yesNo.__allowed_message = ' Please reply "Yes", or "No".' def isBannedName(name): - return name in ('test', 'source', 'include', 'yotta_modules', 'yotta_targets') + return name in ('test', 'source', 'include', paths.Modules_Folder, paths.Targets_Folder) def notBannedName(s): if isBannedName(s): diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index de9faa07..58c2f8be 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -18,11 +18,18 @@ # fsutils, , misc filesystem utils, internal from yotta.lib import fsutils +from yotta.lib import paths + Template_Dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates') logger = logging.getLogger('cmakegen') -Ignore_Subdirs = set(('build','yotta_modules', 'yotta_targets', 'CMake')) +Ignore_Subdirs = { + 'build', + 'CMake', + paths.Modules_Folder, + paths.Targets_Folder, +} jinja_environment = Environment(loader=FileSystemLoader(Template_Dir), trim_blocks=True, lstrip_blocks=True) @@ -472,22 +479,22 @@ def generate( another component. ''' - include_root_dirs = '' + include_root_dirs = [] if application is not None and component is not application: - include_root_dirs += 'include_directories("%s")\n' % replaceBackslashes(application.path) + include_root_dirs.append('include_directories("%s")\n' % replaceBackslashes(application.path)) - include_sys_dirs = '' - include_other_dirs = '' + include_sys_dirs = [] + include_other_dirs = [] for name, c in itertools.chain(((component.getName(), component),), all_dependencies.items()): if c is not component and c.isTestDependency(): continue - include_root_dirs += 'include_directories("%s")\n' % replaceBackslashes(c.path) + include_root_dirs.append(c.path) dep_sys_include_dirs = c.getExtraSysIncludes() for d in dep_sys_include_dirs: - include_sys_dirs += 'include_directories(SYSTEM "%s")\n' % replaceBackslashes(os.path.join(c.path, d)) + include_sys_dirs.append(os.path.join(c.path, d)) dep_extra_include_dirs = c.getExtraIncludes() for d in dep_extra_include_dirs: - include_other_dirs += 'include_directories("%s")\n' % replaceBackslashes(os.path.join(c.path, d)) + include_other_dirs.append(os.path.join(c.path, d)) add_depend_subdirs = '' for name, c in active_dependencies.items(): diff --git a/yotta/lib/component.py b/yotta/lib/component.py index 8357165c..7bfd59da 100644 --- a/yotta/lib/component.py +++ b/yotta/lib/component.py @@ -19,6 +19,7 @@ from yotta.lib import fsutils # Pack, , common parts of Components/Targets, internal from yotta.lib import pack +from yotta.lib import paths # !!! FIXME: should components lock their description file while they exist? # If not there are race conditions where the description file is modified by @@ -27,8 +28,6 @@ # Constants -Modules_Folder = 'yotta_modules' -Targets_Folder = 'yotta_targets' Component_Description_File = 'module.json' Component_Description_File_Fallback = 'package.json' Component_Definitions_File = 'defines.json' @@ -532,10 +531,10 @@ def getDependenciesRecursive(self, return components def modulesPath(self): - return os.path.join(self.path, Modules_Folder) + return os.path.join(self.path, paths.Modules_Folder) def targetsPath(self): - return os.path.join(self.path, Targets_Folder) + return os.path.join(self.path, paths.Targets_Folder) def satisfyDependenciesRecursive( self, diff --git a/yotta/lib/folders.py b/yotta/lib/folders.py index ea26876f..bc58be4c 100644 --- a/yotta/lib/folders.py +++ b/yotta/lib/folders.py @@ -10,6 +10,8 @@ # standard library modules, , , import os import sys +from yotta.lib import paths + def prefix(): if 'YOTTA_PREFIX' in os.environ: @@ -30,15 +32,15 @@ def userSettingsDirectory(): def globalInstallDirectory(): if os.name == 'nt': - return os.path.join(prefix(), 'Lib', 'yotta_modules') + return os.path.join(prefix(), 'Lib', paths.Modules_Folder) else: - return os.path.join(prefix(), 'lib', 'yotta_modules') + return os.path.join(prefix(), 'lib', paths.Modules_Folder) def globalTargetInstallDirectory(): if os.name == 'nt': - return os.path.join(prefix(), 'Lib', 'yotta_targets') + return os.path.join(prefix(), 'Lib', paths.Targets_Folder) else: - return os.path.join(prefix(), 'lib', 'yotta_targets') + return os.path.join(prefix(), 'lib', paths.Targets_Folder) def cacheDirectory(): return os.path.join(userSettingsDirectory(), 'cache') diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index 7adcff8a..cbf9dd8f 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -3,16 +3,13 @@ # Licensed under the Apache License, Version 2.0 # See LICENSE file for details. -# standard library modules, , , import os -# settings, , load and save settings, internal -from yotta.lib import settings - # this module to provide tools for path manipulation and caching - BUILD_OUTPUT_KEY = 'folder' PARSER_OUTPUT_FOLDER_KWARGS = dict +Modules_Folder = 'yotta_modules' +Targets_Folder = 'yotta_targets' def add_parser_argument(parser): @@ -21,23 +18,38 @@ def add_parser_argument(parser): '--output-folder', dest='output_folder', default=None, - help='Specify a build output folder instead of ./build/', + help=( + 'Customise the build directory (default: ./build/)' + 'Must be inside the project directory.' + ), metavar='path/to/output/folder' ) def get_configured_output_path(args, target=None): '''common method for setting/loading/defaulting a build output path''' + from yotta.lib import settings + args = vars(args) + current = os.getcwd() + from_settings = settings.getProperty('build', BUILD_OUTPUT_KEY) # load from command line or local config - path = args.get('output_folder') or settings.getProperty('build', BUILD_OUTPUT_KEY) + path = args.get('output_folder') or from_settings # else organise directories by target name in cwd if not path and target: - path = os.path.join(os.getcwd(), 'build', target.getName()) + path = os.path.join(current, 'build', target.getName()) + + path = os.path.relpath(path, start=current) + + if not os.path.abspath(path).startswith(current): + raise ValueError( + 'The build output directory must be a subdirectory to this project' + ' (requested: %s)' + % path + ) - path = os.path.abspath(path) - # could skip this if path==current ... is performance super important here? - settings.setProperty('build', BUILD_OUTPUT_KEY, path, True) + if path != from_settings: + settings.setProperty('build', BUILD_OUTPUT_KEY, path, True) return path diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index c15d88ce..866b783f 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -59,25 +59,30 @@ add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"{{ config_include_file | relative # include root directories of all components we depend on (directly and # indirectly, including ourself) -{{ include_root_dirs }} +{% for dir in include_root_dirs %} +include_directories("{{ dir | relative | replaceBackslashes }}") +{% endfor %} # recurse into dependencies that aren't built elsewhere {{ add_depend_subdirs }} - {% if include_sys_dirs %} # Some components (I'm looking at you, libc), need to export system header # files with no prefix, these directories are listed in the component # description files: -{{ include_sys_dirs }} -{% endif %} +{% for dir in include_sys_dirs %} +include_directories(SYSTEM "{{ dir | relative | replaceBackslashes }}") +{% endfor %} +{% endif %} {% if include_other_dirs %} # And others (typically CMSIS implementations) need to export non-system header # files. Please don't use this facility. Please. It's much, much better to fix # implementations that import these headers to import them using the full path. -{{ include_other_dirs }} -{% endif %} +{% for dir in include_other_dirs %} +include_directories("{{ dir | relative | replaceBackslashes }}") +{% endfor %} +{% endif %} # modules with custom CMake build systems may append to the # YOTTA_GLOBAL_INCLUDE_DIRS property to add compile-time-determined include # directories: diff --git a/yotta/link.py b/yotta/link.py index d13263df..60332266 100644 --- a/yotta/link.py +++ b/yotta/link.py @@ -3,6 +3,9 @@ # Licensed under the Apache License, Version 2.0 # See LICENSE file for details. +from yotta.lib import paths + + def addOptions(parser): parser.add_argument('module_or_path', default=None, nargs='?', help='Link a globally installed (or globally linked) module into '+ @@ -67,9 +70,9 @@ def execCommand(args, following_args): logging.error("%s is neither a valid module name, nor a path to an existing module.", args.module_or_path) logging.error(err) return 1 - fsutils.mkDirP(os.path.join(os.getcwd(), 'yotta_modules')) + fsutils.mkDirP(os.path.join(os.getcwd(), paths.Modules_Folder)) src = os.path.join(folders.globalInstallDirectory(), link_module_name) - dst = os.path.join(os.getcwd(), 'yotta_modules', link_module_name) + dst = os.path.join(os.getcwd(), paths.Modules_Folder, link_module_name) # if the component is already installed, rm it fsutils.rmRf(dst) else: diff --git a/yotta/link_target.py b/yotta/link_target.py index b704b1a6..cdc24c70 100644 --- a/yotta/link_target.py +++ b/yotta/link_target.py @@ -3,6 +3,9 @@ # Licensed under the Apache License, Version 2.0 # See LICENSE file for details. +from yotta.lib import paths + + def addOptions(parser): parser.add_argument('target_or_path', default=None, nargs='?', help='Link a globally installed (or globally linked) target into '+ @@ -73,9 +76,9 @@ def execCommand(args, following_args): else: logging.error(err) return 1 - fsutils.mkDirP(os.path.join(os.getcwd(), 'yotta_targets')) + fsutils.mkDirP(os.path.join(os.getcwd(), paths.Targets_Folder)) src = os.path.join(folders.globalTargetInstallDirectory(), link_target_name) - dst = os.path.join(os.getcwd(), 'yotta_targets', link_target_name) + dst = os.path.join(os.getcwd(), paths.Targets_Folder, link_target_name) # if the target is already installed, rm it fsutils.rmRf(dst) else: From 78fbe0b61c0a0d2aa5b77d2951ecc1fa2a762d6f Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 25 Aug 2017 17:08:43 +0100 Subject: [PATCH 12/54] works for very simple examples. e.g. simplelog wayyyy too much copypaste --- yotta/build.py | 47 ++++++++++++----------- yotta/lib/cmakegen.py | 27 ++++++------- yotta/lib/paths.py | 6 ++- yotta/lib/templates/base_CMakeLists.txt | 20 ++++++---- yotta/lib/templates/subdir_CMakeLists.txt | 10 ++--- yotta/lib/templates/test_CMakeLists.txt | 12 +++--- yotta/lib/templates/toolchain.cmake | 2 +- 7 files changed, 66 insertions(+), 58 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index 7ecd4343..792c244b 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -76,7 +76,7 @@ def installAndBuild(args, following_args): args_dict = vars(args) if args.export: - logging.warn('overriding yotta namespaces') + logging.info('this will be an export build') new_modules = settings.getProperty('build', 'modules_directory_name') or 'modules' new_targets = settings.getProperty('build', 'targets_directory_name') or 'targets' # to save downloading the cached files, copy the old cache @@ -170,27 +170,7 @@ def installAndBuild(args, following_args): # run pre-build scripts for all components: runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) - if not args_dict.get('generate_only'): - error = target.build( - builddir, c, args, release_build=args.release_build, - build_args=following_args, targets=args.build_targets - ) - - if error: - logging.error(error) - build_status = 1 - else: - # post-build scripts only get run if we were successful: - runScriptWithModules(c, all_deps.values(), 'postBuild', script_environment) - - if install_status: - logging.warning( - "There were also errors installing and resolving dependencies, "+ - "which may have caused the build failure: see above, or run "+ - "`yotta install` for details." - ) - - if not error and args.export and args.export is not True: + if args.export and args.export is not True: # these export exclusions apply at any level of the directory structure excludes = { '.yotta.json', @@ -214,6 +194,29 @@ def installAndBuild(args, following_args): fsutils.rmRf(export_to) shutil.copytree(src='.', dst=export_to, ignore=lambda a, b: excludes) + if args_dict.get('generate_only'): + logging.info('skipping build step') + else: + # build in the current directory + error = target.build( + builddir, c, args, release_build=args.release_build, + build_args=following_args, targets=args.build_targets + ) + + if error: + logging.error(error) + build_status = 1 + else: + # post-build scripts only get run if we were successful: + runScriptWithModules(c, all_deps.values(), 'postBuild', script_environment) + + if install_status: + logging.warning( + "There were also errors installing and resolving dependencies, "+ + "which may have caused the build failure: see above, or run "+ + "`yotta install` for details." + ) + return { 'status': build_status or generate_status or install_status, 'missing_status': missing, diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 58c2f8be..6eb17b0c 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -351,9 +351,9 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade # make the path to the build-info header available both to CMake and # in the preprocessor: - full_build_info_header_path = replaceBackslashes(self.relative(build_info_header_path)) - logger.debug('build info header include path: "%s"', full_build_info_header_path) - definitions.append(('YOTTA_BUILD_INFO_HEADER', '"'+full_build_info_header_path+'"')) + full_build_info_header_path = '"${CMAKE_BINARY_DIR}/%s"' % replaceBackslashes(self.relative(build_info_header_path)) + logger.debug('build info header include path: %s', full_build_info_header_path) + definitions.append(('YOTTA_BUILD_INFO_HEADER', full_build_info_header_path)) for target in self.target.getSimilarTo_Deprecated(): if '*' not in target: @@ -496,16 +496,13 @@ def generate( for d in dep_extra_include_dirs: include_other_dirs.append(os.path.join(c.path, d)) - add_depend_subdirs = '' + add_depend_subdirs = [] for name, c in active_dependencies.items(): - depend_subdir = replaceBackslashes(os.path.join(modbuilddir, name)) - relpath = replaceBackslashes(os.path.relpath(depend_subdir, self.buildroot)) - add_depend_subdirs += \ - 'add_subdirectory(\n' \ - ' "%s"\n' \ - ' "${CMAKE_BINARY_DIR}/%s"\n' \ - ')\n' \ - % (depend_subdir, relpath) + # a project-relative path to the top level dep + depend_subdir = os.path.join(self.relative(modbuilddir), name) + # a project relative path to the sub-dep + relpath = self.relative(depend_subdir) + add_depend_subdirs.append((depend_subdir, relpath)) delegate_to_existing = None delegate_build_dir = None @@ -614,9 +611,9 @@ def generate( # generate the top-level toolchain file: template = jinja_environment.get_template('toolchain.cmake') file_contents = template.render({ #pylint: disable=no-member - # toolchain files are provided in hierarchy - # order, but the template needs them in reverse - # order (base-first): + # toolchain files are provided in hierarchy + # order, but the template needs them in reverse + # order (base-first): "toolchain_files": self.target.getToolchainFiles() }) self._writeFile(toolchain_file_path, file_contents) diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index cbf9dd8f..793aa131 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -38,8 +38,10 @@ def get_configured_output_path(args, target=None): path = args.get('output_folder') or from_settings # else organise directories by target name in cwd - if not path and target: - path = os.path.join(current, 'build', target.getName()) + if not path: + path = os.path.join(current, 'build') + if target: + path = os.path.join(path, target.getName()) path = os.path.relpath(path, start=current) diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index 866b783f..3a4d6d85 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -54,7 +54,7 @@ if(NOT DEFINED YOTTA_FORCE_INCLUDE_FLAG) set(YOTTA_FORCE_INCLUDE_FLAG "-include") endif() endif() -add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"{{ config_include_file | relative | replaceBackslashes }}\"") +add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"${CMAKE_BINARY_DIR}/{{ config_include_file | relative | replaceBackslashes }}\"") {% endif %} # include root directories of all components we depend on (directly and @@ -63,8 +63,16 @@ add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"{{ config_include_file | relative include_directories("{{ dir | relative | replaceBackslashes }}") {% endfor %} +{% if add_depend_subdirs %} # recurse into dependencies that aren't built elsewhere -{{ add_depend_subdirs }} +{% for depend_subdir, relpath in add_depend_subdirs %} +add_subdirectory( + "${CMAKE_BINARY_DIR}/{{ depend_subdir | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ relpath | replaceBackslashes }}" +) + +{% endfor %} +{% endif %} {% if include_sys_dirs %} # Some components (I'm looking at you, libc), need to export system header # files with no prefix, these directories are listed in the component @@ -105,8 +113,7 @@ set(YOTTA_MODULE_NAME {{ component.getName() }}) {% if delegate_to %} # delegate to an existing CMakeLists.txt: add_subdirectory( - "{{ delegate_to | replaceBackslashes }}" - "{{ delegate_build_dir | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ delegate_to | replaceBackslashes }}" ) {% else %} # recurse into subdirectories for this component, using the two-argument @@ -114,8 +121,7 @@ add_subdirectory( # tree, not the working directory {% for srcdir, workingdir in add_own_subdirs %} add_subdirectory( - "{{ srcdir | relative | replaceBackslashes }}" - "${CMAKE_BINARY_DIR}/{{ build_dir | relative | replaceBackslashes }}/{{ workingdir | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" ) {% endfor %} {% endif %} @@ -132,6 +138,6 @@ target_compile_definitions({{ component.getName() }} PRIVATE "-DYOTTA_MODULE_NAM {% if cmake_includes %} # include .cmake files provided by the target: {% for f in cmake_includes %} -include("{{ f | relative | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ f | relative | replaceBackslashes }}") {% endfor %} {% endif %} diff --git a/yotta/lib/templates/subdir_CMakeLists.txt b/yotta/lib/templates/subdir_CMakeLists.txt index 9bde83a1..5966945f 100644 --- a/yotta/lib/templates/subdir_CMakeLists.txt +++ b/yotta/lib/templates/subdir_CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.11) -include_directories("{{ source_directory | replaceBackslashes }}") +include_directories("${CMAKE_BINARY_DIR}/{{ source_directory | relative | replaceBackslashes }}") {% if 's' in languages %} enable_language(ASM) @@ -12,18 +12,18 @@ enable_language(ASM) set(YOTTA_AUTO_{{ object_name.upper() }}_{{ lang | upper }}_FILES {% for file_name, language in source_files %} {% if language in lang %} - "{{ file_name | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ file_name | relative | replaceBackslashes }}" {% endif %} {% endfor %} ) # force dependency on the config header for {{ lang }} files, which CMake otherwise wouldn't track: -set_property(SOURCE ${YOTTA_AUTO_{{ object_name.upper() }}_{{ lang | upper }}_FILES} PROPERTY OBJECT_DEPENDS "{{ config_include_file | replaceBackslashes }}") +set_property(SOURCE ${YOTTA_AUTO_{{ object_name.upper() }}_{{ lang | upper }}_FILES} PROPERTY OBJECT_DEPENDS "${CMAKE_BINARY_DIR}/{{ config_include_file | relative | replaceBackslashes }}") {% endfor %} {% if resource_files %} set(YOTTA_AUTO_{{ object_name.upper() }}_RESOURCE_FILES {% for file_name in resource_files %} - "{{ file_name | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ file_name | relative | replaceBackslashes }}" {% endfor %} ) {% endif %} @@ -74,5 +74,5 @@ target_link_libraries({{ object_name }} ) {% for include in cmake_files %} -include("{{ include | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ include | relative | replaceBackslashes }}") {% endfor %} diff --git a/yotta/lib/templates/test_CMakeLists.txt b/yotta/lib/templates/test_CMakeLists.txt index 44732c7f..99dae2d1 100644 --- a/yotta/lib/templates/test_CMakeLists.txt +++ b/yotta/lib/templates/test_CMakeLists.txt @@ -1,22 +1,22 @@ # NOTE: This file is generated by yotta: changes will be overwritten! -include_directories("{{ source_directory | replaceBackslashes }}") +include_directories("${CMAKE_BINARY_DIR}/{{ source_directory | relative | replaceBackslashes }}") # add include path definitions needed only for tests (from testDependencies): {% for component in test_dependencies %} - include_directories("{{ component.path | replaceBackslashes }}") + include_directories("${CMAKE_BINARY_DIR}/{{ component.path | relative | replaceBackslashes }}") {% endfor %} {% for component in test_dependencies %} {% for d in component.getExtraSysIncludes() %} - include_directories(SYSTEM "{{ pathJoin(component.path, d) | replaceBackslashes }}") + include_directories(SYSTEM "${CMAKE_BINARY_DIR}/{{ pathJoin(component.path, d) | relative | replaceBackslashes }}") {% endfor %} {% endfor %} {% for component in test_dependencies %} {% for d in component.getExtraIncludes() %} - include_directories("{{ pathJoin(component.path, d) | replaceBackslashes }}") + include_directories("${CMAKE_BINARY_DIR}/{{ pathJoin(component.path, d) | relative | replaceBackslashes }}") {% endfor %} {% endfor %} @@ -26,7 +26,7 @@ include_directories("{{ source_directory | replaceBackslashes }}") {% for file_names, object_name, languages in tests %} add_executable({{ object_name }} {{ 'EXCLUDE_FROM_ALL' if exclude_from_all else '' }} {% for filename in file_names %} - "{{ filename | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ filename | relative | replaceBackslashes }}" {% endfor %} ) {% if 'objc' in languages %} @@ -45,5 +45,5 @@ add_dependencies(all_tests {{ object_name }}) {% endfor %} {% for include in cmake_files %} -include("{{ include | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ include | relative | replaceBackslashes }}") {% endfor %} diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index 8e9817cb..c45c08cc 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -6,6 +6,6 @@ endif() set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) {% for toolchain_file in toolchain_files %} -include("{{ toolchain_file | replaceBackslashes }}") +include("{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} From 75792e9976c9f9dd63c574c8140b61ba32548af0 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 25 Aug 2017 17:35:11 +0100 Subject: [PATCH 13/54] successfully builds `ble` + `deps` there are zero references to the project directory worth trying on ci --- yotta/lib/cmakegen.py | 2 +- yotta/lib/templates/base_CMakeLists.txt | 7 ++++--- yotta/lib/templates/dummy_CMakeLists.txt | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 6eb17b0c..cd101410 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -394,7 +394,7 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade # out for gcc-compatible compilers only: config_include_file = os.path.join(builddir, 'yotta_config.h') config_json_file = os.path.join(builddir, 'yotta_config.json') - set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"%s\")\n' % replaceBackslashes(self.relative(config_json_file)) + set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"${CMAKE_BINARY_DIR}/%s\")\n' % replaceBackslashes(self.relative(config_json_file)) self._writeFile( config_include_file, diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index 3a4d6d85..6db7c604 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -60,7 +60,7 @@ add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"${CMAKE_BINARY_DIR}/{{ config_inc # include root directories of all components we depend on (directly and # indirectly, including ourself) {% for dir in include_root_dirs %} -include_directories("{{ dir | relative | replaceBackslashes }}") +include_directories("${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes }}") {% endfor %} {% if add_depend_subdirs %} @@ -78,7 +78,7 @@ add_subdirectory( # files with no prefix, these directories are listed in the component # description files: {% for dir in include_sys_dirs %} -include_directories(SYSTEM "{{ dir | relative | replaceBackslashes }}") +include_directories(SYSTEM "${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes }}") {% endfor %} {% endif %} @@ -87,7 +87,7 @@ include_directories(SYSTEM "{{ dir | relative | replaceBackslashes }}") # files. Please don't use this facility. Please. It's much, much better to fix # implementations that import these headers to import them using the full path. {% for dir in include_other_dirs %} -include_directories("{{ dir | relative | replaceBackslashes }}") +include_directories("${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes }}") {% endfor %} {% endif %} @@ -122,6 +122,7 @@ add_subdirectory( {% for srcdir, workingdir in add_own_subdirs %} add_subdirectory( "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" ) {% endfor %} {% endif %} diff --git a/yotta/lib/templates/dummy_CMakeLists.txt b/yotta/lib/templates/dummy_CMakeLists.txt index 1c703fd3..a5b5499e 100644 --- a/yotta/lib/templates/dummy_CMakeLists.txt +++ b/yotta/lib/templates/dummy_CMakeLists.txt @@ -7,5 +7,5 @@ target_link_libraries({{ libname }} ) {% for include in cmake_files %} -include("{{ include | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ include | relative | replaceBackslashes }}") {% endfor %} From 56c3f9b9000fdcef028e553ac09f07a4609dd3e0 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 29 Aug 2017 09:20:03 +0100 Subject: [PATCH 14/54] stop using args namespaces (some more) --- yotta/build.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index 792c244b..2927245b 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -74,8 +74,9 @@ def installAndBuild(args, following_args): generate_status = 0 error = None args_dict = vars(args) + export_mode_or_path = args_dict.get('export') - if args.export: + if export_mode_or_path: logging.info('this will be an export build') new_modules = settings.getProperty('build', 'modules_directory_name') or 'modules' new_targets = settings.getProperty('build', 'targets_directory_name') or 'targets' @@ -170,18 +171,18 @@ def installAndBuild(args, following_args): # run pre-build scripts for all components: runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) - if args.export and args.export is not True: + if export_mode_or_path and export_mode_or_path is not True: # these export exclusions apply at any level of the directory structure excludes = { - '.yotta.json', '.module.json', - 'module.json', + '.yotta.json', '.yotta_origin.json', + 'module.json', 'target.json', 'yotta_modules', 'yotta_targets' } - export_to = os.path.abspath(args.export) + export_to = os.path.abspath(export_mode_or_path) logging.info('exporting built project to %s', export_to) check = os.path.join(export_to, 'source') if os.path.exists(export_to) and os.listdir(export_to) and not os.path.exists(check): From 0fdbef42f9e05869137e045708499cd15fdb3307 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 29 Aug 2017 09:34:52 +0100 Subject: [PATCH 15/54] use unittest assertions properly, update for ym->modules --- yotta/test/test_test_subcommand.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yotta/test/test_test_subcommand.py b/yotta/test/test_test_subcommand.py index 397b737d..51ddb584 100644 --- a/yotta/test/test_test_subcommand.py +++ b/yotta/test/test_test_subcommand.py @@ -14,11 +14,11 @@ class TestTestSubcommandModule(unittest.TestCase): def test_moduleFromDirname(self): - self.assertTrue(test_subcommand.moduleFromDirname('ym/b/ym/c/d', {'b':'b', 'c':'c'}, 'a') == 'c') - self.assertTrue(test_subcommand.moduleFromDirname('ym/b/q/c/d', {'b':'b', 'c':'c'}, 'a') == 'b') - self.assertTrue(test_subcommand.moduleFromDirname('z/b/q/c/d', {'b':'b', 'c':'c'}, 'a') == 'a') - self.assertTrue(test_subcommand.moduleFromDirname('ym/e/d', {'b':'b', 'c':'c'}, 'a') == 'a') - self.assertTrue(test_subcommand.moduleFromDirname('ym/e/d', {'b':'b', 'c':'c', 'e':'e'}, 'a') == 'e') + self.assertEqual(test_subcommand.moduleFromDirname('modules/b/modules/c/d', {'b':'b', 'c':'c'}, 'a') , 'c') + self.assertEqual(test_subcommand.moduleFromDirname('modules/b/q/c/d', {'b':'b', 'c':'c'}, 'a') , 'b') + self.assertEqual(test_subcommand.moduleFromDirname('z/b/q/c/d', {'b':'b', 'c':'c'}, 'a') , 'a') + self.assertEqual(test_subcommand.moduleFromDirname('modules/e/d', {'b':'b', 'c':'c'}, 'a') , 'a') + self.assertEqual(test_subcommand.moduleFromDirname('modules/e/d', {'b':'b', 'c':'c', 'e':'e'}, 'a') , 'e') # see also yotta/test/cli/test.py for cli-driven testing From 8bb65d61d3ad15666f6ca8ea3569d27f4ab89c79 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 29 Aug 2017 10:04:09 +0100 Subject: [PATCH 16/54] update docs --- docs/tutorial/yotta_link.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/tutorial/yotta_link.md b/docs/tutorial/yotta_link.md index 41912c8b..817a92a3 100644 --- a/docs/tutorial/yotta_link.md +++ b/docs/tutorial/yotta_link.md @@ -51,7 +51,7 @@ info: generate for target: x86-osx-native 0.0.7 at /path/to/helloyotta/yotta_tar -- Generating done -- Build files have been written to: /path/to/helloyotta/build/x86-osx-native [1/3] Building C object ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -FAILED: /usr/bin/cc -I/path/to/helloyotta -I/path/to/helloyotta/yotta_modules/simplelog -I/path/to/helloyotta/yotta_modules/simplelog/source -O2 -g -DNDEBUG -include "/path/to/helloyotta/build/x86-osx-native/yotta_config.h" -MMD -MT ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -MF ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o.d -o ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c +FAILED: /usr/bin/cc -I/path/to/helloyotta -I/path/to/helloyotta/yotta_modules/simplelog -I/path/to/helloyotta/yotta_modules/simplelog/source -O2 -g -DNDEBUG -include "/path/to/helloyotta/build/x86-osx-native/yotta_config.h" -MMD -MT modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -MF ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o.d -o ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c:25:50: error: expected ';' after expression printf("%s %s\n", prefixForLevel(level), msg) // deliberately missing semicolon ^ @@ -64,14 +64,14 @@ error: command ['ninja'] failed Here we can see that the file that was being built when the error occurred was: ``` -[1/3] Building C object ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o +[1/3] Building C object modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o ``` -This is a path relative to the build directory, the `ym` directory is the +This is a path relative to the build directory, the `modules` directory is the subdirectory where dependencies are built, and is always followed immediately -by a dependency name (if there were no `ym/` at the start of the path then the +by a dependency name (if there were no `modules/` at the start of the path then the object would belong to the top-level module/app being built). So from -`/ym/simplelog/...` we can tell that the failed file is in the `simplelog` +`/modules/simplelog/...` we can tell that the failed file is in the `simplelog` dependency. This is also visible from the source file where the error is reported: From 7cc282fdf797321bc2c21557ac515ec1ba73785f Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 29 Aug 2017 10:08:19 +0100 Subject: [PATCH 17/54] toolchain path, build cli clarification --- yotta/build.py | 6 +++--- yotta/lib/templates/toolchain.cmake | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index 2927245b..a94a68e4 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -77,7 +77,7 @@ def installAndBuild(args, following_args): export_mode_or_path = args_dict.get('export') if export_mode_or_path: - logging.info('this will be an export build') + logging.info('this build will be configured for exporting') new_modules = settings.getProperty('build', 'modules_directory_name') or 'modules' new_targets = settings.getProperty('build', 'targets_directory_name') or 'targets' # to save downloading the cached files, copy the old cache @@ -183,8 +183,8 @@ def installAndBuild(args, following_args): 'yotta_targets' } export_to = os.path.abspath(export_mode_or_path) - logging.info('exporting built project to %s', export_to) - check = os.path.join(export_to, 'source') + logging.info('exporting unbuilt project source and dependencies to %s', export_to) + check = os.path.join(export_to, 'source') # this might be insufficient, or indeed misleading. if os.path.exists(export_to) and os.listdir(export_to) and not os.path.exists(check): # as the export can be any path we do a sanity check before calling 'rmRf' raise Exception( diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index c45c08cc..1358fca8 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -6,6 +6,6 @@ endif() set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) {% for toolchain_file in toolchain_files %} -include("{{ toolchain_file | relative | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} From 3ce22b2a26232aea2410f5c673ecdd7c2055c6b8 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 29 Aug 2017 17:17:49 +0100 Subject: [PATCH 18/54] debugging circle using print statements. great. --- yotta/lib/templates/toolchain.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index 1358fca8..65c3d8dc 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -5,7 +5,11 @@ if(YOTTA_META_TOOLCHAIN_FILE_INCLUDED) endif() set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) +message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") +message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") +message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") +message("PROJECT_BINARY_DIR : ${PROJECT_BINARY_DIR}") {% for toolchain_file in toolchain_files %} -include("${CMAKE_BINARY_DIR}/{{ toolchain_file | relative | replaceBackslashes }}") +include("${PROJECT_SOURCE_DIR}/{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} From 0f3e27743b8a1cc852769fcc78a4fced2e7ad123 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 30 Aug 2017 10:39:21 +0100 Subject: [PATCH 19/54] in toolchain, force a string replace on toolchain include files --- yotta/lib/templates/toolchain.cmake | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index 65c3d8dc..bb9ab931 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -5,11 +5,17 @@ if(YOTTA_META_TOOLCHAIN_FILE_INCLUDED) endif() set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) +# this is a poor attempt to resolve build issues where CMAKE generates its own temp directories +# and within those directories, project-relative paths will not work ... unless we commit this atrocity: +string(REPLACE "CMakeFiles/CMakeTmp/" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") + +message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") + message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") message("PROJECT_BINARY_DIR : ${PROJECT_BINARY_DIR}") {% for toolchain_file in toolchain_files %} -include("${PROJECT_SOURCE_DIR}/{{ toolchain_file | relative | replaceBackslashes }}") +include("${CMAKE_BINDIR_NO_NESTING}/{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} From f5fbca18bc6e5ab3ca7d8533bd570cfd2e6fe09c Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 30 Aug 2017 10:45:15 +0100 Subject: [PATCH 20/54] ... --- yotta/lib/templates/toolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index bb9ab931..698ef981 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -7,7 +7,7 @@ set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) # this is a poor attempt to resolve build issues where CMAKE generates its own temp directories # and within those directories, project-relative paths will not work ... unless we commit this atrocity: -string(REPLACE "CMakeFiles/CMakeTmp/" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") +string(REPLACE "/CMakeFiles/CMakeTmp" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") From afc82027b8c98ae9975baf806a87fcb7fba9e7bc Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 30 Aug 2017 11:28:44 +0100 Subject: [PATCH 21/54] toolchain with absolute paths? --- yotta/lib/templates/toolchain.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index 698ef981..b3e6e176 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -9,13 +9,13 @@ set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) # and within those directories, project-relative paths will not work ... unless we commit this atrocity: string(REPLACE "/CMakeFiles/CMakeTmp" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") -message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") +#message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") + +#message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") +#message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") +#message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") +#message("PROJECT_BINARY_DIR: ${PROJECT_BINARY_DIR}") -message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") -message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") -message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") -message("PROJECT_BINARY_DIR : ${PROJECT_BINARY_DIR}") {% for toolchain_file in toolchain_files %} -include("${CMAKE_BINDIR_NO_NESTING}/{{ toolchain_file | relative | replaceBackslashes }}") +include("{{ toolchain_file | replaceBackslashes }}") {% endfor %} - From fa11631803b6a4cf76dd5bf0c286788f22eabb5d Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 30 Aug 2017 11:28:44 +0100 Subject: [PATCH 22/54] Revert "toolchain with absolute paths?" This reverts commit afc82027b8c98ae9975baf806a87fcb7fba9e7bc. --- yotta/lib/templates/toolchain.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index b3e6e176..698ef981 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -9,13 +9,13 @@ set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) # and within those directories, project-relative paths will not work ... unless we commit this atrocity: string(REPLACE "/CMakeFiles/CMakeTmp" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") -#message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") - -#message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") -#message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") -#message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") -#message("PROJECT_BINARY_DIR: ${PROJECT_BINARY_DIR}") +message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") +message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") +message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") +message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") +message("PROJECT_BINARY_DIR : ${PROJECT_BINARY_DIR}") {% for toolchain_file in toolchain_files %} -include("{{ toolchain_file | replaceBackslashes }}") +include("${CMAKE_BINDIR_NO_NESTING}/{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} + From e9d0423513b85c32544e05a23231a2d8695abccf Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 30 Aug 2017 13:03:27 +0100 Subject: [PATCH 23/54] add a dockerfile for testing --- dockerfile | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 dockerfile diff --git a/dockerfile b/dockerfile new file mode 100644 index 00000000..2fca5157 --- /dev/null +++ b/dockerfile @@ -0,0 +1,34 @@ +FROM ubuntu + +RUN apt-get update && apt-get install -yq \ + git \ + curl \ + ninja-build \ + python \ + python-dev \ + python-distribute \ + python-pip \ +&& rm -rf /var/lib/apt/lists/* + +RUN python -m pip install -U pip +RUN pip install tox + +# fix up cmake +RUN apt-get remove -yq cmake +RUN curl -fsSL http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-i386.sh > /tmp/install-cmake.sh +RUN chmod +x /tmp/install-cmake.sh +RUN /tmp/install-cmake.sh --prefix=/usr/local --exclude-subdir + +ENV PYTHONIOENCODING: UTF-8 +ENV YOTTA_GITHUB_AUTHTOKEN: 8d1dfa2011f74b1f26504918982e1e2ba154b910 + +WORKDIR yt_testbuild +ADD . . + +RUN bash ci_init.sh +RUN tox --notest + +# TODO: +# CMAKE not installed +# Why don't all the pip installs work +# Maybe setup.py isn't working or being invoked correctly [by tox?] From 73091234c38299ecacf5bbf2478eafbdf6f89d8e Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 6 Sep 2017 17:39:57 +0100 Subject: [PATCH 24/54] Templated replacement for yotta header files --- yotta/lib/cmakegen.py | 2 +- yotta/lib/target.py | 2 +- yotta/lib/templates/base_CMakeLists.txt | 3 +++ yotta/test/cli/test_debug.py | 11 ++++++----- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index cd101410..ad5c45e3 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -397,7 +397,7 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"${CMAKE_BINARY_DIR}/%s\")\n' % replaceBackslashes(self.relative(config_json_file)) self._writeFile( - config_include_file, + config_include_file + '.cmake_template', '#ifndef __YOTTA_CONFIG_H__\n'+ '#define __YOTTA_CONFIG_H__\n'+ add_defs_header+ diff --git a/yotta/lib/target.py b/yotta/lib/target.py index 2835bdd8..c1015863 100644 --- a/yotta/lib/target.py +++ b/yotta/lib/target.py @@ -470,7 +470,7 @@ def exec_helper(self, cmd, builddir): else: return '%s is not installed' % (cmd[0]) else: - return 'command %s failed' % (cmd) + return 'command execution %s failed with %s' % (cmd, e) if child.returncode: return 'command %s failed' % (cmd) diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index 6db7c604..f09d1d9f 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -19,6 +19,9 @@ cmake_policy(SET CMP0017 OLD) # toolchain file for {{ target_name }} set(CMAKE_TOOLCHAIN_FILE "{{ toolchain_file | relative | replaceBackslashes }}") +# template replacement +configure_file( yotta_config.h.cmake_template yotta_config.h ) + # provide function for post-processing executables function (yotta_postprocess_target target_type_ target_name_) if(COMMAND yotta_apply_target_rules) diff --git a/yotta/test/cli/test_debug.py b/yotta/test/cli/test_debug.py index 2902393a..93812341 100644 --- a/yotta/test/cli/test_debug.py +++ b/yotta/test/cli/test_debug.py @@ -45,13 +45,14 @@ def _nopDebugTargetDescription(name): class TestCLIDebug(unittest.TestCase): @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") def test_noop_debug(self): + target_name = 'debug-test-target' test_dir = util.writeTestFiles(util.Test_Trivial_Exe, True) - target_dir = os.path.realpath(os.path.join(test_dir, 'yotta_targets', 'debug-test-target')) - build_dir = os.path.realpath(os.path.join(test_dir, 'build', 'debug-test-target')) + target_dir = os.path.realpath(os.path.join(test_dir, 'yotta_targets', target_name)) + build_dir = os.path.realpath(os.path.join('build', target_name)) - util.writeTestFiles(_nopDebugTargetDescription('debug-test-target'), test_dir=target_dir) - output = util.runCheckCommand(['--target', 'debug-test-target', 'build'], test_dir) - output = util.runCheckCommand(['--target', 'debug-test-target', 'debug'], test_dir) + util.writeTestFiles(_nopDebugTargetDescription(target_name), test_dir=target_dir) + output = util.runCheckCommand(['--target', target_name, 'build'], test_dir) + output = util.runCheckCommand(['--target', target_name, 'debug'], test_dir) json_output = output[:output.index(JSON_MARKER)] result = json.loads(json_output) From a4e1722eaace28f7172284f409f278183d632721 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 7 Sep 2017 09:20:22 +0100 Subject: [PATCH 25/54] yotta self reference --- yotta/test/cli/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/yotta/test/cli/cli.py b/yotta/test/cli/cli.py index 541cedb1..0fd632bc 100644 --- a/yotta/test/cli/cli.py +++ b/yotta/test/cli/cli.py @@ -10,7 +10,7 @@ import os def run(arguments, cwd='.'): - yottadir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..') + yottadir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) runyotta = [ sys.executable, '-c', @@ -24,11 +24,9 @@ def run(arguments, cwd='.'): stdin = subprocess.PIPE ) out, err = child.communicate() + # no command should ever produce a traceback: if 'traceback' in (out.decode('utf-8')+err.decode('utf-8')).lower(): print(out+err) assert(False) return out.decode('utf-8'), err.decode('utf-8'), child.returncode - - - From b494095a49bfcc90a2a27a53d6e2e2cd94c05c83 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 7 Sep 2017 09:20:36 +0100 Subject: [PATCH 26/54] test_debug relpath --- yotta/test/cli/test_debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/test/cli/test_debug.py b/yotta/test/cli/test_debug.py index 93812341..a87db364 100644 --- a/yotta/test/cli/test_debug.py +++ b/yotta/test/cli/test_debug.py @@ -48,7 +48,7 @@ def test_noop_debug(self): target_name = 'debug-test-target' test_dir = util.writeTestFiles(util.Test_Trivial_Exe, True) target_dir = os.path.realpath(os.path.join(test_dir, 'yotta_targets', target_name)) - build_dir = os.path.realpath(os.path.join('build', target_name)) + build_dir = os.path.join('build', target_name) util.writeTestFiles(_nopDebugTargetDescription(target_name), test_dir=target_dir) output = util.runCheckCommand(['--target', target_name, 'build'], test_dir) From fae3e20e6a4fe6a847a5d0d10076852ce42be04f Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 7 Sep 2017 16:41:52 +0100 Subject: [PATCH 27/54] tweak template comments --- yotta/lib/templates/base_CMakeLists.txt | 2 +- yotta/lib/templates/dummy_CMakeLists.txt | 4 ++-- yotta/lib/templates/subdir_CMakeLists.txt | 2 +- yotta/lib/templates/test_CMakeLists.txt | 3 +-- yotta/lib/templates/toolchain.cmake | 10 ++-------- 5 files changed, 7 insertions(+), 14 deletions(-) diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index f09d1d9f..fdf9de67 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! {% if toplevel %} cmake_minimum_required(VERSION 2.8.11) diff --git a/yotta/lib/templates/dummy_CMakeLists.txt b/yotta/lib/templates/dummy_CMakeLists.txt index a5b5499e..459cc63f 100644 --- a/yotta/lib/templates/dummy_CMakeLists.txt +++ b/yotta/lib/templates/dummy_CMakeLists.txt @@ -1,6 +1,6 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! -add_library({{ libname }} {{ cfile_name | replaceBackslashes }}) +add_library({{ libname }} {{ cfile_name | relative | replaceBackslashes }}) target_link_libraries({{ libname }} {{ link_dependencies | join('\n ') }} diff --git a/yotta/lib/templates/subdir_CMakeLists.txt b/yotta/lib/templates/subdir_CMakeLists.txt index 5966945f..ebfed0c8 100644 --- a/yotta/lib/templates/subdir_CMakeLists.txt +++ b/yotta/lib/templates/subdir_CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! cmake_minimum_required(VERSION 2.8.11) diff --git a/yotta/lib/templates/test_CMakeLists.txt b/yotta/lib/templates/test_CMakeLists.txt index 99dae2d1..89776526 100644 --- a/yotta/lib/templates/test_CMakeLists.txt +++ b/yotta/lib/templates/test_CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! include_directories("${CMAKE_BINARY_DIR}/{{ source_directory | relative | replaceBackslashes }}") @@ -22,7 +22,6 @@ include_directories("${CMAKE_BINARY_DIR}/{{ source_directory | relative | replac # define the tests themselves: - {% for file_names, object_name, languages in tests %} add_executable({{ object_name }} {{ 'EXCLUDE_FROM_ALL' if exclude_from_all else '' }} {% for filename in file_names %} diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index 698ef981..64a2757e 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -1,4 +1,4 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! if(YOTTA_META_TOOLCHAIN_FILE_INCLUDED) return() @@ -6,15 +6,9 @@ endif() set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) # this is a poor attempt to resolve build issues where CMAKE generates its own temp directories -# and within those directories, project-relative paths will not work ... unless we commit this atrocity: +# --> within those directories, project-relative paths will not work ... unless we commit this atrocity: string(REPLACE "/CMakeFiles/CMakeTmp" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") -message("CMAKE_BINDIR_NO_NESTING: ${CMAKE_BINDIR_NO_NESTING}") - -message("CMAKE_BINARY_DIR : ${CMAKE_BINARY_DIR}") -message("PROJECT_SOURCE_DIR: ${PROJECT_SOURCE_DIR}") -message("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") -message("PROJECT_BINARY_DIR : ${PROJECT_BINARY_DIR}") {% for toolchain_file in toolchain_files %} include("${CMAKE_BINDIR_NO_NESTING}/{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} From f4098b2edcac3f5a76e153494e737cbbddc641ac Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 8 Sep 2017 09:47:05 +0100 Subject: [PATCH 28/54] dockerfile updates --- .dockerignore | 6 ++++++ dockerfile | 48 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5c6f3474 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +# things to ignore when copying this source directory into the docker image +dockerfile +.dockerignore +**/.idea +**/.vscode +**/.DS_Store diff --git a/dockerfile b/dockerfile index 2fca5157..d13c19b5 100644 --- a/dockerfile +++ b/dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu +FROM i686/ubuntu RUN apt-get update && apt-get install -yq \ git \ @@ -11,24 +11,54 @@ RUN apt-get update && apt-get install -yq \ && rm -rf /var/lib/apt/lists/* RUN python -m pip install -U pip -RUN pip install tox + +# install yotta development dependencies +RUN pip install \ + tox \ + cython \ + nose \ + pylint \ + coverage \ + green \ + ipython # fix up cmake RUN apt-get remove -yq cmake + +# FIXME: 64 or 32? RUN curl -fsSL http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-i386.sh > /tmp/install-cmake.sh +#RUN curl -fsSL https://cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh > /tmp/install-cmake.sh + RUN chmod +x /tmp/install-cmake.sh -RUN /tmp/install-cmake.sh --prefix=/usr/local --exclude-subdir +RUN /tmp/install-cmake.sh --prefix=/usr/local/ --exclude-subdir ENV PYTHONIOENCODING: UTF-8 ENV YOTTA_GITHUB_AUTHTOKEN: 8d1dfa2011f74b1f26504918982e1e2ba154b910 +# FIXME: "CMAKE not installed" Maybe a difference between Ubuntu and CircleCI? +ENV PATH="/usr/local:${PATH}" + WORKDIR yt_testbuild -ADD . . -RUN bash ci_init.sh +# add setup.py &tc (for the dependencies) first, to preserve docker cache +ADD setup.py . +ADD yotta/version.txt yotta/version.txt +ADD pypi_readme.rst . +ADD bin/ bin/ + +# install yotta dependencies +RUN pip install -e . + +# prepare tox in the same way as CircleCI +ADD tox.ini . RUN tox --notest -# TODO: -# CMAKE not installed -# Why don't all the pip installs work -# Maybe setup.py isn't working or being invoked correctly [by tox?] +# metadata +LABEL description="An image for devtesting Yotta in a stable environment. One day it might replace the CircleCI script." +LABEL incantation="docker build -t yotta . && docker run --rm --name yotta -it yotta /bin/bash" + +# development workflow +# TODO: volume mount +ADD . . + +RUN bash ci_init.sh From 4f18ae3b2deb97f3c18983345b40cb382165fefd Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 12 Sep 2017 17:46:03 +0100 Subject: [PATCH 29/54] Fixes test_testCustomCMake --- yotta/lib/templates/base_CMakeLists.txt | 7 ++++--- yotta/lib/templates/dummy_CMakeLists.txt | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index fdf9de67..b33c86ee 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -70,8 +70,8 @@ include_directories("${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes # recurse into dependencies that aren't built elsewhere {% for depend_subdir, relpath in add_depend_subdirs %} add_subdirectory( - "${CMAKE_BINARY_DIR}/{{ depend_subdir | replaceBackslashes }}" "${CMAKE_BINARY_DIR}/{{ relpath | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ depend_subdir | replaceBackslashes }}" ) {% endfor %} @@ -116,7 +116,8 @@ set(YOTTA_MODULE_NAME {{ component.getName() }}) {% if delegate_to %} # delegate to an existing CMakeLists.txt: add_subdirectory( - "${CMAKE_BINARY_DIR}/{{ delegate_to | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ delegate_to | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ delegate_build_dir | relative | replaceBackslashes }}" ) {% else %} # recurse into subdirectories for this component, using the two-argument @@ -125,7 +126,7 @@ add_subdirectory( {% for srcdir, workingdir in add_own_subdirs %} add_subdirectory( "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" - "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ workingdir | replaceBackslashes }}" ) {% endfor %} {% endif %} diff --git a/yotta/lib/templates/dummy_CMakeLists.txt b/yotta/lib/templates/dummy_CMakeLists.txt index 459cc63f..06f98e53 100644 --- a/yotta/lib/templates/dummy_CMakeLists.txt +++ b/yotta/lib/templates/dummy_CMakeLists.txt @@ -1,6 +1,6 @@ # NOTE: This file is generated from {{ self }}: changes will be overwritten! -add_library({{ libname }} {{ cfile_name | relative | replaceBackslashes }}) +add_library({{ libname }} {{ cfile_name | replaceBackslashes }}) target_link_libraries({{ libname }} {{ link_dependencies | join('\n ') }} From a39434211567dfbc5a6557e6074dc7c7773d6233 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Wed, 13 Sep 2017 15:12:13 +0100 Subject: [PATCH 30/54] unicode is bad for the environment --- yotta/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/build.py b/yotta/build.py index a94a68e4..f10b5d37 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -157,7 +157,7 @@ def installAndBuild(args, following_args): logging.debug("config done, merged config: %s", config['merged_config_json']) script_environment = { - 'YOTTA_MERGED_CONFIG_FILE': config['merged_config_json'] + 'YOTTA_MERGED_CONFIG_FILE': str(config['merged_config_json']) } # run pre-generate scripts for all components: runScriptWithModules(c, all_deps.values(), 'preGenerate', script_environment) From baecb75b275a7137911b193cf3eb29091dc81339 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 11:45:05 +0100 Subject: [PATCH 31/54] tweaks for dev docker --- .dockerignore | 1 + dockerfile | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index 5c6f3474..6b08601d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ dockerfile **/.idea **/.vscode **/.DS_Store +*.git diff --git a/dockerfile b/dockerfile index d13c19b5..8ee2bc4e 100644 --- a/dockerfile +++ b/dockerfile @@ -1,8 +1,9 @@ FROM i686/ubuntu -RUN apt-get update && apt-get install -yq \ - git \ +RUN apt-get -yqq update && apt-get -yqq install \ curl \ + git \ + nano \ ninja-build \ python \ python-dev \ @@ -13,7 +14,7 @@ RUN apt-get update && apt-get install -yq \ RUN python -m pip install -U pip # install yotta development dependencies -RUN pip install \ +RUN pip install -q \ tox \ cython \ nose \ @@ -47,7 +48,7 @@ ADD pypi_readme.rst . ADD bin/ bin/ # install yotta dependencies -RUN pip install -e . +RUN pip install -q -e . # prepare tox in the same way as CircleCI ADD tox.ini . @@ -55,10 +56,18 @@ RUN tox --notest # metadata LABEL description="An image for devtesting Yotta in a stable environment. One day it might replace the CircleCI script." -LABEL incantation="docker build -t yotta . && docker run --rm --name yotta -it yotta /bin/bash" +LABEL incantation="docker build -t yotta . && docker run --rm --name yotta -v your_yotta_dir:/yt_testbuild -it yotta /bin/bash" -# development workflow -# TODO: volume mount +# development workflow - this bit now optional if you do a volume mount ADD . . +RUN ls -lah + RUN bash ci_init.sh + +# TODO: volume mount +# You (the user) must mount your working directory into the image using the -v command at run time, because: +# "...you can’t mount a host directory from within the Dockerfile. +# The VOLUME instruction does not support specifying a host-dir parameter. +# You must specify the mountpoint when you create or run the container." +# - https://docs.docker.com/engine/reference/builder/#notes-about-specifying-volumes From f4530a80dbc1c57beaa080c9c0013da6b9a7c9bb Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 11:46:04 +0100 Subject: [PATCH 32/54] clarify depend_subdirs --- yotta/lib/cmakegen.py | 7 +++---- yotta/lib/templates/base_CMakeLists.txt | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index ad5c45e3..8c9053a6 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -499,10 +499,9 @@ def generate( add_depend_subdirs = [] for name, c in active_dependencies.items(): # a project-relative path to the top level dep - depend_subdir = os.path.join(self.relative(modbuilddir), name) - # a project relative path to the sub-dep - relpath = self.relative(depend_subdir) - add_depend_subdirs.append((depend_subdir, relpath)) + source_dir = os.path.join(modbuilddir, name) + binary_dir = source_dir + add_depend_subdirs.append((source_dir, binary_dir)) delegate_to_existing = None delegate_build_dir = None diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index b33c86ee..37abf744 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -68,10 +68,10 @@ include_directories("${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes {% if add_depend_subdirs %} # recurse into dependencies that aren't built elsewhere -{% for depend_subdir, relpath in add_depend_subdirs %} +{% for source_dir, binary_dir in add_depend_subdirs %} add_subdirectory( - "${CMAKE_BINARY_DIR}/{{ relpath | replaceBackslashes }}" - "${CMAKE_BINARY_DIR}/{{ depend_subdir | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ source_dir | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ binary_dir | relative | replaceBackslashes }}" ) {% endfor %} From efb829fa8ff09f996216c4134e2017ed8c8b7d5e Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 11:46:22 +0100 Subject: [PATCH 33/54] fix add_own_subdirs --- yotta/lib/templates/base_CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index 37abf744..8bfb9c68 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -126,7 +126,7 @@ add_subdirectory( {% for srcdir, workingdir in add_own_subdirs %} add_subdirectory( "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" - "${CMAKE_BINARY_DIR}/{{ workingdir | replaceBackslashes }}" + "${CMAKE_CURRENT_BINARY_DIR}/{{ workingdir | replaceBackslashes }}" ) {% endfor %} {% endif %} From d90625c75339addc1bd9315f1f00fd65f931095e Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 14:24:14 +0100 Subject: [PATCH 34/54] improve tempfile handling --- yotta/test/cli/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/yotta/test/cli/util.py b/yotta/test/cli/util.py index ec5056e0..d058f28f 100644 --- a/yotta/test/cli/util.py +++ b/yotta/test/cli/util.py @@ -278,12 +278,10 @@ def writeTestFiles(files, add_space_in_path=False, test_dir=None): ''' write a dictionary of filename:contents into a new temporary directory ''' if test_dir is None: - test_dir = tempfile.mkdtemp() - if add_space_in_path: - test_dir = test_dir + ' spaces in path' + test_dir = tempfile.mkdtemp(suffix=' spaces in path' if add_space_in_path else '') for path, contents in files.items(): - path_dir, file_name = os.path.split(path) + path_dir, file_name = os.path.split(path) path_dir = os.path.join(test_dir, path_dir) fsutils.mkDirP(path_dir) with open(os.path.join(path_dir, file_name), 'w') as f: From 066b16458b664a8ace95e51c53089d14d246e3a7 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 14:24:53 +0100 Subject: [PATCH 35/54] add a test for export behaviour --- yotta/lib/paths.py | 3 +- yotta/test/cli/test_build.py | 70 +++++++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index 793aa131..faa0f766 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -7,6 +7,7 @@ # this module to provide tools for path manipulation and caching BUILD_OUTPUT_KEY = 'folder' +DEFAULT_BUILD_DIR = 'build' PARSER_OUTPUT_FOLDER_KWARGS = dict Modules_Folder = 'yotta_modules' Targets_Folder = 'yotta_targets' @@ -39,7 +40,7 @@ def get_configured_output_path(args, target=None): # else organise directories by target name in cwd if not path: - path = os.path.join(current, 'build') + path = os.path.join(current, DEFAULT_BUILD_DIR) if target: path = os.path.join(path, target.getName()) diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index fe80c8d2..0bc08882 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -5,6 +5,7 @@ # See LICENSE file for details. # standard library modules, , , +import os import unittest import subprocess import copy @@ -14,6 +15,7 @@ # internal modules: from yotta.test.cli import cli from yotta.test.cli import util +from yotta.lib import paths Test_Complex = { 'module.json': '''{ @@ -48,17 +50,6 @@ } ''', -'source/a.c': ''' -#include "a/a.h" -#include "b/b.h" -#include "c/c.h" -#include "d/d.h" - -int a(){ - return 1 + b() + c() + d(); // 35 -} -''', - 'a/a.h':''' #ifndef __A_H__ #define __A_H__ @@ -663,7 +654,6 @@ def test_Defines_Application(self): self.assertIn("1234 yotta", output) util.rmRf(test_dir) - @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") def test_Defines_Library(self): test_dir = util.writeTestFiles(Test_Defines_Library) @@ -687,6 +677,62 @@ def test_Toplevel_Library(self): self.assertIn("42", output) util.rmRf(test_dir) + @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") + def test_exportSpaceInPath(self): + """ + export should dump all the CMake files into a specified directory + the user should then be able to do their builds in that directory + :return: + """ + test_dir = util.writeTestFiles(util.Test_Trivial_Exe, True) + export_dir = util.writeTestFiles({}, True) + + # export a generate-only output to an alternate directory + stdout = self.runCheckCommand(['--target', util.nativeTarget(), 'build', '-x', export_dir, '-g'], test_dir) + + # search for any reference to the original directory + # /tmp/tmpUo7pB0 spaces in path --> tmpUo7pB0 + try: + output = subprocess.check_output( + ['grep', '-r', '-n', '-e', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0]], + cwd=export_dir + ).decode() + except subprocess.CalledProcessError as e: + # grep returns 1 for 'nothing found' and 2 for 'error' + # https://www.gnu.org/software/grep/manual/grep.html#Exit-Status + self.assertEqual(e.returncode, 1) + else: + raise Exception('should not have found anything') + + # build the project + built_dir = os.path.join(export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(',')) + try: + output = subprocess.check_output( + ['cmake', '-G', 'Ninja'], + cwd=built_dir + ).decode() + except subprocess.CalledProcessError as e: + print e.output + raise + + # link the project + output = subprocess.check_output( + ['ninja'], + cwd=built_dir + ).decode() + + # run the project + output = subprocess.check_output( + ['./test-trivial-exe'], + cwd=os.path.join(built_dir, 'source') + ).decode() + + # it works! + self.assertIn('trivial-exe-running', output) + + util.rmRf(test_dir) + util.rmRf(export_dir) + def runCheckCommand(self, args, test_dir): stdout, stderr, statuscode = cli.run(args, cwd=test_dir) if statuscode != 0: From 71d683d65882af53d20d462005db19ab21a366b0 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 14:36:43 +0100 Subject: [PATCH 36/54] python3 print statement --- yotta/test/cli/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 0bc08882..c86c622c 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -712,7 +712,7 @@ def test_exportSpaceInPath(self): cwd=built_dir ).decode() except subprocess.CalledProcessError as e: - print e.output + print(e.output) raise # link the project From ce7e45865f2e00c96521fa05b046aaf5de80c37f Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 14:51:38 +0100 Subject: [PATCH 37/54] inconsistent docs --- docs/tutorial/yotta_link.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorial/yotta_link.md b/docs/tutorial/yotta_link.md index 817a92a3..eae47272 100644 --- a/docs/tutorial/yotta_link.md +++ b/docs/tutorial/yotta_link.md @@ -50,8 +50,8 @@ info: generate for target: x86-osx-native 0.0.7 at /path/to/helloyotta/yotta_tar -- Configuring done -- Generating done -- Build files have been written to: /path/to/helloyotta/build/x86-osx-native -[1/3] Building C object ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -FAILED: /usr/bin/cc -I/path/to/helloyotta -I/path/to/helloyotta/yotta_modules/simplelog -I/path/to/helloyotta/yotta_modules/simplelog/source -O2 -g -DNDEBUG -include "/path/to/helloyotta/build/x86-osx-native/yotta_config.h" -MMD -MT modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -MF ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o.d -o ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c +[1/3] Building C object modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o +FAILED: /usr/bin/cc -I/path/to/helloyotta -I/path/to/helloyotta/yotta_modules/simplelog -I/path/to/helloyotta/yotta_modules/simplelog/source -O2 -g -DNDEBUG -include "/path/to/helloyotta/build/x86-osx-native/yotta_config.h" -MMD -MT modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -MF modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o.d -o modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c:25:50: error: expected ';' after expression printf("%s %s\n", prefixForLevel(level), msg) // deliberately missing semicolon ^ From 4da98611b3ce487ef2aa9b05254d0dd7dd9571ff Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 15:19:06 +0100 Subject: [PATCH 38/54] remove paths & dockerfile for this branch --- .dockerignore | 7 ----- dockerfile | 73 ---------------------------------------------- yotta/build.py | 3 +- yotta/clean.py | 25 ++++------------ yotta/debug.py | 3 +- yotta/lib/paths.py | 47 +---------------------------- yotta/start.py | 3 +- 7 files changed, 9 insertions(+), 152 deletions(-) delete mode 100644 .dockerignore delete mode 100644 dockerfile diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 6b08601d..00000000 --- a/.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -# things to ignore when copying this source directory into the docker image -dockerfile -.dockerignore -**/.idea -**/.vscode -**/.DS_Store -*.git diff --git a/dockerfile b/dockerfile deleted file mode 100644 index 8ee2bc4e..00000000 --- a/dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -FROM i686/ubuntu - -RUN apt-get -yqq update && apt-get -yqq install \ - curl \ - git \ - nano \ - ninja-build \ - python \ - python-dev \ - python-distribute \ - python-pip \ -&& rm -rf /var/lib/apt/lists/* - -RUN python -m pip install -U pip - -# install yotta development dependencies -RUN pip install -q \ - tox \ - cython \ - nose \ - pylint \ - coverage \ - green \ - ipython - -# fix up cmake -RUN apt-get remove -yq cmake - -# FIXME: 64 or 32? -RUN curl -fsSL http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-i386.sh > /tmp/install-cmake.sh -#RUN curl -fsSL https://cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh > /tmp/install-cmake.sh - -RUN chmod +x /tmp/install-cmake.sh -RUN /tmp/install-cmake.sh --prefix=/usr/local/ --exclude-subdir - -ENV PYTHONIOENCODING: UTF-8 -ENV YOTTA_GITHUB_AUTHTOKEN: 8d1dfa2011f74b1f26504918982e1e2ba154b910 - -# FIXME: "CMAKE not installed" Maybe a difference between Ubuntu and CircleCI? -ENV PATH="/usr/local:${PATH}" - -WORKDIR yt_testbuild - -# add setup.py &tc (for the dependencies) first, to preserve docker cache -ADD setup.py . -ADD yotta/version.txt yotta/version.txt -ADD pypi_readme.rst . -ADD bin/ bin/ - -# install yotta dependencies -RUN pip install -q -e . - -# prepare tox in the same way as CircleCI -ADD tox.ini . -RUN tox --notest - -# metadata -LABEL description="An image for devtesting Yotta in a stable environment. One day it might replace the CircleCI script." -LABEL incantation="docker build -t yotta . && docker run --rm --name yotta -v your_yotta_dir:/yt_testbuild -it yotta /bin/bash" - -# development workflow - this bit now optional if you do a volume mount -ADD . . - -RUN ls -lah - -RUN bash ci_init.sh - -# TODO: volume mount -# You (the user) must mount your working directory into the image using the -v command at run time, because: -# "...you can’t mount a host directory from within the Dockerfile. -# The VOLUME instruction does not support specifying a host-dir parameter. -# You must specify the mountpoint when you create or run the container." -# - https://docs.docker.com/engine/reference/builder/#notes-about-specifying-volumes diff --git a/yotta/build.py b/yotta/build.py index f10b5d37..d449cf49 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -40,7 +40,6 @@ def addOptions(parser, add_build_targets=True): '[optional] Specify an output path to receive the clean build' ) ) - paths.add_parser_argument(parser) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) parser.add_argument('-d', '--debug-build', dest='release_build', action='store_false', default=True) # the target class adds its own build-system specific options. In the @@ -133,7 +132,7 @@ def installAndBuild(args, following_args): # version specs), which it will display install_status = install.execCommand(args, []) - builddir = paths.get_configured_output_path(args, target) + builddir = os.path.join(os.getcwd(), paths.DEFAULT_BUILD_DIR, target.getName()) all_deps = c.getDependenciesRecursive( target = target, diff --git a/yotta/clean.py b/yotta/clean.py index 801ccf52..3c16ca2b 100644 --- a/yotta/clean.py +++ b/yotta/clean.py @@ -5,7 +5,6 @@ # standard library modules, , , import os -import logging # validate, , validate things, internal from yotta.lib import validate @@ -13,26 +12,12 @@ # fsutils, , misc filesystem utils, internal from yotta.lib import fsutils -# paths -from yotta.lib import paths - - def addOptions(parser): - paths.add_parser_argument(parser) - + pass def execCommand(args, following_args): - paths_to_remove = [] - - current = validate.currentDirectoryModule() - if current: - paths_to_remove.append(os.path.join(current.path, 'build')) - - specific = paths.get_configured_output_path(args) - if specific: - paths_to_remove.append(specific) + c = validate.currentDirectoryModule() + if not c: + return 1 - for path in paths_to_remove: - if os.path.exists(path) and not validate.directoryModule(path): - logging.info('removing: %s' % path) - fsutils.rmRf(path) + fsutils.rmRf(os.path.join(c.path, 'build')) diff --git a/yotta/debug.py b/yotta/debug.py index 170bc320..2b9a485f 100644 --- a/yotta/debug.py +++ b/yotta/debug.py @@ -22,7 +22,6 @@ def addOptions(parser): parser.add_argument('program', default=None, nargs='?', help='name of the program to be debugged' ) - paths.add_parser_argument(parser) def execCommand(args, following_args): @@ -38,7 +37,7 @@ def execCommand(args, following_args): logging.error(error) return 1 - builddir = paths.get_configured_output_path(args, target) + builddir = os.path.join(os.getcwd(), paths.DEFAULT_BUILD_DIR, target.getName()) if args.program is None: if c.isApplication(): diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index faa0f766..0c28305c 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -5,54 +5,9 @@ import os -# this module to provide tools for path manipulation and caching +# this module to provide tools for path manipulation and caching, avoiding circular imports BUILD_OUTPUT_KEY = 'folder' DEFAULT_BUILD_DIR = 'build' PARSER_OUTPUT_FOLDER_KWARGS = dict Modules_Folder = 'yotta_modules' Targets_Folder = 'yotta_targets' - - -def add_parser_argument(parser): - parser.add_argument( - '-o', - '--output-folder', - dest='output_folder', - default=None, - help=( - 'Customise the build directory (default: ./build/)' - 'Must be inside the project directory.' - ), - metavar='path/to/output/folder' - ) - - -def get_configured_output_path(args, target=None): - '''common method for setting/loading/defaulting a build output path''' - from yotta.lib import settings - - args = vars(args) - current = os.getcwd() - from_settings = settings.getProperty('build', BUILD_OUTPUT_KEY) - - # load from command line or local config - path = args.get('output_folder') or from_settings - - # else organise directories by target name in cwd - if not path: - path = os.path.join(current, DEFAULT_BUILD_DIR) - if target: - path = os.path.join(path, target.getName()) - - path = os.path.relpath(path, start=current) - - if not os.path.abspath(path).startswith(current): - raise ValueError( - 'The build output directory must be a subdirectory to this project' - ' (requested: %s)' - % path - ) - - if path != from_settings: - settings.setProperty('build', BUILD_OUTPUT_KEY, path, True) - return path diff --git a/yotta/start.py b/yotta/start.py index 001e43bc..8977be1c 100644 --- a/yotta/start.py +++ b/yotta/start.py @@ -22,7 +22,6 @@ def addOptions(parser): parser.add_argument('program', default=None, nargs='?', help='name of the program to be started' ) - paths.add_parser_argument(parser) def execCommand(args, following_args): @@ -42,7 +41,7 @@ def execCommand(args, following_args): logging.error(error) return 1 - builddir = paths.get_configured_output_path(args, target) + builddir = os.path.join(cwd, 'build', target.getName()) if args.program is None: # if no program was specified, default to the name of the executable From c235600f8fc1f35b7ff0ece160d9762a5b616327 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 15:20:25 +0100 Subject: [PATCH 39/54] circleci: give tox more time --- circle.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index c6017601..25996f7e 100644 --- a/circle.yml +++ b/circle.yml @@ -30,7 +30,8 @@ dependencies: test: override: - - tox + - tox: + timeout: 1800 post: - mkdir -p $CIRCLE_TEST_REPORTS/junit From 398dbb08fec862f74dec6be75d392fa60a75315d Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 15:40:59 +0100 Subject: [PATCH 40/54] fix for noop test for relative/non-relative paths --- yotta/test/cli/test_debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/test/cli/test_debug.py b/yotta/test/cli/test_debug.py index a87db364..87c301cd 100644 --- a/yotta/test/cli/test_debug.py +++ b/yotta/test/cli/test_debug.py @@ -48,7 +48,7 @@ def test_noop_debug(self): target_name = 'debug-test-target' test_dir = util.writeTestFiles(util.Test_Trivial_Exe, True) target_dir = os.path.realpath(os.path.join(test_dir, 'yotta_targets', target_name)) - build_dir = os.path.join('build', target_name) + build_dir = os.path.realpath(os.path.join(test_dir, 'build', target_name)) util.writeTestFiles(_nopDebugTargetDescription(target_name), test_dir=target_dir) output = util.runCheckCommand(['--target', target_name, 'build'], test_dir) From 29429160225f81864ef55020bffd07fcc0c0879d Mon Sep 17 00:00:00 2001 From: David Hyman Date: Thu, 14 Sep 2017 18:08:54 +0100 Subject: [PATCH 41/54] re-add `ym` as a default for backwards-compatibility --- .gitignore | 2 ++ yotta/build.py | 10 ++++++---- yotta/lib/cmakegen.py | 2 +- yotta/lib/paths.py | 1 + yotta/test/cli/test_build.py | 24 ++++++++++++++++++++++++ yotta/test_subcommand.py | 5 +++-- 6 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e596b33b..6fba8b13 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ git-access-testing github-access-testing *~ *.idea +.tox +*nosetests.xml diff --git a/yotta/build.py b/yotta/build.py index d449cf49..6e097b91 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -77,19 +77,21 @@ def installAndBuild(args, following_args): if export_mode_or_path: logging.info('this build will be configured for exporting') - new_modules = settings.getProperty('build', 'modules_directory_name') or 'modules' - new_targets = settings.getProperty('build', 'targets_directory_name') or 'targets' + new_modules = settings.getProperty('build', 'vendor_modules_directory') or 'vendor_modules' + new_targets = settings.getProperty('build', 'vendor_targets_directory') or 'vendor_targets' + new_build_modules = settings.getProperty('build', 'built_modules_directory') or 'modules' # to save downloading the cached files, copy the old cache try: if not os.path.exists(new_modules): - shutil.copytree('yotta_modules', new_modules) - shutil.copytree('yotta_targets', new_targets) + shutil.copytree(paths.Modules_Folder, new_modules) + shutil.copytree(paths.Targets_Folder, new_targets) except Exception as e: logging.error(e) logging.error('failed to use existing cache; rebuilding') paths.Modules_Folder = new_modules paths.Targets_Folder = new_targets + paths.BUILT_MODULES_DIR = new_build_modules args_dict.setdefault('build_targets', []) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 8c9053a6..c0fdfa77 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -123,7 +123,7 @@ def generateRecursive(self, component, all_components, builddir=None, modbuilddi if builddir is None: builddir = self.buildroot if modbuilddir is None: - modbuilddir = os.path.join(builddir, 'modules') + modbuilddir = os.path.join(builddir, paths.BUILT_MODULES_DIR) if processed_components is None: processed_components = dict() diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index 0c28305c..63ee53dd 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -8,6 +8,7 @@ # this module to provide tools for path manipulation and caching, avoiding circular imports BUILD_OUTPUT_KEY = 'folder' DEFAULT_BUILD_DIR = 'build' +BUILT_MODULES_DIR = 'ym' PARSER_OUTPUT_FOLDER_KWARGS = dict Modules_Folder = 'yotta_modules' Targets_Folder = 'yotta_targets' diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index c86c622c..6615567f 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -677,6 +677,30 @@ def test_Toplevel_Library(self): self.assertIn("42", output) util.rmRf(test_dir) + @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") + def test_exportIsClean(self): + """ + exported directory doesn't contain old references to yotta + :return: + """ + test_dir = util.writeTestFiles(util.Test_Complex_Lib, True) + export_dir = util.writeTestFiles({}, True) + + # export a generate-only output to an alternate directory + stdout = self.runCheckCommand(['--target', util.nativeTarget(), 'build', '-x', export_dir, '-g'], test_dir) + self.assertFalse(os.path.exists(os.path.join(export_dir, paths.Modules_Folder))) + self.assertFalse(os.path.exists(os.path.join(export_dir, paths.Targets_Folder))) + self.assertFalse(os.path.exists(os.path.join( + export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(','), paths.BUILT_MODULES_DIR + ))) + + # duplicate the new strings (for brittleness?) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'vendor_modules'))) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'vendor_targets'))) + self.assertTrue(os.path.exists(os.path.join( + export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(','), 'modules' + ))) + @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") def test_exportSpaceInPath(self): """ diff --git a/yotta/test_subcommand.py b/yotta/test_subcommand.py index 025558f7..148ba9f3 100644 --- a/yotta/test_subcommand.py +++ b/yotta/test_subcommand.py @@ -18,6 +18,7 @@ from yotta import build # --config option, , , internal from yotta import options +from yotta.lib import paths def addOptions(parser): @@ -51,7 +52,7 @@ def findCTests(builddir, recurse_yotta_modules=False): add_test_re = re.compile('add_test\\(([^" ]*)\s*"(.*)"\\)', flags=re.IGNORECASE) for root, dirs, files in os.walk(builddir, topdown=True): if not recurse_yotta_modules: - dirs = [d for d in dirs if d != 'modules'] + dirs = [d for d in dirs if d != paths.BUILT_MODULES_DIR] if 'CTestTestfile.cmake' in files: with open(os.path.join(root, 'CTestTestfile.cmake'), 'r') as ctestf: dir_tests = [] @@ -81,7 +82,7 @@ def moduleFromDirname(build_subdir, all_modules, toplevel_module): modtop = True submod = False else: - if part == 'modules' and modtop: + if part == paths.BUILT_MODULES_DIR and modtop: submod = True else: submod = False From 28e046eb92f707734126d354ec08dc1e41de1e56 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 10:05:14 +0100 Subject: [PATCH 42/54] timeouts on test export subprocess commands --- setup.py | 3 ++- yotta/test/cli/test_build.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 56f89e66..7fee7127 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ def read(fname): 'mbed_test_wrapper>=0.0.3,<2.0.0', 'valinor>=0.0.0,<1.0', 'intelhex>=2.0,<3.0', - 'jsonpointer>=1.9,<2.0' + 'jsonpointer>=1.9,<2.0', + 'subprocess32>=3.2.7<4.0' ] + platform_deps ) diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 6615567f..79acd1ce 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -7,16 +7,22 @@ # standard library modules, , , import os import unittest -import subprocess import copy import re import datetime +import sys # internal modules: from yotta.test.cli import cli from yotta.test.cli import util from yotta.lib import paths +# doesn't work on windows - but we'll not use the functionality for those tests +if os.name == 'posix' and sys.version_info[0] < 3: + import subprocess32 as subprocess +else: + import subprocess + Test_Complex = { 'module.json': '''{ "name": "test-testdep-a", @@ -719,7 +725,8 @@ def test_exportSpaceInPath(self): try: output = subprocess.check_output( ['grep', '-r', '-n', '-e', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0]], - cwd=export_dir + cwd=export_dir, + timeout=5 ).decode() except subprocess.CalledProcessError as e: # grep returns 1 for 'nothing found' and 2 for 'error' @@ -733,7 +740,8 @@ def test_exportSpaceInPath(self): try: output = subprocess.check_output( ['cmake', '-G', 'Ninja'], - cwd=built_dir + cwd=built_dir, + timeout=5 ).decode() except subprocess.CalledProcessError as e: print(e.output) @@ -742,13 +750,15 @@ def test_exportSpaceInPath(self): # link the project output = subprocess.check_output( ['ninja'], - cwd=built_dir + cwd=built_dir, + timeout=5 ).decode() # run the project output = subprocess.check_output( ['./test-trivial-exe'], - cwd=os.path.join(built_dir, 'source') + cwd=os.path.join(built_dir, 'source'), + timeout=5 ).decode() # it works! From 9b61e1a7f851189985d7c7deea5566400fd88974 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 10:23:45 +0100 Subject: [PATCH 43/54] speculative grep fix --- yotta/test/cli/test_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 79acd1ce..a71f9c90 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -723,8 +723,9 @@ def test_exportSpaceInPath(self): # search for any reference to the original directory # /tmp/tmpUo7pB0 spaces in path --> tmpUo7pB0 try: + # grep; skip sockets, recursive, phrase, directory, output = subprocess.check_output( - ['grep', '-r', '-n', '-e', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0]], + ['grep', '-D', 'skip', '-r', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0], '.'], cwd=export_dir, timeout=5 ).decode() From ba14037289ae601abe5941d3cac7c89ad0d65cd1 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 10:29:05 +0100 Subject: [PATCH 44/54] fix test_test_subcommand --- yotta/lib/paths.py | 4 ---- yotta/test/cli/test_build.py | 2 +- yotta/test/test_test_subcommand.py | 35 +++++++++++++++++++++--------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index 63ee53dd..42fb23a1 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -3,12 +3,8 @@ # Licensed under the Apache License, Version 2.0 # See LICENSE file for details. -import os - # this module to provide tools for path manipulation and caching, avoiding circular imports -BUILD_OUTPUT_KEY = 'folder' DEFAULT_BUILD_DIR = 'build' BUILT_MODULES_DIR = 'ym' -PARSER_OUTPUT_FOLDER_KWARGS = dict Modules_Folder = 'yotta_modules' Targets_Folder = 'yotta_targets' diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index a71f9c90..0b3ab522 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -723,7 +723,7 @@ def test_exportSpaceInPath(self): # search for any reference to the original directory # /tmp/tmpUo7pB0 spaces in path --> tmpUo7pB0 try: - # grep; skip sockets, recursive, phrase, directory, + # grep; skip sockets, recursive, phrase, directory output = subprocess.check_output( ['grep', '-D', 'skip', '-r', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0], '.'], cwd=export_dir, diff --git a/yotta/test/test_test_subcommand.py b/yotta/test/test_test_subcommand.py index 51ddb584..07cbfa34 100644 --- a/yotta/test/test_test_subcommand.py +++ b/yotta/test/test_test_subcommand.py @@ -8,18 +8,33 @@ # standard library modules, , , import unittest - # module to test: from yotta import test_subcommand +from yotta.lib import paths + class TestTestSubcommandModule(unittest.TestCase): def test_moduleFromDirname(self): - self.assertEqual(test_subcommand.moduleFromDirname('modules/b/modules/c/d', {'b':'b', 'c':'c'}, 'a') , 'c') - self.assertEqual(test_subcommand.moduleFromDirname('modules/b/q/c/d', {'b':'b', 'c':'c'}, 'a') , 'b') - self.assertEqual(test_subcommand.moduleFromDirname('z/b/q/c/d', {'b':'b', 'c':'c'}, 'a') , 'a') - self.assertEqual(test_subcommand.moduleFromDirname('modules/e/d', {'b':'b', 'c':'c'}, 'a') , 'a') - self.assertEqual(test_subcommand.moduleFromDirname('modules/e/d', {'b':'b', 'c':'c', 'e':'e'}, 'a') , 'e') - - # see also yotta/test/cli/test.py for cli-driven testing - - + # Fstrings would be handy here + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/b/{m}/c/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'c' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/b/q/c/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'b' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + 'z/b/q/c/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'a' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/e/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'a' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/e/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c', 'e': 'e'}, 'a'), + 'e' + ) + + # see also yotta/test/cli/test.py for cli-driven testing From bed35b823b71808101a1dc72b738b25319f92a22 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 10:32:15 +0100 Subject: [PATCH 45/54] remove the subprocess32 stuff --- setup.py | 3 +-- yotta/test/cli/test_build.py | 18 ++++-------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 7fee7127..56f89e66 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,6 @@ def read(fname): 'mbed_test_wrapper>=0.0.3,<2.0.0', 'valinor>=0.0.0,<1.0', 'intelhex>=2.0,<3.0', - 'jsonpointer>=1.9,<2.0', - 'subprocess32>=3.2.7<4.0' + 'jsonpointer>=1.9,<2.0' ] + platform_deps ) diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 0b3ab522..6a8e09c3 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -17,12 +17,6 @@ from yotta.test.cli import util from yotta.lib import paths -# doesn't work on windows - but we'll not use the functionality for those tests -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - Test_Complex = { 'module.json': '''{ "name": "test-testdep-a", @@ -726,8 +720,7 @@ def test_exportSpaceInPath(self): # grep; skip sockets, recursive, phrase, directory output = subprocess.check_output( ['grep', '-D', 'skip', '-r', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0], '.'], - cwd=export_dir, - timeout=5 + cwd=export_dir ).decode() except subprocess.CalledProcessError as e: # grep returns 1 for 'nothing found' and 2 for 'error' @@ -741,8 +734,7 @@ def test_exportSpaceInPath(self): try: output = subprocess.check_output( ['cmake', '-G', 'Ninja'], - cwd=built_dir, - timeout=5 + cwd=built_dir ).decode() except subprocess.CalledProcessError as e: print(e.output) @@ -751,15 +743,13 @@ def test_exportSpaceInPath(self): # link the project output = subprocess.check_output( ['ninja'], - cwd=built_dir, - timeout=5 + cwd=built_dir ).decode() # run the project output = subprocess.check_output( ['./test-trivial-exe'], - cwd=os.path.join(built_dir, 'source'), - timeout=5 + cwd=os.path.join(built_dir, 'source') ).decode() # it works! From 0557159c4893116d4d8b4458e3407e074ef55dae Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 10:58:38 +0100 Subject: [PATCH 46/54] oops, need that bit --- yotta/test/cli/test_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 6a8e09c3..7d6ac59e 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -10,7 +10,8 @@ import copy import re import datetime -import sys +import subprocess + # internal modules: from yotta.test.cli import cli From da8f1889ac10b29c9858413ed8afa2ac4577dc52 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 11:29:56 +0100 Subject: [PATCH 47/54] unused imports --- yotta/debug.py | 1 - yotta/start.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/yotta/debug.py b/yotta/debug.py index 2b9a485f..1d7ad7a6 100644 --- a/yotta/debug.py +++ b/yotta/debug.py @@ -10,7 +10,6 @@ # validate, , validate things, internal from yotta.lib import validate # settings, , load and save settings, internal -from yotta.lib import settings # paths from yotta.lib import paths # --config option, , , internal diff --git a/yotta/start.py b/yotta/start.py index 8977be1c..c7ae50f7 100644 --- a/yotta/start.py +++ b/yotta/start.py @@ -10,9 +10,6 @@ # validate, , validate things, internal from yotta.lib import validate # settings, , load and save settings, internal -from yotta.lib import settings -# paths -from yotta.lib import paths # --config option, , , internal from yotta import options From b87ac7f2bc7d6e376e724f186ec162519e1ff336 Mon Sep 17 00:00:00 2001 From: Rob Moran Date: Fri, 15 Sep 2017 12:54:32 +0200 Subject: [PATCH 48/54] drop tox timeout --- circle.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 25996f7e..c6017601 100644 --- a/circle.yml +++ b/circle.yml @@ -30,8 +30,7 @@ dependencies: test: override: - - tox: - timeout: 1800 + - tox post: - mkdir -p $CIRCLE_TEST_REPORTS/junit From a5bed51ce71df3e7c6d47232b274e4e87995ccca Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 12:17:33 +0100 Subject: [PATCH 49/54] revert references to ym --- docs/tutorial/yotta_link.md | 12 ++++++------ yotta/test_subcommand.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/tutorial/yotta_link.md b/docs/tutorial/yotta_link.md index eae47272..41912c8b 100644 --- a/docs/tutorial/yotta_link.md +++ b/docs/tutorial/yotta_link.md @@ -50,8 +50,8 @@ info: generate for target: x86-osx-native 0.0.7 at /path/to/helloyotta/yotta_tar -- Configuring done -- Generating done -- Build files have been written to: /path/to/helloyotta/build/x86-osx-native -[1/3] Building C object modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -FAILED: /usr/bin/cc -I/path/to/helloyotta -I/path/to/helloyotta/yotta_modules/simplelog -I/path/to/helloyotta/yotta_modules/simplelog/source -O2 -g -DNDEBUG -include "/path/to/helloyotta/build/x86-osx-native/yotta_config.h" -MMD -MT modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -MF modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o.d -o modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c +[1/3] Building C object ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o +FAILED: /usr/bin/cc -I/path/to/helloyotta -I/path/to/helloyotta/yotta_modules/simplelog -I/path/to/helloyotta/yotta_modules/simplelog/source -O2 -g -DNDEBUG -include "/path/to/helloyotta/build/x86-osx-native/yotta_config.h" -MMD -MT ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -MF ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o.d -o ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o -c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c /path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c:25:50: error: expected ';' after expression printf("%s %s\n", prefixForLevel(level), msg) // deliberately missing semicolon ^ @@ -64,14 +64,14 @@ error: command ['ninja'] failed Here we can see that the file that was being built when the error occurred was: ``` -[1/3] Building C object modules/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o +[1/3] Building C object ym/simplelog/source/CMakeFiles/simplelog.dir/path/to/helloyotta/yotta_modules/simplelog/source/simplelog.c.o ``` -This is a path relative to the build directory, the `modules` directory is the +This is a path relative to the build directory, the `ym` directory is the subdirectory where dependencies are built, and is always followed immediately -by a dependency name (if there were no `modules/` at the start of the path then the +by a dependency name (if there were no `ym/` at the start of the path then the object would belong to the top-level module/app being built). So from -`/modules/simplelog/...` we can tell that the failed file is in the `simplelog` +`/ym/simplelog/...` we can tell that the failed file is in the `simplelog` dependency. This is also visible from the source file where the error is reported: diff --git a/yotta/test_subcommand.py b/yotta/test_subcommand.py index 148ba9f3..e8835705 100644 --- a/yotta/test_subcommand.py +++ b/yotta/test_subcommand.py @@ -74,7 +74,7 @@ def moduleFromDirname(build_subdir, all_modules, toplevel_module): modtop = True submod = False module = toplevel_module - # /modules//modules//somedir/somedir --> submod2 + # /ym//ym//somedir/somedir --> submod2 for part in fsutils.fullySplitPath(build_subdir): if submod: if part in all_modules: From 0313a5127764c1a768cf4ef5ad849223379b1bf2 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Fri, 15 Sep 2017 14:57:59 +0100 Subject: [PATCH 50/54] tweak export helpstring --- yotta/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index 6e097b91..72a97896 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -36,8 +36,7 @@ def addOptions(parser, add_build_targets=True): parser.add_argument('-x', '--export', dest='export', default=False, const=True, nargs='?', help=( - 'Export mode. If flag is set, generates builds without reference to Yotta.' - '[optional] Specify an output path to receive the clean build' + 'Export mode. If flag is set, generates a buildable directory structure at the specified path.' ) ) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) From caff88d9172d694c703c0bd551714e626aaafcea Mon Sep 17 00:00:00 2001 From: David Hyman Date: Mon, 18 Sep 2017 11:02:05 +0100 Subject: [PATCH 51/54] Add a couple of references to `export` in the docs --- docs/reference/buildsystem.md | 16 ++++++++++++++++ docs/reference/commands.md | 2 ++ 2 files changed, 18 insertions(+) diff --git a/docs/reference/buildsystem.md b/docs/reference/buildsystem.md index e728c9f2..81a2c5eb 100644 --- a/docs/reference/buildsystem.md +++ b/docs/reference/buildsystem.md @@ -273,3 +273,19 @@ subdirectory. Within this directory build products are further divided by the name of the [target](tutorial/targets.html) being built. This makes it safe to switch between building for different targets without cleaning. + +## # Exporting Builds + +Yotta can export the generated file structure to be later built elsewhere using: + +``` +yotta build -g -x some/output/path +``` + +In this case `-g` is used to stop the local compile from happening immediately, +and `-x` copies the build files to the specified directory. + +To build and link your project you would then have to run the cmake commands +against that output directory. The generated files use relative paths, +making them portable; for example to send to an independent build server. + diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 3ac8dd1b..7f7086d7 100755 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -80,6 +80,8 @@ Options: This option is deprecated because it is now the default, unless `--debug-build` is specified. + * **`--export`, `-x`**: export mode. If flag is set, generates a buildable directory structure at the specified path. + * **`--cmake-generator`, `-G`**: specify the CMake Generator. CMake can generate project files for various editors and IDEs. * **`name ...`**: one or more modules may be specified, in which case only these From e42079f87819d8734f211d3a06fd12b4271f70d5 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 19 Sep 2017 13:21:55 +0100 Subject: [PATCH 52/54] 'vendor_modules' -> 'modules' + black->white list --- yotta/build.py | 29 +++++++++++++++++------------ yotta/test/cli/test_build.py | 4 ++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index a4c0b861..b5f77b0f 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -77,8 +77,8 @@ def installAndBuild(args, following_args): if export_mode_or_path: logging.info('this build will be configured for exporting') - new_modules = settings.getProperty('build', 'vendor_modules_directory') or 'vendor_modules' - new_targets = settings.getProperty('build', 'vendor_targets_directory') or 'vendor_targets' + new_modules = settings.getProperty('build', 'vendor_modules_directory') or 'modules' + new_targets = settings.getProperty('build', 'vendor_targets_directory') or 'targets' new_build_modules = settings.getProperty('build', 'built_modules_directory') or 'modules' # to save downloading the cached files, copy the old cache try: @@ -173,16 +173,21 @@ def installAndBuild(args, following_args): runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) if export_mode_or_path and export_mode_or_path is not True: - # these export exclusions apply at any level of the directory structure - excludes = { - '.module.json', - '.yotta.json', - '.yotta_origin.json', - 'module.json', - 'target.json', - 'yotta_modules', - 'yotta_targets' + # includes relative to cwd / project top level + includes = { + 'source', + paths.DEFAULT_BUILD_DIR, + paths.Modules_Folder, + paths.Targets_Folder, + 'yotta-cloud-client', # not sure about whitelisting... where is this coming from? } + + def ignore_on_copy(current_dir, local_paths): + # only at the project root + if current_dir == '.': + # exclude = !include + return [p for p in local_paths if p not in includes] + return [] export_to = os.path.abspath(export_mode_or_path) logging.info('exporting unbuilt project source and dependencies to %s', export_to) check = os.path.join(export_to, 'source') # this might be insufficient, or indeed misleading. @@ -194,7 +199,7 @@ def installAndBuild(args, following_args): % (check) ) fsutils.rmRf(export_to) - shutil.copytree(src='.', dst=export_to, ignore=lambda a, b: excludes) + shutil.copytree(src='.', dst=export_to, ignore=ignore_on_copy) if args_dict.get('generate_only'): logging.info('skipping build step') diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 7d6ac59e..583e4e45 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -696,8 +696,8 @@ def test_exportIsClean(self): ))) # duplicate the new strings (for brittleness?) - self.assertTrue(os.path.exists(os.path.join(export_dir, 'vendor_modules'))) - self.assertTrue(os.path.exists(os.path.join(export_dir, 'vendor_targets'))) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'modules'))) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'targets'))) self.assertTrue(os.path.exists(os.path.join( export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(','), 'modules' ))) From 09cc9a3e08e596ee9bac94a2941ad701c47e105a Mon Sep 17 00:00:00 2001 From: David Hyman Date: Tue, 19 Sep 2017 16:37:45 +0100 Subject: [PATCH 53/54] i) tidy up after exporting ii) builddir for export closer to top level --- yotta/build.py | 8 ++++++-- yotta/lib/paths.py | 1 + yotta/test/cli/test_build.py | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/yotta/build.py b/yotta/build.py index b5f77b0f..deb9a771 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -134,7 +134,8 @@ def installAndBuild(args, following_args): # version specs), which it will display install_status = install.execCommand(args, []) - builddir = os.path.join(os.getcwd(), paths.DEFAULT_BUILD_DIR, target.getName()) + builddir = os.path.join(os.getcwd(), paths.DEFAULT_BUILD_DIR, target.getName()) if not export_mode_or_path else \ + os.path.join(os.getcwd(), paths.DEFAULT_EXPORT_BUILD_RELPATH) all_deps = c.getDependenciesRecursive( target = target, @@ -176,7 +177,7 @@ def installAndBuild(args, following_args): # includes relative to cwd / project top level includes = { 'source', - paths.DEFAULT_BUILD_DIR, + paths.DEFAULT_EXPORT_BUILD_RELPATH, paths.Modules_Folder, paths.Targets_Folder, 'yotta-cloud-client', # not sure about whitelisting... where is this coming from? @@ -200,6 +201,9 @@ def ignore_on_copy(current_dir, local_paths): ) fsutils.rmRf(export_to) shutil.copytree(src='.', dst=export_to, ignore=ignore_on_copy) + fsutils.rmRf(builddir) + fsutils.rmRf(paths.Modules_Folder) + fsutils.rmRf(paths.Targets_Folder) if args_dict.get('generate_only'): logging.info('skipping build step') diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py index 42fb23a1..a61a4184 100644 --- a/yotta/lib/paths.py +++ b/yotta/lib/paths.py @@ -4,6 +4,7 @@ # See LICENSE file for details. # this module to provide tools for path manipulation and caching, avoiding circular imports +DEFAULT_EXPORT_BUILD_RELPATH = 'export' DEFAULT_BUILD_DIR = 'build' BUILT_MODULES_DIR = 'ym' Modules_Folder = 'yotta_modules' diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index 583e4e45..2cc15459 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -692,14 +692,14 @@ def test_exportIsClean(self): self.assertFalse(os.path.exists(os.path.join(export_dir, paths.Modules_Folder))) self.assertFalse(os.path.exists(os.path.join(export_dir, paths.Targets_Folder))) self.assertFalse(os.path.exists(os.path.join( - export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(','), paths.BUILT_MODULES_DIR + export_dir, paths.DEFAULT_EXPORT_BUILD_RELPATH, paths.BUILT_MODULES_DIR ))) # duplicate the new strings (for brittleness?) self.assertTrue(os.path.exists(os.path.join(export_dir, 'modules'))) self.assertTrue(os.path.exists(os.path.join(export_dir, 'targets'))) self.assertTrue(os.path.exists(os.path.join( - export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(','), 'modules' + export_dir, paths.DEFAULT_EXPORT_BUILD_RELPATH, 'modules' ))) @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") @@ -731,7 +731,7 @@ def test_exportSpaceInPath(self): raise Exception('should not have found anything') # build the project - built_dir = os.path.join(export_dir, paths.DEFAULT_BUILD_DIR, util.nativeTarget().rstrip(',')) + built_dir = os.path.join(export_dir, paths.DEFAULT_EXPORT_BUILD_RELPATH) try: output = subprocess.check_output( ['cmake', '-G', 'Ninja'], From e1f139996b5ef987c677b1062fed85fbe4b1bb46 Mon Sep 17 00:00:00 2001 From: David Hyman Date: Mon, 4 Dec 2017 17:36:04 +0000 Subject: [PATCH 54/54] Fix for extraneous quotes Add changes from @Sami Haapakoski --- yotta/lib/cmakegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index c0fdfa77..0ca3c05a 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -481,7 +481,7 @@ def generate( include_root_dirs = [] if application is not None and component is not application: - include_root_dirs.append('include_directories("%s")\n' % replaceBackslashes(application.path)) + include_root_dirs.append(application.path) include_sys_dirs = [] include_other_dirs = []