Merge pull request #363 from gyorb/tmsblgh-master

Updated OSX support

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 
 *.py[cod]
 
@@ -20,10 +21,10 @@ thrift_api/gen-js/
 thrift_api/gen-py/
 
 # test_files build files
-tests/test_projects/test_files/src/call_and_message.o
-tests/test_projects/test_files/src/divide_zero.o
-tests/test_projects/test_files/src/new_delete.o
-tests/test_projects/test_files/src/null_dereference.o
-tests/test_projects/test_files/src/stack_address_escape.o
-tests/test_projects/test_files/src/file_to_be_skipped.o
+tests/test_projects/test_files/call_and_message.o
+tests/test_projects/test_files/divide_zero.o
+tests/test_projects/test_files/new_delete.o
+tests/test_projects/test_files/null_dereference.o
+tests/test_projects/test_files/stack_address_escape.o
+tests/test_projects/test_files/file_to_be_skipped.o
 

.travis.yml

@@ -1,14 +1,39 @@
-sudo: required
-dist: trusty
+language: 
+    - python
 
-language: python
+services:
+    - postgresql
 
-python:
-    - "2.7"
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      dist: trusty
+      python: "2.7"
+    - os: osx
+      osx_image: xcode6.4
+      sudo: false
+      language: generic
+
+before_install:
+    - git clone https://github.com/llvm-mirror/clang.git ~/llvm\
+    - chmod a+x ~/llvm/tools/scan-build-py/bin/intercept-build;
+    - export PATH=~/llvm/tools/scan-build-py/bin:$PATH;
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update;                        fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install python;                fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install thrift;                fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install doxygen;               fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install coreutils;             fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl http://llvm.org/releases/3.7.0/clang+llvm-3.7.0-x86_64-apple-darwin.tar.xz -o clang+llvm-3.7.0-x86_64-apple-darwin.tar.xz; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then tar xf clang+llvm-3.7.0-x86_64-apple-darwin.tar.xz -C ~/; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=~/clang+llvm-3.7.0-x86_64-apple-darwin/bin/:$PATH; fi
 
 install:
     - pip install -r ./.ci/python_requirements
     - pip install nose
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PG_DATA=$(brew --prefix)/var/postgres; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pg_ctl -w start -l postgres.log --pgdata ${PG_DATA}; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then createuser -s postgres && psql -c 'create database travis_ci_test;' -U postgres && cat postgres.log; fi
 
 addons:
     apt:
@@ -29,7 +54,7 @@ addons:
 script:
     - ./change_clang_version.py
 
-    - . tests/test_env_setup.sh
+    - source tests/test_env_setup.sh
 
     - ./build_package.py -o $TEST_CODECHECKER_PACKAGE_DIR -v
 

README.md

@@ -1,4 +1,3 @@
-
 [![Build Status](https://travis-ci.org/Ericsson/codechecker.svg?branch=master)](https://travis-ci.org/Ericsson/codechecker)
 -----
 # Introduction
@@ -60,6 +59,56 @@ pip install -r .ci/basic_python_requirements
 cd ..
 ~~~~~~
 
+## OS X
+
+### Basic dependecy install & setup
+Tested on OS X El Capitan 10.11.5 & macOS Sierra 10.12 Beta
+~~~~~~{.sh}
+
+# on El Capitan System Integrity Protection (SIP) need to turn off
+- Click the  (Apple) menu.
+- Select Restart...
+- Hold down command-R to boot into the Recovery System.
+- Click the Utilities menu and select Terminal.
+- Type csrutil disable and press return.
+- Close the Terminal app.
+- Click the  (Apple) menu and select Restart....
+
+# check out and build clang with extra tools
+How to: http://clang.llvm.org/get_started.html
+
+# get dependencies
+brew update
+brew install doxygen thrift gcc git
+
+# get source code
+git clone https://github.com/tmsblgh/codechecker-osx-migration.git
+cd codechecker
+
+# install required basic python modules
+pip install -r .ci/basic_python_requirements
+
+# create codechecker package
+./build_package.py -o ~/codechecker_package
+cd ..
+~~~~~~
+
+### Check a test project
+#### Check if clang or clang tidy is available
+~~~~~~{.sh}
+which clang
+which clang-tidy
+~~~~~~
+If 'clang' or 'clang-tidy' commands are not available the package can be configured to use another/newer clang binary for the analisys.  
+Edit the 'CodeChecker/config/package_layout.json' config files "runtime/analyzers"
+section in the generated package and modify the analyzers section to the analyzers
+available in the PATH
+```
+"analyzers" : {
+  "clangsa" : "clang",
+  "clang-tidy" : "clang-tidy"
+  },
+```
 
 ### Check a test project
 #### Check if clang or clang tidy is available
@@ -88,6 +137,14 @@ This step can be skipped if you always give the path of CodeChecker command.
 ~~~~~~{.sh}
 export PATH=~/codechecker_package/CodeChecker/bin:$PATH
 ~~~~~~
+Scan-build-py (intercept-build)
+~~~~~~{.sh}
+export PATH=~/{user path}/llvm/tools/clang/tools/scan-build-py/bin:$PATH
+~~~~~~
+Clang
+~~~~~~{.sh}
+export PATH=~/{user path}/build/bin:$PATH
+~~~~~~
 
 #### Check the project
 Check the project using SQLite. The database is placed in the working

build_package.py

@@ -17,6 +17,9 @@
 import subprocess
 import time
 import shlex
+import platform
+
+from distutils.spawn import find_executable
 
 LOG = logging.getLogger('Packager')
 
@@ -190,6 +193,13 @@ def handle_external_file(dep, clean, env, verbose):
     download_cmd.extend(shlex.split(source_package['download_cmd']))
     file_url = source_package['url']
     download_cmd.append(file_url)
+
+    option = source_package['option']
+    download_cmd.append(option)
+
+    file_name = source_package['name']
+    download_cmd.append(file_name)
+
     LOG.info('Downloading ...')
     if run_cmd(download_cmd, directory, env, verbose):
         LOG.error('Failed to get dependency')
@@ -207,6 +217,7 @@ def handle_external_file(dep, clean, env, verbose):
             file_name = os.path.join(directory, file_name)
             with tarfile.open(file_name) as tar:
                 tar.extractall(directory)
+            os.remove(file_name)
         else:
             LOG.error('Unsupported file type')
     elif file_ext in supported_exts['uncompressed']:
@@ -354,42 +365,53 @@ def build_package(repository_root, build_package_config, env=None):
     # create package folder layout
     create_folder_layout(output_dir, package_layout)
 
-    # build ld logger
-    ld_logger_path = build_package_config['ld_logger_path']
-    if ld_logger_path:
-        ld_logger_build = os.path.join(ld_logger_path, 'build')
-
-        ld_logger32 = build_package_config.get('ld_logger_32')
-        ld_logger64 = build_package_config.get('ld_logger_64')
-        rebuild = build_package_config.get('rebuild_ld_logger') or clean
+    # check scan-build-py (intercept)
+    LOG.info('Checking source: llvm scan-build-py (intercept)')
 
-        arch = None
-        if ld_logger32 == ld_logger64:
-            # build both versions
-            pass
-        elif ld_logger32:
-            arch = '32'
-        elif ld_logger64:
-            arch = '64'
-
-        if build_ld_logger(ld_logger_path, env, arch, rebuild, verbose):
-            LOG.error('Failed to build ld logger')
-            sys.exit()
+    intercept_build_executable = find_executable('intercept-build')
+
+    if intercept_build_executable != None:
+        LOG.info('Available')
+    else:
+        LOG.error('Not exists, build ld logger in Linux only')
+        # build ld logger because intercept is not available
+        if platform.system() == 'Linux':
+            ld_logger_path = build_package_config['ld_logger_path']
+            if ld_logger_path:
+                ld_logger_build = os.path.join(ld_logger_path, 'build')
+
+                ld_logger32 = build_package_config.get('ld_logger_32')
+                ld_logger64 = build_package_config.get('ld_logger_64')
+                rebuild = build_package_config.get('rebuild_ld_logger') or clean
+
+                arch = None
+                if ld_logger32 == ld_logger64:
+                    # build both versions
+                    pass
+                elif ld_logger32:
+                    arch = '32'
+                elif ld_logger64:
+                    arch = '64'
+
+                if build_ld_logger(ld_logger_path, env, arch, rebuild, verbose):
+                    LOG.error('Failed to build ld logger')
+                    sys.exit()
 
-        # copy ld logger files
-        target = os.path.join(package_root, package_layout['ld_logger'])
+                # copy ld logger files
+                target = os.path.join(package_root, package_layout['ld_logger'])
 
-        copy_tree(ld_logger_build, target)
+                copy_tree(ld_logger_build, target)
 
-        curr_dir = os.getcwd()
-        os.chdir(os.path.join(package_root, package_layout['bin']))
-        logger_symlink = os.path.join('../', package_layout['ld_logger'],
-                                      'bin', 'ldlogger')
-        os.symlink(logger_symlink, 'ldlogger')
-        os.chdir(curr_dir)
+                curr_dir = os.getcwd()
+                os.chdir(os.path.join(package_root, package_layout['bin']))
+                logger_symlink = os.path.join('../', package_layout['ld_logger'],
+                                              'bin', 'ldlogger')
+                os.symlink(logger_symlink, 'ldlogger')
+                os.chdir(curr_dir)
 
-    else:
-        LOG.info('Skipping ld logger from package')
+            else:
+                LOG.info('Skipping ld logger from package')
+
 
     # generate gen files with thrift
     thrift_files_dir = os.path.join(repository_root, 'thrift_api')

change_clang_version.py

@@ -1,13 +1,17 @@
 #!/usr/bin/env python
 
 import json
+import platform
 
 data = None
 
 with open('config/package_layout.json') as f:
     data = json.load(f)
 
-data['runtime']['analyzers']['clangsa'] = 'clang-3.7'
+if platform.system() == 'Linux':
+    data['runtime']['analyzers']['clangsa'] = 'clang-3.7'
+if platform.system() == 'Darwin':
+    data['runtime']['analyzers']['clangsa'] = 'clang'
 
 with open('config/package_layout.json', 'w') as f:
     json.dump(data, f, indent=4)

codechecker_lib/analysis_manager.py

@@ -71,6 +71,7 @@ def check(check_data):
     try:
         # if one analysis fails the check fails
         return_codes = 0
+        skipped = False
         for source in action.sources:
 
             # if there is no skiplist handler there was no skip list file

codechecker_lib/build_manager.py

@@ -10,9 +10,14 @@
 import pickle
 import subprocess
 import sys
+import shutil
+import platform
 
 from codechecker_lib import logger
 from codechecker_lib import analyzer_env
+from codechecker_lib import host_check
+
+from distutils.spawn import find_executable
 
 LOG = logger.get_new_logger('BUILD MANAGER')
 
@@ -60,11 +65,26 @@ def perform_build_command(logfile, command, context, silent=False):
         original_env = os.environ.copy()
 
     return_code = 0
-    # Run user's commands in shell
-    log_env = analyzer_env.get_log_env(logfile, context, original_env)
 
-    if 'CC_LOGGER_GCC_LIKE' not in log_env:
-      log_env['CC_LOGGER_GCC_LIKE'] = 'gcc:g++:clang:clang++:cc:c++'
+    # Run user's commands with intercept
+    if host_check.check_intercept(original_env):
+        LOG.debug_analyzer("with intercept ...")
+        final_command = command
+        command = "intercept-build " + "--cdb "+ logfile + " " + final_command
+        log_env = original_env
+        LOG.debug_analyzer(command)
+
+    # Run user's commands in shell
+    else:
+        # TODO better platform detection
+        if platform.system() == 'Linux':
+            LOG.debug_analyzer("with ld logger ...")
+            log_env = analyzer_env.get_log_env(logfile, context, original_env)
+            if 'CC_LOGGER_GCC_LIKE' not in log_env:
+                log_env['CC_LOGGER_GCC_LIKE'] = 'gcc:g++:clang:clang++:cc:c++'
+        else:
+            LOG.error("Intercept-build is required to run CodeChecker in OS X.")
+            sys.exit(1)
 
     LOG.debug_analyzer(log_env)
     try:
@@ -124,16 +144,21 @@ def generate_log_file(args, context, silent=False):
     log_file = None
     try:
         if args.command:
-            # check if logger bin exists
-            if not os.path.isfile(context.path_logger_bin):
-                LOG.debug_analyzer('Logger binary not found! Required for logging.')
-                sys.exit(1)
-
-            # check if logger lib exists
-            if not os.path.exists(context.path_logger_lib):
-                LOG.debug_analyzer('Logger library directory not found! Libs are requires' \
-                          'for logging.')
-                sys.exit(1)
+
+            intercept_build_executable = find_executable('intercept-build')
+
+            if intercept_build_executable == None:
+                if platform.system() == 'Linux':
+                    # check if logger bin exists
+                    if not os.path.isfile(context.path_logger_bin):
+                        LOG.error('Logger binary not found! Required for logging.')
+                        sys.exit(1)
+
+                    # check if logger lib exists
+                    if not os.path.exists(context.path_logger_lib):
+                        LOG.error('Logger library directory not found! Libs are requires' \
+                                  'for logging.')
+                        sys.exit(1)
 
             log_file = default_compilation_db(args.workspace)
             if os.path.exists(log_file):