mixdown

#! /usr/bin/env python

# Copyright (c) 2010-2014, Lawrence Livermore National Security, LLC
# Produced at Lawrence Livermore National Laboratory
# LLNL-CODE-462894
# All rights reserved.
#
# This file is part of MixDown. Please read the COPYRIGHT file
# for Our Notice and the LICENSE file for the GNU Lesser General Public
# License.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License (as published by
# the Free Software Foundation) version 3 dated June 2007.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import multiprocessing, os, sys, tarfile, time, urllib
from md import commands, defines, exceptions, importer, logger, options, overrides, profiler, project, utilityFunctions

#--------------------------------Main---------------------------------
def main():
    logger.setLogger("console")
    logger.writeMessage("MixDown - A tool to simplify building\n")
    try:
        success = True
        partialImport = False
        mdOptions = options.Options()
        mdProject = None

        if not mdOptions.processCommandline(sys.argv):
            sys.exit(1)

        timeStarted = time.time()

        if mdOptions.importMode:
            mdProject, partialImport = importer.importTargets(mdOptions)
            if mdProject == None or partialImport:
                success = False
        elif mdOptions.profileMode:
            success = profiler.profile(mdOptions)
        else:
            if mdOptions.cleanMixDown:
                cleanMixDown(mdOptions)

            mdProject = setup(mdOptions)
            if mdProject == None:
                success = False
            else:
                mdOptions.setStatusLogPath(mdOptions.defines[defines.mdPrefix[0]], mdProject.name)
                if mdOptions.restart:
                    readSuccess, statusLogExisted = mdProject.readStatusLog(mdOptions)
                    if statusLogExisted:
                        if not readSuccess:
                            logger.writeError("Reading previous project status log failed, starting build at beginning.")
                        else:
                            mdProject.determineStartPoint()

                #Define Hierarchy: options defines <  project defines < override group defines < command line defines
                mdOptions.defines.combine(mdProject.defines)
                if mdOptions.overrideGroup:
                    mdOptions.defines.combine(mdOptions.overrideGroup)
                mdOptions.defines.combine(mdOptions.commandLineDefines)

                if mdOptions.threadCount == 1:
                    for target in reversed(mdProject.targets):
                        commands.buildTarget(target, mdOptions, mdProject)
                        if target.success == False:
                            success = False
                            if not mdOptions.continueBuilding:
                                break
                else:
                    lock = multiprocessing.RLock()
                    jobQueue = multiprocessing.Queue()
                    resultQueue = multiprocessing.Queue()
                    maxDepth = mdProject.targets[len(mdProject.targets)-1].dependencyDepth
                    for currDepth in range(maxDepth, -1, -1):
                        processes = []
                        jobCount = 0

                        for target in reversed(mdProject.targets):
                            if target.dependencyDepth == currDepth:
                                jobQueue.put(target)
                                jobCount += 1

                        if jobCount < mdOptions.threadCount:
                            processCount = jobCount
                        else:
                            processCount = mdOptions.threadCount

                        for _ in range(processCount):
                            jobQueue.put(None)
                        for _ in range(processCount):
                            p = multiprocessing.Process(target=commands.buildTargetThreaded, args=(jobQueue,resultQueue,mdOptions,mdProject,lock,))
                            processes.append(p)

                        for p in processes:
                            p.start()
                        for p in processes:
                            p.join()

                        while not resultQueue.empty():
                            resultTarget = resultQueue.get(False)
                            mdProject.replaceTarget(resultTarget)

                        for target in reversed(mdProject.targets):
                            if target.dependencyDepth == currDepth and not target.success:
                                if jobCount != 1:
                                    logger.writeError(target.name + " failed.  Due to threading, you may need to scroll up to read error message.")
                                success = False
                        if not success:
                            break

        timeFinished = time.time()
        timeElapsed = timeFinished - timeStarted

        if mdOptions.profileMode:
            message = "Profile"
        else:
            message = "Project"
        if mdProject != None:
            message += " " + mdProject.name

        if mdProject != None and not mdOptions.importMode and not mdOptions.cleanMode and not mdOptions.profileMode:
            mdProject.writeStatusLog(mdOptions)
            if mdOptions.continueBuilding:
                targetsSkippedCommandLine = []
                targetsSuccess = []
                targetsDependancyFailed = []
                targetsFailed = []
                for t in mdProject.targets:
                    if not mdOptions.targetSpecifiedToBuild(t.name):
                        targetsSkippedCommandLine.append(t.name)
                    elif t.success == True:
                        targetsSuccess.append(t.name)
                    elif t.skippedDueToDependanciesFailing:
                        targetsDependancyFailed.append(t.name)
                    else:
                        targetsFailed.append(t.name)

                if len(targetsSuccess) > 0:
                    logger.writeMessage("\nSuccess:\n{0}".format(" ".join(targetsSuccess)))
                if len(targetsFailed) > 0:
                    logger.writeMessage("\nFailed:\n{0}".format(" ".join(targetsFailed)))
                if len(targetsDependancyFailed) > 0:
                    logger.writeMessage("\nSkipped due to failed dependancy:\n{0}".format(" ".join(targetsDependancyFailed)))
                if len(targetsSkippedCommandLine) > 0:
                    logger.writeMessage("\nSkipped due to command line option:\n{0}".format(" ".join(targetsSkippedCommandLine)))

        if not success:
            message += " failed."
        else:
            message += " success."
            if mdOptions.importMode:
                logger.writeMessage("\nProject " + mdProject.name + " has been imported to file " + mdProject.path)
            elif mdOptions.cleanMode:
                logger.writeMessage("\n" + mdProject.name + " has been cleaned in " + os.path.abspath(mdOptions.buildDir))
            elif mdOptions.profileMode:
                logger.writeMessage("\nComputer has been profiled to file " + os.path.abspath(mdOptions.overrideFile))
            else:
                logger.writeMessage("\n" + mdProject.name + " has been built in " + os.path.abspath(mdOptions.buildDir))
                logger.writeMessage(mdProject.name + " has been installed to " + os.path.abspath(mdOptions.defines[defines.mdPrefix[0]]))
        message = "\nTotal time " + logger.secondsToHMS(timeElapsed) + "\n" + message
        logger.writeMessage(message)
        if partialImport:
            logger.writeError("\nProject partially imported.  Please refer to comments in created MixDown project file to determine reason.")
    except exceptions.ToolNotInstalledException, e:
        logger.writeError("%s not found." % e.tool)
        logger.writeError("This project requires that %s is installed on your system." % e.tool)
        success = False
    finally:
        logger.close()
    if success:
        sys.exit()
    else:
        sys.exit(1)

#--------------------------------Setup---------------------------------
def setup(mdOptions):
    logger.setLogger(mdOptions.logger, mdOptions.logDir)
    if not mdOptions.prefixDefined and not mdOptions.cleanMode:
        logger.writeMessage("No prefix defined, defaulting to '" + mdOptions.defines[defines.mdPrefix[0]] + "'")

    if mdOptions.projectFile == "":
        logger.writeMessage("Project file was not specified on command-line.\nTrying to find project file...")
        cwd = os.path.abspath(os.getcwd())
        projectFiles = utilityFunctions.findFilesWithExtension(cwd, ".md")
        if len(projectFiles) == 0:
            logger.writeError("No project file ('.md') found in current working directory. Must specifiy on command line.", filePath=cwd)
            sys.exit(1)
        if len(projectFiles) > 1:
            logger.writeError("More than one project file ('.md') found in current working directory. Must specifiy on command line.", filePath=cwd)
            sys.exit(1)
        mdOptions.projectFile = projectFiles[0]
        logger.writeMessage("Found project file: " + mdOptions.projectFile + "\n")

    if mdOptions.overrideFile == "" and len(mdOptions.overrideGroupNames) > 0:
        logger.writeMessage("Override file was not specified while override group name was specified.\nTrying to find override file...")
        cwd = os.path.abspath(os.getcwd())
        overrideFiles = utilityFunctions.findFilesWithExtension(cwd, ".mdo")
        if len(overrideFiles) == 0:
            logger.writeError("No override file ('.mdo') found in current working directory. Must specifiy '-o' on command line.", filePath=cwd)
            sys.exit(1)
        if len(overrideFiles) > 1:
            logger.writeError("More than one override file ('.mdo') found in current working directory. Must specifiy '-o' on command line.", filePath=cwd)
            sys.exit(1)
        mdOptions.overrideFile = overrideFiles[0]
        logger.writeMessage("Found override file: " + mdOptions.overrideFile + "\n")

    if mdOptions.verbose:
        logger.writeMessage(str(mdOptions))

    if mdOptions.overrideFile != "":
        overrideGroups = overrides.readGroups(mdOptions.overrideFile)
        if overrideGroups == None:
            sys.exit(1)
        finalGroup = overrides.selectGroups(overrideGroups, mdOptions.overrideGroupNames)
        if finalGroup == None:
            logger.writeError("Selecting Override group failed.")
            sys.exit(1)
        else:
            mdOptions.overrideGroup = finalGroup
            defines.setOverrideDefines(mdOptions.defines, finalGroup)

    if not mdOptions.validate():
        return None

    mdProject = project.Project(mdOptions.projectFile)
    if not mdProject.read():
        return None
    if not mdProject.addSkipStepFromOptions(mdOptions):
        return None
    mdProject.setTargetFieldsAsDefines(mdOptions.defines)

    if not mdProject.examine(mdOptions):
        return None
    if not mdProject.validate(mdOptions):
        return None

    if mdOptions.cleanMode:
        for currTarget in mdProject.targets:
            currTarget.path = currTarget.determineOutputPath(mdOptions)
    else:
        cleaningOutputReported = False
        for currTarget in mdProject.targets:
            if currTarget.outputPath != "" and os.path.exists(currTarget.outputPath):
                if cleaningOutputReported:
                    logger.writeMessage("Cleaning MixDown and Target output directories...")
                    cleaningOutputReported = True
                utilityFunctions.removeDir(currTarget.outputPath)

        prefixDefine = mdOptions.defines[defines.mdPrefix[0]]
        if prefixDefine != "":
            #TODO: only add lib64 if on 64bit machines
            libraryPaths = os.path.join(prefixDefine, "lib64") + ":" + os.path.join(prefixDefine, "lib")
            if os.environ.has_key("LD_LIBRARY_PATH"):
                originalLibraryPath = str.strip(os.environ["LD_LIBRARY_PATH"])
                if originalLibraryPath != "":
                    libraryPaths += ":" + originalLibraryPath
            os.environ["LD_LIBRARY_PATH"] = libraryPaths

    return mdProject

def cleanMixDown(mdOptions):
    try:
        logger.writeMessage("Cleaning MixDown directories...\n")
        utilityFunctions.removeDir(mdOptions.buildDir)
        utilityFunctions.removeDir(mdOptions.downloadDir)
        utilityFunctions.removeDir(mdOptions.logDir)
    except IOError, e:
        logger.writeError(e, exitProgram=True)

#----------------------------------------------------------------------

if __name__ == "__main__":
    main()