main.scons

# Main Ironclad build script
#
# Builds one configuration/framework/platform combination.
# By default the build is in-source (intermediate build artifacts are placed next to source files).
# The build output is placed in directory 'build'.
# Recognizes one command-line build variable 'MODE'
# which can have values 'release' (default) or 'debug'
# To build in-source in debug MODE use:
# scons -f main.scons MODE=debug


#===============================================================================
# Various useful functions

import functools
import operator, os, sys
import subprocess
from pathlib import Path
from SCons.Scanner.C import CScanner

EnsurePythonVersion(3, 9)

def splitstring(f):
    """if f decorated by @splitstring: f(x, "a b c", k=v) == f(x, ["a", "b", "c"], k=v)"""
    def g(_, s, **kwargs):
        if isinstance(s, str):
            s = s.split()
        return f(_, s, **kwargs)
    return g

@splitstring
def glommap(f, inputs, **kwargs):
    """glommap(f, "a b c", k=v) == [*f("a", k=v), *f("b", k=v), *f("c", k=v)]"""
    def g(input):
        return list(f(input, **kwargs))
    return list(functools.reduce(operator.add, map(g, inputs), []))

@splitstring
def pathmap(base, files):
    """pathmap("path/to", "a b c") == ["path/to/a", "path/to/b", "path/to/c"]"""
    return list(map(lambda x: os.path.join(base, x), files))

@splitstring
def submap(template, inserts):
    """submap("klm %s xyz", "a b c") == ["klm a xyz", "klm b xyz", "klm c xyz"]"""
    return list(map(lambda x: template % x, inserts))


#===============================================================================
# BUILD CONFIGURATION

MODE = ARGUMENTS.get('mode', 'release')
if not (MODE in ['debug', 'release']):
   print("Error: expected 'debug' or 'release', found: " + MODE)
   Exit(1)
FMWK = 'net462' if sys.platform == 'win32' else 'net6.0'

PROJECT_DIR = Dir('#').abspath              # project root, where all commands are run
BUILD_DIR = Dir('#').rel_path(Dir('.'))     # for intermediate build artifacts (in-source by default)
OUT_DIR = 'build'                           # for build output (final) artifacts (default location)
Import('*')                                 # can override aby of the variables above


#===============================================================================
# PLATFORM-SPECIFIC GLOBALS

WIN32  = sys.platform == 'win32'
LINUX  = sys.platform == 'linux'
DARWIN = sys.platform == 'darwin'

if WIN32:
    #==================================================================
    # These variables are needed on any platform

    RID = 'win-x64'
    NATIVE_TOOLS = 'msvc mslink as'
    PYD_SUFFIX = '.pyd'
    DOTNET_FLAGS = ''

    ASFLAGS = '-f win64'

    # where to find/how to invoke executables
    NASM = R'nasm'
    CASTXML = R'castxml'
    DOTNET = R'dotnet'

    IPY = Path(f"{PROJECT_DIR}/IronPython.3.4.1/{FMWK}/ipy.exe")  # try to use private debug build
    if not IPY.exists():
        # use standard location
        if FMWK.startswith('net4'):
            IPY = Path("C:/ProgramData/chocolatey/lib/ironpython/ipy.exe")
        else:
            IPY = Path("~/.dotnet/tools/ipy.exe").expanduser()
    IPY = f'"{IPY.resolve()}"'

    #==================================================================
    # These variables should only be necessary on win32
    # (except for CPYTHON34_DLL and CPYTHON34_DLL_NAME, which are needed on any plaftorm)

    MSVCR100_DLL = R'C:\Windows\System32\msvcr100.dll'

    GENDEF_CMD = 'gendef - $SOURCE >$TARGET'
    DLLTOOL_CMD = 'dlltool -D $NAME -d $SOURCE -l $TARGET'
    PEXPORTS_CMD = 'pexports $SOURCE > $TARGET'
    RES_CMD = 'windres --input $SOURCE --output $TARGET --output-format=coff'

    # Find root of CPython installation, used for exports generation and to find DLLs/packages for testing
    # Note: this has to be 64-bit version of CPython 3.4
    CPYTHON34_ROOT = Path(subprocess.check_output(['py.exe', '-3.4-64', '-c', 'import sys, os; print(os.path.dirname(os.path.abspath(sys.executable)))'], universal_newlines=True).strip())
    # If it fails, change to match your installation; by default it is C:\Python34
    #CPYTHON34_ROOT = Path(R'C:\Python34')

    # Find Python DLL
    CPYTHON34_DLL_NAME = Path('python34.dll')
    CPYTHON34_DLL = CPYTHON34_ROOT / CPYTHON34_DLL_NAME
    if not CPYTHON34_DLL.exists():
        # it can be somewhere else
        CPYTHON34_DLL = Path(os.environ['SystemRoot']) / "System32" / CPYTHON34_DLL_NAME
    if not CPYTHON34_DLL.exists():
        print(f"error: {CPYTHON34_DLL_NAME} not found")
        Exit(1)

    # Check if we have MSVC 16.0 and Win7 SDK
    VC16_INSTALLDIR = R'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC'
    WINDOWS7_SDKDIR = R'C:\Program Files\Microsoft SDKs\Windows\v7.1'
    if USE_MSVC1600 and not (os.path.exists(VC16_INSTALLDIR + R'\bin\amd64') and os.path.exists(WINDOWS7_SDKDIR)):
        print('Cannot build using MSVC v16.0: installation of MSVC 16.0 and/or Windows 7 SDK not found')
        Exit(1)

elif LINUX:
    RID = 'linux-x64'
    NATIVE_TOOLS = 'clang link as'

    ASFLAGS = '-f elf64'

    # where to find/how to invoke executables
    NASM = 'nasm'
    CASTXML = 'castxml'
    DOTNET = 'dotnet'
    IPY = Path(f"/usr/src/ironpython-3.4.1/{FMWK}/ipy")  # try to use private build
    if not IPY.exists():
        # use standard location
        IPY = Path("~/.dotnet/tools/ipy").expanduser()
    if not IPY.exists():
        # try VSCode devcontainer location
        try: IPY = Path("~vscode/.dotnet/tools/ipy").expanduser()
        except RuntimeError: pass
    if not IPY.exists():
        # maybe it is on the executables' search path
        IPY = 'ipy'

    # Find CPython dynamic load library
    # Note: this has to be 64-bit version of CPython 3.4
    for CPYTHON34_LIBROOT in map(Path, '/usr/local/lib /opt/anaconda3/lib /usr/lib/x86_64-linux-gnu'.split()):
        for CPYTHON34_DLL_NAME in map(Path, 'libpython3.4.so libpython3.4m.so'.split()):
            CPYTHON34_DLL = CPYTHON34_LIBROOT / CPYTHON34_DLL_NAME
            if CPYTHON34_DLL.exists():
                break
        else:
            continue
        IS_ANACONDA = CPYTHON34_DLL_NAME.stem[-1] == 'm'
        break
    else:
        print(f"error: shared library for CPython 3.4 not found")
        Exit(1)

    PYD_SUFFIX =  '.cpython-34m.so' if IS_ANACONDA else '.cpython-34-x86_64-linux-gnu.so';
    DOTNET_FLAGS = f'--property:IsAnaconda={IS_ANACONDA}'

elif DARWIN:
    # TODO: darwin
    # RID = 'osx-x64' or 'osx-arm64'
    print("Platform", sys.platform, "not supported yet")
    Exit(1)

else:
    print("Platform", sys.platform, "not supported")
    Exit(1)


#===============================================================================
# PLATFORM-AGNOSTIC GLOBALS
# If any turn out to need to be platform-specific, please move them

CPYTHON = '"' + sys.executable + '"'  # Python used to run generators from "./tools"

OS_ENV = os.environ.copy()
OS_ENV['IRONPYTHONPATH'] = PROJECT_DIR
OS_ENV['PYTHONPATH'] = PROJECT_DIR
# TODO: it should not be necessary to pollute execution with entire os environment

MGD_DLL_SUFFIX = '.dll'
PDB_SUFFIX = '.pdb'

ASFLAGS = f'{ASFLAGS} -wall'
CPPDEFINES = 'Py_ENABLE_SHARED Py_BUILD_CORE IRONCLAD'
CPPPATH = 'stub/Include'
DOTNET_CMD = f'{DOTNET} %(cmd)s {DOTNET_FLAGS} --configuration {MODE.title()} --framework {FMWK} --output ${{TARGET.dir}} $SOURCE'
MGD_BUILD_CMD = DOTNET_CMD % {'cmd' : f'build --runtime {RID} --no-self-contained'}
MGD_CLEAN_CMD = DOTNET_CMD % {'cmd' : 'clean'}
CASTXML_CMD = f'{CASTXML} "$SOURCE" -o "$TARGET" --castxml-output=1 $CLANGFLAGS $CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS'

# COMMON are globals in all used environments (native, managed, tests, etc.)
COMMON = dict(CPYTHON=CPYTHON, IPY=IPY, BUILD_DIR=BUILD_DIR, OUT_DIR=OUT_DIR, PROJECT_DIR=PROJECT_DIR)

test_deps = []
before_test = test_deps.append


#===============================================================================
#===============================================================================
# This section builds all the unmanaged parts
# (python34.dll and several test files)
#===============================================================================

native = Environment(ENV=OS_ENV, tools=Split(NATIVE_TOOLS) + ['python'], AS=NASM, CPYTHON34_DLL=CPYTHON34_DLL, **COMMON)

if WIN32:
    native.Append(
        ASFLAGS         = Split(ASFLAGS),
        CPPDEFINES      = Split(CPPDEFINES),
        CPPPATH         = Split(R'stub\Include\PC') + Split(CPPPATH),
        CLANGFLAGS      = Split('--target=x86_64-pc-windows-msvc -fms-compatibility-version=16.00.40219'),  # used by clang-cl AND castxml
        CCFLAGS         = Split('/GS-'),
        LINKFLAGS       = Split('/subsystem:windows'),
        SHLINKFLAGS     = Split('$_DLL_ENTRYPOINT'),
        no_import_lib   = 1,  # undocumented SCons msvc tools flag; may be renamed in future
        LIBS            = Split('kernel32 user32'),
        _DLL_ENTRYPOINT = '${"/entry:" + entry if entry else "/noentry"}',
        entry           = '',  # override in SharedLibrary invocation to specify a DLL entry point if needed
    )
    if USE_MSVC1600:
        # Reconfigure environment to use MSVC 16.0 (from VS 10.0 aka 2010)
        native.PrependENVPath('PATH', WINDOWS7_SDKDIR + R'\Bin\x64')
        native.PrependENVPath('PATH', VC16_INSTALLDIR + R'\bin\amd64')
        native['ENV']['INCLUDE'] = ';'.join([
            VC16_INSTALLDIR + R'\include',
            WINDOWS7_SDKDIR + R'\Include',
        ])
        native['ENV']['LIB'] = ';'.join([
            VC16_INSTALLDIR + R'\lib\amd64',
            WINDOWS7_SDKDIR + R'\Lib\x64',
        ])
        native['ENV']['LIBPATH'] = VC16_INSTALLDIR + R'\lib\amd64'
        native.Append(
            LIBS=Split('msvcrt'),
            SHLINKFLAGS=Split('/ignore:4197'),
        )
    else:
        # Use Clang & lld
        native.Replace(CC='clang-cl', LINK='lld-link')
        native.Append(
            CCFLAGS=Split('$CLANGFLAGS') + Split('-fuse-ld=lld'),
            CPPDEFINES=Split('__MSVCRT_VERSION__=0x1000 _NO_CRT_STDIO_INLINE'),
            LINKFLAGS=Split('/nodefaultlib:libucrt /nodefaultlib:libcmt'),
            SHLINKFLAGS=Split('/noimplib'),
        )

    if MODE == 'debug':
        native.Replace(PDB='${TARGET.base}' + PDB_SUFFIX)
        native.Append(ASFLAGS='-g')
        native.Append(LINKFLAGS='/debug:full')

else:
    native.Append(
        ASFLAGS         = Split(ASFLAGS),
        CPPDEFINES      = Split(CPPDEFINES),
        CPPPATH         = Split('stub/Include/Linux') + Split(CPPPATH),
        CLANGFLAGS      = Split('--target=x86_64-pc-linux-gnu'),  # used by clang AND castxml
        CCFLAGS         = Split('$CLANGFLAGS') + Split('-pthread -fwrapv'),
        LINKFLAGS       = Split('-fuse-ld=lld'),
    )

    # Uncommenmt the following line to temporarily silence multiple definition errors
    # which can happen if new CPython code is added to the stub but _pure_c_sybols is not updated yet.
    # In such case, the code from the CPython stub files overrides the jumptable exports.
    #native.Append(LINKFLAGS='-Xlinker --allow-multiple-definition')

    if MODE == 'debug':
        native.Append(ASFLAGS='-g')
        native.Append(CCFLAGS='-g')
    else:
        native.Append(CCFLAGS='-O3')

    # TODO: darwin

native['BUILDERS']['CastXml'] = Builder(action=CASTXML_CMD, source_scanner=CScanner(), suffix='.xml', CPPDEFPREFIX='-D', INCPREFIX='-I')

#===============================================================================
# Unmanaged libraries for build/ironclad

if WIN32 and not USE_MSVC1600:
    # Create implib for msvcrt
    msvcrt_def = native.Command('stub/msvcr100.def', MSVCR100_DLL, GENDEF_CMD)
    msvcrt_lib = native.Command('stub/msvcr100.lib', msvcrt_def, DLLTOOL_CMD, NAME='msvcr100.dll')
else:
    msvcrt_lib = []

# Generate data from prebuilt python dll
*exports, python_def = native.Command(['data/api/_exported_functions.generated', 'data/api/_exported_data.generated', 'stub/python.def'], ['tools/generateexports.py'],
    '$CPYTHON $SOURCE $CPYTHON34_DLL $BUILD_DIR/data/api $BUILD_DIR/stub')

# Generate stub code
buildstub_names = '_extra_functions _mgd_api_data _pure_c_symbols'
buildstub_src = exports + pathmap('data/api', buildstub_names)
buildstub_out = pathmap('stub', 'jumps.generated.asm stubinit.generated.c Include/_extra_functions.generated.h')
native.Command(buildstub_out, ['tools/generatestub.py'] + buildstub_src,
    '$CPYTHON $SOURCE $BUILD_DIR/data/api $BUILD_DIR/stub')

# Compile stub code
jumps_obj = native.SharedObject('stub/jumps.generated.asm')
stubmain_obj = native.SharedObject('stub/stubmain.c')

# Generate information from python headers etc
stubmain_xml = native.CastXml('data/api/_stubmain.generated.xml', 'stub/stubmain.c')

# Build and link python34.dll
cpy_src_dirs = 'Modules Objects Parser Python'
cpy_srcs = glommap(lambda x: native.Glob('stub/%s/*.c' % x), cpy_src_dirs)
cpy_objs = glommap(native.SharedObject, cpy_srcs) + [jumps_obj, stubmain_obj]
if WIN32:
    cpy_objs += [msvcrt_lib, python_def]
before_test(native.SharedLibrary(f'#$OUT_DIR/ironclad/{CPYTHON34_DLL_NAME}', cpy_objs, entry='DllMain'))

if USE_MSVC1600:
    import atexit
    @atexit.register
    def remove_implib_files():
        Path(f"{OUT_DIR}/ironclad/python34.lib").unlink(missing_ok=True)
        Path(f"{OUT_DIR}/ironclad/python34.exp").unlink(missing_ok=True)

#===============================================================================
# Unmanaged test data

before_test(native.SharedLibrary('tests/data/setvalue', 'tests/data/src/setvalue.c', SHLIBPREFIX='', SHLIBSUFFIX=PYD_SUFFIX))
before_test(native.SharedLibrary('tests/data/exportsymbols', 'tests/data/src/exportsymbols.c'))
before_test(native.SharedLibrary('tests/data/fakepython', 'tests/data/src/fakepython.c',))

if WIN32:
    # Some tests will load and unload dlls which depend on msvcr100; if msvcr100's ref count
    # hits 0 and it gets reloaded, bad things happen. The test framework loads this dll, and
    # keeps it loaded, to prevent aforesaid bad things.
    before_test(native.SharedLibrary('tests/data/implicit-load-msvcr100', 'tests/data/src/empty.c'))


#===============================================================================
#===============================================================================
# This section builds the CLR part
#===============================================================================

managed = Environment(ENV=OS_ENV, tools=['python'], **COMMON)

def dotnet_emitter(target, source, env):
    return [[t, str(t).removesuffix(MGD_DLL_SUFFIX) + PDB_SUFFIX] for t in target], source
managed['BUILDERS']['Dll'] = Builder(action=MGD_BUILD_CMD, suffix=MGD_DLL_SUFFIX, emitter=dotnet_emitter)

#===============================================================================
# Generated C#

api_src = managed.Glob('data/api/*')
api_out_names = 'Delegates Dispatcher MagicMethods PythonApi PythonStructs'
api_out = pathmap('src', submap('%s.Generated.cs', api_out_names))
managed.Command(api_out, ['tools/generateapiplumbing.py'] + api_src,
    '$CPYTHON $SOURCE $BUILD_DIR/data/api $BUILD_DIR/src')

mapper_src = managed.Glob('data/mapper/_*')
mapper_out = submap('src/mapper/PythonMapper%s.Generated.cs', map(lambda x: x.name, mapper_src))
managed.Command(mapper_out, ['tools/generatemapper.py'] + mapper_src,
    '$CPYTHON $SOURCE $BUILD_DIR/data/mapper $BUILD_DIR/src/mapper')

snippets_src = managed.Glob('data/snippets/py/*.py')
snippets_out = ['src/CodeSnippets.Generated.cs']
managed.Command(snippets_out, ['tools/generatecodesnippets.py'] + snippets_src,
    '$CPYTHON $SOURCE $BUILD_DIR/data/snippets/py $BUILD_DIR/src')

#===============================================================================
# Build the actual managed library

ironclad_dll_src = list(map(managed.Glob, ('src/*.cs', 'src/mapper/*.cs')))
ironclad_dll = managed.Dll('#$OUT_DIR/ironclad/ironclad', 'src/ironclad.csproj')
managed.Depends(ironclad_dll, ironclad_dll_src)
before_test(ironclad_dll)

if managed.GetOption('clean'):
    # 'Execute' runs while reading the SCons script (before actual build)
    # and in the variant dir (BUILD_DIR)
    managed.Execute(managed.subst(MGD_CLEAN_CMD,
                                  target=File('$PROJECT_DIR/$OUT_DIR/ironclad/ironclad'),
                                  source=File('src/ironclad.csproj')))


#===============================================================================
#===============================================================================
# This section installs the package initialization module (__init__.py)
#===============================================================================

pkginit = Environment(tools=['filesystem'], **COMMON)
before_test(pkginit.CopyAs('#$OUT_DIR/ironclad/__init__.py', 'data/ironclad__init__.py'))


#===============================================================================
#===============================================================================
# This section runs the tests, assuming you've run 'scons test'
#===============================================================================

tests = Environment(ENV=OS_ENV, **COMMON)
tests['ENV']['TESTDATA_BUILDDIR'] = os.path.join(BUILD_DIR, "tests", "data")
tests.PrependENVPath('IRONPYTHONPATH', OUT_DIR)  # to find ironclad
if WIN32:
    tests.AppendENVPath('IRONPYTHONPATH', CPYTHON34_ROOT / "DLLs")  # required to import/access dlls
    tests.AppendENVPath('IRONPYTHONPATH', CPYTHON34_ROOT / "Lib" / "site-packages")  # pysvn test
else:
    tests.AppendENVPath('IRONPYTHONPATH', CPYTHON34_LIBROOT / "python3.4" / "lib-dynload")
    tests.AppendENVPath('IRONPYTHONPATH', CPYTHON34_LIBROOT / "python3.4" / "site-packages")

tests.AlwaysBuild(tests.Alias('test', test_deps,
    '$IPY runtests.py'))