Merge pull request #10716 from NREL/python_packaging

Initial Python App Packaging

.github/workflows/release_linux.yml

@@ -1,4 +1,4 @@
-name: Releases
+name: Linux Releases
 
 on:
   push:

.github/workflows/release_mac.yml

@@ -1,4 +1,4 @@
-name: Releases
+name: Mac Releases
 
 on:
   push:

.github/workflows/release_windows.yml

@@ -1,4 +1,4 @@
-name: Releases
+name: Windows Releases
 
 on:
   push:

cmake/PythonCopyStandardLib.py

@@ -61,7 +61,7 @@
 # this script must be called with two args:
 # 1 - the path to the EnergyPlus executable in the install-tree, which is used to determine where to copy the library
 #     since this is in the install-tree, you'll need to use a cmake generator expression
-# 2 - name of the folder to create to store the copied in python standard library, usually python_standard_library
+# 2 - name of the folder to create to store the copied in python standard library, usually python_lib
 import ctypes
 import os
 import platform
@@ -117,6 +117,13 @@ def find_libs(dir_path):
     dll_dir = os.path.join(python_root_dir, 'DLLs')
     shutil.copytree(dll_dir, target_dir, dirs_exist_ok=True)
 
+# And also on Windows, we now need the grab the Tcl/Tk folder that contains config, scripts, and blobs
+if platform.system() == 'Windows':
+    python_root_dir = os.path.dirname(standard_lib_dir)
+    tcl_dir = os.path.join(python_root_dir, 'tcl')
+    shutil.copytree(tcl_dir, target_dir, dirs_exist_ok=True)
+# TODO: Need to do this on Mac as well, see the bottom of cmake/PythonFixUpOnMac.cmake for more info
+
 # then I'm going to try to clean up any __pycache__ folders in the target dir to reduce installer size
 for root, dirs, _ in os.walk(target_dir):
     for this_dir in dirs:

cmake/PythonFixUpOnMac.cmake

@@ -71,3 +71,20 @@ foreach(PREREQ IN LISTS PREREQUISITES)
     execute_process(COMMAND "install_name_tool" -change "${PREREQ}" "@loader_path/${LIB_INT_FILENAME}" "${LOCAL_PYTHON_LIBRARY}")
   endif()
 endforeach()
+
+##############3
+# we need to look into the tkinter binary's runtime dependencies, copy over the tcl and tk dylibs, update them with install_name_tool, and make sure they all get signed, like this:
+
+# libtcl
+# cp /opt/homebrew/opt/tcl-tk/lib/libtcl8.6.dylib /path/to/python_lib/lib-dynload/
+# install_name_tool -change "/opt/homebrew/opt/tcl-tk/lib/libtcl8.6.dylib" "@loader_path/libtcl8.6.dylib" /path/to/python_lib/lib-dynload/_tkinter.cpython-312-darwin.so
+
+# Do the same for libtk
+# cp /opt/homebrew/opt/tcl-tk/lib/libtk8.6.dylib /path/to/python_lib/lib-dynload/
+# install_name_tool -change "/opt/homebrew/opt/tcl-tk/lib/libtk8.6.dylib" "@loader_path/libtk8.6.dylib" /path/to/python_lib/lib-dynload/_tkinter.cpython-312-darwin.so
+
+# Resign _tkinter
+# codesign -vvvv -s "Developer ID Application: National ..." -f --timestamp -i "org.nrel.EnergyPlus" -o runtime /path/to/python_lib/lib-dynload/_tkinter.cpython-312-darwin.so
+# codesign -vvvv -s "Developer ID Application: National ..." -f --timestamp -i "org.nrel.EnergyPlus" -o runtime /path/to/python_lib/lib-dynload/libtcl8.6.dylib
+# codesign -vvvv -s "Developer ID Application: National ..." -f --timestamp -i "org.nrel.EnergyPlus" -o runtime /path/to/python_lib/lib-dynload/libtk8.6.dylib
+##############3

cmake/install_codesign_script.cmake

@@ -20,7 +20,7 @@ Pre-conditions:
 
 This script will codesign the ``FILES_TO_SIGN``, as well as the globbed copied Python .so and the root dylibs (such as ``libintl8.dylib``)
 
-* ``python_standard_lib/lib-dynload/*.so``
+* ``python_lib/lib-dynload/*.so``
 * ``lib*.dylib``
 
 To do so, it uses the `CodeSigning`_ functions :cmake:command:`codesign_files_macos`
@@ -113,11 +113,11 @@ foreach(path ${_all_root_dylibs})
   endif()
 endforeach()
 
-file(GLOB PYTHON_SOS "${CMAKE_INSTALL_PREFIX}/python_standard_lib/lib-dynload/*.so")
+file(GLOB PYTHON_SOS "${CMAKE_INSTALL_PREFIX}/python_lib/lib-dynload/*.so")
 
 print_relative_paths(PREFIX "FULL_PATHS=" ABSOLUTE_PATHS ${FULL_PATHS})
 print_relative_paths(PREFIX "ROOT_DYLIBS=" ABSOLUTE_PATHS ${ROOT_DYLIBS})
-print_relative_paths(PREFIX "PYTHON_SOS, in ${CMAKE_INSTALL_PREFIX}/python_standard_lib/lib-dynload/=" ABSOLUTE_PATHS ${PYTHON_SOS} NAME_ONLY)
+print_relative_paths(PREFIX "PYTHON_SOS, in ${CMAKE_INSTALL_PREFIX}/python_lib/lib-dynload/=" ABSOLUTE_PATHS ${PYTHON_SOS} NAME_ONLY)
 
 include(${CMAKE_CURRENT_LIST_DIR}/CodeSigning.cmake)
 codesign_files_macos(

scripts/dev/create_shortcut.ps1

@@ -0,0 +1,11 @@
+param (
+    [string]$TargetPath,
+    [string]$ShortcutPath,
+    [string]$Arguments
+)
+
+$WScriptShell = New-Object -ComObject WScript.Shell
+$Shortcut = $WScriptShell.CreateShortcut($ShortcutPath)
+$Shortcut.TargetPath = $TargetPath
+$Shortcut.Arguments = $Arguments
+$Shortcut.Save()

src/EnergyPlus/CMakeLists.txt

@@ -521,6 +521,8 @@ set(SRC
     Pumps.hh
     PurchasedAirManager.cc
     PurchasedAirManager.hh
+    PythonEngine.cc
+    PythonEngine.hh
     RefrigeratedCase.cc
     RefrigeratedCase.hh
     ReportCoilSelection.cc
@@ -965,15 +967,24 @@ if(LINK_WITH_PYTHON)
   add_custom_command(
     TARGET energyplusapi
     POST_BUILD # TODO: I don't think we want to quote the generator expression
-    COMMAND ${Python_EXECUTABLE} "${PROJECT_SOURCE_DIR}/cmake/PythonCopyStandardLib.py" "$<TARGET_FILE:energyplusapi>" "python_standard_lib")
+    COMMAND ${Python_EXECUTABLE} "${PROJECT_SOURCE_DIR}/cmake/PythonCopyStandardLib.py" "$<TARGET_FILE:energyplusapi>" "python_lib"
+    COMMAND ${CMAKE_COMMAND} -E env --unset=PIP_REQUIRE_VIRTUALENV
+            ${Python_EXECUTABLE} -m pip install --target="$<TARGET_FILE_DIR:energyplusapi>/python_lib" --upgrade energyplus-launch==3.7.2
+  )
 endif()
 
 if(BUILD_PACKAGE)
   # if we are building package, we need to drop in some API/Plugin stuff
   if(LINK_WITH_PYTHON)
     # we'll want to grab the standard lib for python plugins
     # TODO: I don't think we want to quote the generator expression
-    install(DIRECTORY "$<TARGET_FILE_DIR:energyplus>/python_standard_lib/" DESTINATION "./python_standard_lib")
+    install(DIRECTORY "$<TARGET_FILE_DIR:energyplus>/python_lib/" DESTINATION "./python_lib")
+	if(WIN32)
+	  # on Windows, with Build Package, and also Link With Python, we can also drop in shortcuts to the new auxiliary CLI
+	  # NOTE: The install command COMPONENTS should line up so that they are run at the same packaging step
+      install(CODE "execute_process(COMMAND powershell.exe -ExecutionPolicy Bypass -File ${PROJECT_SOURCE_DIR}/scripts/dev/create_shortcut.ps1 -TargetPath \"$<TARGET_FILE:energyplus>\" -ShortcutPath \"$<TARGET_FILE_DIR:energyplus>/EPLaunchPython.lnk\" -Arguments \"auxiliary eplaunch\")" COMPONENT Auxiliary)
+	  install(FILES $<TARGET_FILE_DIR:energyplus>/EPLaunchPython.lnk DESTINATION "./" COMPONENT Auxiliary)
+	endif()
   endif()
   # we'll want to always provide the C API headers
   install(FILES ${API_HEADERS} DESTINATION "./include/EnergyPlus/api")

src/EnergyPlus/CommandLineInterface.cc

@@ -61,6 +61,10 @@
 #include <EnergyPlus/PluginManager.hh>
 #include <EnergyPlus/UtilityRoutines.hh>
 
+#if LINK_WITH_PYTHON
+#include <EnergyPlus/PythonEngine.hh>
+#endif
+
 namespace EnergyPlus {
 
 namespace CommandLineInterface {
@@ -230,6 +234,61 @@ Built on Platform: {}
         // bool debugCLI = false;
         app.add_flag("--debug-cli", debugCLI, "Print the result of the CLI assignments to the console and exit")->group(""); // Empty group to hide it
 
+#if LINK_WITH_PYTHON
+#if __APPLE__
+        // for now on Apple we are not providing the command line interface to EP-Launch due to packaging issues
+        // once that is fixed, this __APPLE__ block will be removed and we'll just have this on all platforms
+#else
+        auto *auxiliaryToolsSubcommand = app.add_subcommand("auxiliary", "Run Auxiliary Python Tools");
+        auxiliaryToolsSubcommand->require_subcommand(); // should default to requiring 1 or more additional args?
+
+        std::vector<std::string> python_fwd_args;
+        auto *epLaunchSubCommand = auxiliaryToolsSubcommand->add_subcommand("eplaunch", "EnergyPlus Launch");
+        epLaunchSubCommand->add_option("args", python_fwd_args, "Extra Arguments forwarded to EnergyPlus Launch")->option_text("ARG ...");
+        epLaunchSubCommand->positionals_at_end(true);
+        epLaunchSubCommand->footer("You can pass extra arguments after the eplaunch keyword, they will be forwarded to EnergyPlus Launch.");
+
+        epLaunchSubCommand->callback([&state, &python_fwd_args] {
+            EnergyPlus::Python::PythonEngine engine(state);
+            // There's probably better to be done, like instantiating the pythonEngine with the argc/argv then calling PyRun_SimpleFile but whatever
+            std::string cmd = R"python(import sys
+sys.argv.clear()
+sys.argv.append("energyplus")
+)python";
+            for (const auto &arg : python_fwd_args) {
+                cmd += fmt::format("sys.argv.append(\"{}\")\n", arg);
+            }
+
+            fs::path programDir = FileSystem::getParentDirectoryPath(FileSystem::getAbsolutePath(FileSystem::getProgramPath()));
+            fs::path const pathToPythonPackages = programDir / "python_lib";
+            std::string sPathToPythonPackages = std::string(pathToPythonPackages.string());
+            std::replace(sPathToPythonPackages.begin(), sPathToPythonPackages.end(), '\\', '/');
+            cmd += fmt::format("sys.path.insert(0, \"{}\")\n", sPathToPythonPackages);
+
+            std::string tclConfigDir = "";
+            for (auto &p : std::filesystem::directory_iterator(pathToPythonPackages)) {
+                if (p.is_directory()) {
+                    std::string dirName = p.path().filename().string();
+                    if (dirName.find("tcl", 0) == 0 && dirName.find(".", 0) > 0) {
+                        tclConfigDir = dirName;
+                        break;
+                    }
+                }
+            }
+            cmd += "from os import environ\n";
+            cmd += fmt::format("environ[\'TCL_LIBRARY\'] = \"{}/{}\"\n", sPathToPythonPackages, tclConfigDir);
+
+            cmd += R"python(
+from eplaunch.tk_runner import main_gui
+main_gui()
+)python";
+            // std::cout << "Trying to execute this python snippet: " << std::endl << cmd << std::endl;
+            engine.exec(cmd);
+            exit(0);
+        });
+#endif
+#endif
+
         app.footer("Example: energyplus -w weather.epw -r input.idf");
 
         const bool eplusRunningViaAPI = state.dataGlobal->eplusRunningViaAPI;
@@ -695,33 +754,16 @@ state.dataStrGlobals->inputFilePath='{:g}',
         // Duplicate the kind of reading the Windows "GetINISetting" would
         // do.
 
-        // REFERENCES:
-        // na
-
         // Using/Aliasing
         using namespace EnergyPlus;
         using namespace DataStringGlobals;
 
-        // Locals
-        // SUBROUTINE ARGUMENT DEFINITIONS:
-
-        // SUBROUTINE PARAMETER DEFINITIONS:
-
-        // INTERFACE BLOCK SPECIFICATIONS
-        // na
-
-        // DERIVED TYPE DEFINITIONS
-        // na
-
-        // SUBROUTINE LOCAL VARIABLE DECLARATIONS:
-
         std::string Param;
         std::string::size_type ILB;
         std::string::size_type IRB;
         std::string::size_type IEQ;
         std::string::size_type IPAR;
         std::string::size_type IPOS;
-        std::string::size_type ILEN;
 
         // Formats
 
@@ -731,7 +773,6 @@ state.dataStrGlobals->inputFilePath='{:g}',
 
         Param = KindofParameter;
         strip(Param);
-        ILEN = len(Param);
         inputFile.rewind();
         bool Found = false;
         bool NewHeading = false;

src/EnergyPlus/PluginManager.cc

@@ -58,6 +58,19 @@
 #include <nlohmann/json.hpp>
 
 #if LINK_WITH_PYTHON
+
+#ifdef _DEBUG
+// We don't want to try to import a debug build of Python here
+// so if we are building a Debug build of the C++ code, we need
+// to undefine _DEBUG during the #include command for Python.h.
+// Otherwise it will fail
+#undef _DEBUG
+#include <Python.h>
+#define _DEBUG
+#else
+#include <Python.h>
+#endif
+
 #include <fmt/format.h>
 template <> struct fmt::formatter<PyStatus>
 {
@@ -467,7 +480,7 @@ PluginManager::PluginManager(EnergyPlusData &state) : eplusRunningViaPythonAPI(s
     } else {
         programDir = FileSystem::getParentDirectoryPath(FileSystem::getAbsolutePath(FileSystem::getProgramPath()));
     }
-    fs::path const pathToPythonPackages = programDir / "python_standard_lib";
+    fs::path const pathToPythonPackages = programDir / "python_lib";
 
     initPython(state, pathToPythonPackages);
 
@@ -478,7 +491,7 @@ PluginManager::PluginManager(EnergyPlusData &state) : eplusRunningViaPythonAPI(s
     PyRun_SimpleString("import sys"); // allows us to report sys.path later
 
     // we also need to set an extra import path to find some dynamic library loading stuff, again make it relative to the binary
-    addToPythonPath(state, programDir / "python_standard_lib/lib-dynload", false);
+    addToPythonPath(state, programDir / "python_lib/lib-dynload", false);
 
     // now for additional paths:
     // we'll always want to add the program executable directory to PATH so that Python can find the installed pyenergyplus package

src/EnergyPlus/PluginManager.hh

@@ -60,16 +60,9 @@
 #include <EnergyPlus/EnergyPlus.hh>
 
 #if LINK_WITH_PYTHON
-#ifdef _DEBUG
-// We don't want to try to import a debug build of Python here
-// so if we are building a Debug build of the C++ code, we need
-// to undefine _DEBUG during the #include command for Python.h.
-// Otherwise it will fail
-#undef _DEBUG
-#include <Python.h>
-#define _DEBUG
-#else
-#include <Python.h>
+#ifndef PyObject_HEAD
+struct _object;
+using PyObject = _object;
 #endif
 #endif
 
@@ -175,6 +168,7 @@ namespace PluginManagement {
 #endif
     };
 
+    // TODO: Make this use PythonEngine so we don't duplicate code
     class PluginManager
     {
     public: