Skip to content

Commit

Permalink
Started avoiding library copy
Browse files Browse the repository at this point in the history 
For #461
  • Loading branch information
@mwetter
mwetter committed Jan 21, 2022
1 parent 6c7760c commit 2887e65
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 59 deletions.
5 changes: 4 additions & 1 deletion buildingspy/development/openmodelica_run.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,15 @@ def _add_exception(return_dict, e, cmd):

def _run_process(return_dict, cmd, worDir, timeout):
import subprocess
import os
env = os.environ.copy()

output = subprocess.check_output(
cmd,
cwd = worDir,
timeout=timeout,
stderr=subprocess.STDOUT,
env = env,
shell=False)

return_dict['success'] = True
Expand Down Expand Up @@ -55,7 +58,7 @@ setCommandLineOptions("-d=stateselection");
//setCommandLineOptions("-d=execstat");
setMatchingAlgorithm("PFPlusExt");
setIndexReductionMethod("dynamicStateSelection");
loadFile("{{ library_name }}/package.mo");
//loadFile("{{ library_name }}/package.mo");
translated := translateModel({model}, method="{{ solver }}", tolerance={{ rtol }}, numberOfIntervals={{ ncp }}, variableFilter="{{ filter }}");
getErrorString();

Expand Down
128 changes: 70 additions & 58 deletions buildingspy/development/regressiontest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
import buildingspy.io.reporter as rep


def runSimulation(worDir, cmd):
def runSimulation(worDir, cmd, env):
""" Run the simulation.
:param worDir: The working directory.
Expand All @@ -51,12 +51,6 @@ def runSimulation(worDir, cmd):
.. note:: This method is outside the class definition to
allow parallel computing.
"""
# JModelica requires the working directory to be part of MODELICAPATH
env = os.environ.copy() # will be passed to the subprocess.Popen call
if 'MODELICAPATH' in os.environ:
env['MODELICAPATH'] = "{}:{}".format(worDir, os.environ['MODELICAPATH'])
else:
env['MODELICAPATH'] = worDir

logFilNam = os.path.join(worDir, 'stdout.log')
#
Expand Down Expand Up @@ -1348,7 +1342,7 @@ def _getSimulationResults(self, data, warnings, errors):
of results that need to be printed together.
"""
# Get the working directory that contains the ".mat" file
fulFilNam = os.path.join(data['ResultDirectory'], self.getLibraryName(), data['ResultFile'])
fulFilNam = os.path.join(data['ResultDirectory'], data['ResultFile'])
if self._modelica_tool != 'dymola':
fulFilNam = os.path.join(data['ResultDirectory'], data['ResultFile'])
ret = []
Expand Down Expand Up @@ -1443,8 +1437,7 @@ def _getDymolaTranslationStatistics(self, data, warnings, errors):
In case of an error, this method returns `None`.
"""
# Get the working directory that contains the ".log" file
fulFilNam = os.path.join(data['ResultDirectory'],
self.getLibraryName(), data['dymola']['TranslationLogFile'])
fulFilNam = os.path.join(data['ResultDirectory'], data['dymola']['TranslationLogFile'])
return of.get_model_statistics(fulFilNam, self._modelica_tool)

def _legacy_comp(self, tOld, yOld, tNew, yNew, tGriOld, tGriNew, varNam, filNam, tol):
Expand Down Expand Up @@ -3099,8 +3092,7 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
# Number of generated unit tests
nUniTes = 0

runFil = open(os.path.join(self._temDir[iPro], self.getLibraryName(
), "runAll.mos"), mode="w", encoding="utf-8")
runFil = open(os.path.join(self._temDir[iPro], "runAll.mos"), mode="w", encoding="utf-8")
runFil.write(
f"""
// File autogenerated for process {iPro + 1} of {self._nPro}
Expand All @@ -3127,11 +3119,10 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
sett[{posDDE}] = \"DDE=0\"; // Disable DDE
SetDymolaCompiler(comp, sett);
""")
runFil.write('cd(\"{}/{}\");\n'.format(
(self._temDir[iPro]).replace("\\", "/"),
self.getLibraryName()))
runFil.write('cd(\"{}\");\n'.format(
(self._temDir[iPro]).replace("\\", "/")))
runFil.write(f"""
openModel("package.mo")
openModel("{self.getLibraryName()}/package.mo", changeDirectory=false)
// Add a flag so that translation info appears in console output.
// This allows checking for numerical derivatives.
// Dymola will write this output to a file when savelog(filename) is called.
Expand Down Expand Up @@ -3166,8 +3157,10 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
mosFilNam = os.path.join(self.getLibraryName(),
"Resources", "Scripts", "Dymola",
tra_data_pro[i]['ScriptFile'])
absMosFilNam = os.path.join(self._temDir[iPro], mosFilNam)
absMosFilNam = os.path.join(self._temDir[0], mosFilNam)
values = {
"libraryInstallDirectory": f"../{os.path.split(self._temDir[0])[-1]}",
"libraryName": self.getLibraryName(),
"mosWithPath": mosFilNam.replace(
"\\",
"/"),
Expand All @@ -3193,7 +3186,6 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
"/"),
"translationLog": os.path.join(
self._temDir[iPro],
self.getLibraryName(),
tra_data_pro[i]['model_name'] +
".translation.log").replace(
"\\",
Expand Down Expand Up @@ -3245,7 +3237,7 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
# Modelica.Utilities.Streams.readFile
template = r"""
{set_non_pedantic}
rScript=RunScript("Resources/Scripts/Dymola/{scriptFile}");
rScript=RunScript("{libraryInstallDirectory}/{libraryName}/Resources/Scripts/Dymola/{scriptFile}");
{set_pedantic}
savelog("{model_name}.translation.log");
if Modelica.Utilities.Files.exist("dslog.txt") then
Expand Down Expand Up @@ -3326,7 +3318,7 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
if tra_data_pro[i]['dymola']['exportFMU']:
template = r"""
Modelica.Utilities.Files.removeFile("{FMUName}");
RunScript("Resources/Scripts/Dymola/{scriptFile}");
RunScript("{libraryInstallDirectory}/{libraryName}/Resources/Scripts/Dymola/{scriptFile}");
savelog("{model_name}.translation.log");
if Modelica.Utilities.Files.exist("dslog.txt") then
Modelica.Utilities.Files.move("dslog.txt", "{model_name}.dslog.log");
Expand All @@ -3348,7 +3340,7 @@ def _print_end_of_json(isLastItem, fileHandle, logFileName):
runFil.write(template.format(**values))
template = r"""
Modelica.Utilities.Streams.print(" \"FMUExport\" : {{", "{statisticsLog}");
Modelica.Utilities.Streams.print(" \"command\" :\"RunScript(\\\"Resources/Scripts/Dymola/{scriptFile}\\\");\",", "{statisticsLog}");
Modelica.Utilities.Streams.print(" \"command\" :\"RunScript(\\\"{libraryInstallDirectory}/{libraryName}/Resources/Scripts/Dymola/{scriptFile}\\\");\",", "{statisticsLog}");
Modelica.Utilities.Streams.print(" \"translationLog\" : \"{translationLog}\",", "{statisticsLog}");
Modelica.Utilities.Streams.print(" \"result\" : " + String(iSuc > 0), "{statisticsLog}");
"""
Expand Down Expand Up @@ -3570,39 +3562,46 @@ def deleteTemporaryDirectories(self, delete):
def _setTemporaryDirectories(self):
self._temDir = []

# Make temporary directory, copy library into the directory and
# write run scripts to directory
# Make temporary directory
for iPro in range(self._nPro):
# print("Calling parallel loop for iPro={}, self._nPro={}".format(iPro, self._nPro))
dirNam = tempfile.mkdtemp(
prefix='tmp-' + self.getLibraryName() + '-' + str(iPro) + "-")
self._temDir.append(dirNam)
# Directory that contains the library as a sub directory
libDir = self._libHome

shutil.copytree(
libDir,
os.path.join(
dirNam,
self.getLibraryName()),
symlinks=True,
ignore=shutil.ignore_patterns(
'.svn',
'.git',
'*.mat',
'*.log',
'request.',
'status.',
'dsmodel.c',
'dymosim',
'tmp-*',
'funnel-comp',
'fmi-library', # Not all of src is excluded as some .mo models link to files from src
'Documentation',
'ReferenceResults',
'help',
'compareVars',
'__pychache__'))
# copy library into the directory and
# write run scripts to directory
main_dir = self._temDir[0]
for iPro in range(self._nPro):
new_bui_dir = os.path.join(self._temDir[iPro], self.getLibraryName())
if iPro == 0:
shutil.copytree(
self._libHome,
new_bui_dir,
symlinks=True,
ignore=shutil.ignore_patterns(
'.svn',
'.git',
'*.mat',
'*.log',
'request.',
'status.',
'dsmodel.c',
'dymosim',
'tmp-*',
'funnel-comp',
'fmi-library', # Not all of src is excluded as some .mo models link to files from src
'Documentation',
'ReferenceResults',
'help',
'compareVars',
'__pychache__'))
# else:
# # For other processors, just make a symbolic link
# src = os.path.join(main_dir, os.path.split(self._libHome)[-1])
# os.symlink(src = src, dst = new_bui_dir, target_is_directory = True)
# Make sure everything is written to disc before proceeding.
os.sync()
return

def _run_simulation_info(self):
Expand Down Expand Up @@ -3651,6 +3650,7 @@ def run(self):
"""
from sys import platform
import os

self.checkPythonModuleAvailability()

Expand Down Expand Up @@ -3710,11 +3710,11 @@ def run(self):
# Run simulations
tem_dir = []
libNam = self.getLibraryName()
for di in self._temDir:
if self._modelica_tool == 'dymola':
tem_dir.append(os.path.join(di, libNam))
else:
tem_dir.append(di)
# for di in self._temDir:
# if self._modelica_tool == 'dymola':
# tem_dir.append(os.path.join(di, libNam))
# else:
# tem_dir.append(di)

if not self._useExistingResults:
if self._modelica_tool == 'dymola':
Expand All @@ -3734,24 +3734,36 @@ def run(self):
cmd = ["python", "./run.py"]
elif self._modelica_tool != 'dymola':
cmd = [self.getModelicaCommand(), "run.py"]

# JModelica requires the working directory to be part of MODELICAPATH
env = os.environ.copy() # will be passed to the subprocess.Popen call
if 'MODELICAPATH' in os.environ:
env['MODELICAPATH'] = "{}:{}".format(self._temDir[0], os.environ['MODELICAPATH'])
else:
env['MODELICAPATH'] = self._temDir[0]

if self._modelica_tool == 'openmodelica':
env['MODELICAPATH'] = f"{env['MODELICAPATH']}"
env['OPENMODELICALIBRARY'] = env['MODELICAPATH']

if self._nPro > 1:
po = multiprocessing.Pool(self._nPro)
po.map(functools.partial(runSimulation,
cmd=cmd),
[x for x in tem_dir])
cmd=cmd,
env=env),
[x for x in self._temDir])
po.close()
po.join()
else:
if len(self._data) > 0:
runSimulation(tem_dir[0], cmd)
runSimulation(self._temDir[0], cmd, env)

# Concatenate simulator output files into one file
with open(self._simulator_log_file, mode="w", encoding="utf-8") as logFil:
for d in self._temDir:
for temLogFilNam in glob.glob(
os.path.join(
d,
self.getLibraryName(),
'*.translation.log')):
if os.path.exists(temLogFilNam):
with open(temLogFilNam, mode="r", encoding="utf-8-sig") as fil:
Expand All @@ -3767,7 +3779,7 @@ def run(self):
with open(self._statistics_log, mode="w", encoding="utf-8") as logFil:
stat = list()
for d in self._temDir:
temLogFilNam = os.path.join(d, self.getLibraryName(), self._statistics_log)
temLogFilNam = os.path.join(d, self._statistics_log)
if os.path.exists(temLogFilNam):
with open(temLogFilNam.replace('Temp\tmp', 'Temp\\tmp'), mode="r", encoding="utf-8-sig") as temSta:
try:
Expand Down

0 comments on commit 2887e65

Please sign in to comment.