From f89a0b92e7a40831c298d60e195fe1178edef69e Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Wed, 21 Aug 2024 12:51:39 -0500 Subject: [PATCH 01/78] Add new Mac build/test GHA workflow --- .github/workflows/mac_build.yml | 90 +++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/mac_build.yml diff --git a/.github/workflows/mac_build.yml b/.github/workflows/mac_build.yml new file mode 100644 index 00000000000..c59f65cb203 --- /dev/null +++ b/.github/workflows/mac_build.yml @@ -0,0 +1,90 @@ +name: Mac Build + +on: + push: + branches: [ develop ] + tags: + - 'v*' # Push events matching v*, i.e. v1.0, v20.15.10 + pull_request: + branches: [ develop ] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FC: gfortran-13 + SDKROOT: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk + Python_REQUIRED_VERSION: 3.12.2 + +jobs: + build_installer_artifact: + runs-on: ${{ matrix.os }} + continue-on-error: false + strategy: + fail-fast: false + matrix: + macos_dev_target: [12.1, 13.0] + include: + - macos_dev_target: 12.1 + os: macos-12 + arch: x86_64 + python-arch: x64 + - macos_dev_target: 13.0 + os: macos-14 + arch: arm64 + python-arch: arm64 + + steps: + - uses: actions/checkout@v4 + + # TODO: I assume since we aren't doing any packaging here, we can just use actions/setup-python...but leaving this for now + - name: Remove python ${{ env.Python_REQUIRED_VERSION }} from the toolcache + run: | + ls $RUNNER_TOOL_CACHE/Python || true + rm -Rf "$RUNNER_TOOL_CACHE/Python/${{ env.Python_REQUIRED_VERSION }}" + rm -Rf "$RUNNER_TOOL_CACHE/Python/${{ env.Python_REQUIRED_VERSION }}*/" + + - name: Set up Python ${{ env.Python_REQUIRED_VERSION }} + id: setup-python + uses: jmarrec/setup-python@v5 + with: + python-version: ${{ env.Python_REQUIRED_VERSION }} + + - name: Install System dependencies + shell: bash + run: | + set -x + brew update + # The MACOSX_DEPLOYMENT_TARGET environment variable sets the default value for the CMAKE_OSX_DEPLOYMENT_TARGET variable. + # We use cmake commands to build some subprojects, so setting it globally + echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV + echo "Installing gcc@13 for gfortran support of -static-libquadmath" + brew list gcc@13 || brew install gcc@13 + which gfortran-13 || echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV + brew install ninja + + - name: Create Build Directory + run: cmake -E make_directory ./build/ + + - name: Configure CMake + working-directory: ./build + shell: bash + run: | + cmake -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ + -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ + -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ + -DBUILD_FORTRAN:BOOL=ON -DBUILD_PACKAGE:BOOL=OFF \ + -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ + -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ + ../ + + - name: Build Default + working-directory: ./build + shell: bash + run: ninja + + - name: Run Tests + working-directory: ./build + shell: bash + run: ctest -E Basement -j 3 + + # TODO: Add regression builds, as well as use GHA caching to store baseline branch results for efficiency From 7c261aa284c0af199f81e2fec38f41cab1d7c519 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Wed, 21 Aug 2024 12:52:38 -0500 Subject: [PATCH 02/78] Temporarily rename DCI file while playing with Mac workflow file --- .decent_ci.yaml => .renamed.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .decent_ci.yaml => .renamed.yaml (100%) diff --git a/.decent_ci.yaml b/.renamed.yaml similarity index 100% rename from .decent_ci.yaml rename to .renamed.yaml From b0cee1174226c85ce5a370350d4ae79dbf854960 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Wed, 21 Aug 2024 14:38:09 -0500 Subject: [PATCH 03/78] Fixup test name and add build_testing --- .github/workflows/mac_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac_build.yml b/.github/workflows/mac_build.yml index c59f65cb203..f68cf5d7f50 100644 --- a/.github/workflows/mac_build.yml +++ b/.github/workflows/mac_build.yml @@ -15,7 +15,7 @@ env: Python_REQUIRED_VERSION: 3.12.2 jobs: - build_installer_artifact: + mac_build_and_test: runs-on: ${{ matrix.os }} continue-on-error: false strategy: @@ -72,7 +72,7 @@ jobs: -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ - -DBUILD_FORTRAN:BOOL=ON -DBUILD_PACKAGE:BOOL=OFF \ + -DBUILD_TESTING:BOOL=ON -DBUILD_FORTRAN:BOOL=ON -DBUILD_PACKAGE:BOOL=OFF \ -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ ../ From b6d1e1463103c9157950f82d0e874a2a49253e66 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Thu, 22 Aug 2024 09:46:34 -0500 Subject: [PATCH 04/78] Add upterm to debug failing tests --- .github/workflows/mac_build.yml | 2 ++ tst/EnergyPlus/unit/Timer.unit.cc | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac_build.yml b/.github/workflows/mac_build.yml index f68cf5d7f50..264ce6bd412 100644 --- a/.github/workflows/mac_build.yml +++ b/.github/workflows/mac_build.yml @@ -82,6 +82,8 @@ jobs: shell: bash run: ninja + - uses: lhotari/action-upterm@v1 + - name: Run Tests working-directory: ./build shell: bash diff --git a/tst/EnergyPlus/unit/Timer.unit.cc b/tst/EnergyPlus/unit/Timer.unit.cc index 6817974a17f..dae5b5da3ec 100644 --- a/tst/EnergyPlus/unit/Timer.unit.cc +++ b/tst/EnergyPlus/unit/Timer.unit.cc @@ -70,7 +70,7 @@ TEST_F(EnergyPlusFixture, Timer_ticktock) std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time_ms)); t.tock(); // In some occurrences CI is reporting slightly above than 100 values, probably system was quite busy at that time, - // but we don't want to have the test failing occassionally + // but we don't want to have the test failing occasionally EXPECT_GE(t.duration().count(), sleep_time_ms); EXPECT_LT(t.duration().count(), sleep_time_ms * 2); EXPECT_GE(t.elapsedSeconds(), sleep_time_s); From ed45160ed9282c8e8bc8aae8e161a53df0eda5be Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Thu, 22 Aug 2024 12:38:14 -0500 Subject: [PATCH 05/78] Hacky regression stuff, lets see how close we are --- .github/workflows/mac_build.yml | 67 +++++- requirements.txt | 6 +- scripts/dev/gha_regressions.py | 218 ++++++++++++++++++ .../unit/CommandLineInterface.unit.cc | 2 +- tst/EnergyPlus/unit/Timer.unit.cc | 2 +- 5 files changed, 281 insertions(+), 14 deletions(-) create mode 100644 scripts/dev/gha_regressions.py diff --git a/.github/workflows/mac_build.yml b/.github/workflows/mac_build.yml index 264ce6bd412..dd8e5e5f5b2 100644 --- a/.github/workflows/mac_build.yml +++ b/.github/workflows/mac_build.yml @@ -33,8 +33,6 @@ jobs: python-arch: arm64 steps: - - uses: actions/checkout@v4 - # TODO: I assume since we aren't doing any packaging here, we can just use actions/setup-python...but leaving this for now - name: Remove python ${{ env.Python_REQUIRED_VERSION }} from the toolcache run: | @@ -61,11 +59,19 @@ jobs: which gfortran-13 || echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV brew install ninja - - name: Create Build Directory - run: cmake -E make_directory ./build/ + # BUILD AND TEST INTEGRATION FILES ON THE BASELINE BRANCH + + - name: [Baseline] Checkout + uses: actions/checkout@v4 + with: + ref: develop + path: baseline + + - name: [Baseline] Create Build Directory + run: cmake -E make_directory ./baseline/build/ - - name: Configure CMake - working-directory: ./build + - name: [Baseline] Configure CMake + working-directory: ./baseline/build shell: bash run: | cmake -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release \ @@ -77,16 +83,55 @@ jobs: -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ ../ - - name: Build Default - working-directory: ./build + - name: [Baseline] Build + working-directory: ./baseline/build shell: bash run: ninja - - uses: lhotari/action-upterm@v1 + - name: [Baseline] Test + working-directory: ./baseline/build + shell: bash + run: ctest -E Basement -R integration -j 3 # TODO: Speed up basement so we don't have to skip it. + + # BUILD AND TEST EVERYTHING ON THE CURRENT BRANCH - - name: Run Tests - working-directory: ./build + - name: [Branch] Checkout + uses: actions/checkout@v4 + with: + path: branch + + - name: [Branch] Create Build Directory + run: cmake -E make_directory ./branch/build/ + + - name: [Branch] Configure CMake + working-directory: ./branch/build + shell: bash + run: | + cmake -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ + -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ + -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ + -DBUILD_TESTING:BOOL=ON -DBUILD_FORTRAN:BOOL=ON -DBUILD_PACKAGE:BOOL=OFF \ + -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ + -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ + ../ + + - name: [Branch] Build + working-directory: ./branch/build + shell: bash + run: ninja + + - name: [Branch] Test + working-directory: ./branch/build shell: bash run: ctest -E Basement -j 3 + # TODO: Allow this to fail temporarily but continue so that we also get regression results + + - name: Install Regression Tool + run: pip install energyplus-regressions # TODO: Not sure if we need to use a different python path here, and we could pip it or clone it + + - name: Run Regressions + run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ + # NOW INSTALL THE REGRESSION TOOL AND DO COMPARISONS OF THE INTEGRATION RESULTS # TODO: Add regression builds, as well as use GHA caching to store baseline branch results for efficiency diff --git a/requirements.txt b/requirements.txt index 9968de6fe52..c5eaec54a07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,6 @@ # requirements for building an EnergyPlus wheel -wheel \ No newline at end of file +wheel + +# requirements for the CI regression testing scripts +energyplus-regressions +boto3 diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py new file mode 100644 index 00000000000..653ec87f1f8 --- /dev/null +++ b/scripts/dev/gha_regressions.py @@ -0,0 +1,218 @@ +import json +import sys +from pathlib import Path + +from energyplus_regressions.runtests import SuiteRunner +from energyplus_regressions.structures import TextDifferences, TestEntry + + +def print_notice(relative_test_file: str, message: str) -> None: + # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" + print(f"::notice file=testfiles/{relative_test_file}::{message}") + + +def print_warning(relative_test_file: str, message: str) -> None: + # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" + print(f"::warning file=testfiles/{relative_test_file}::{message}") + + +def print_error(relative_test_file: str, message: str) -> None: + # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" + print(f"::error file=testfiles/{relative_test_file}::{message}") + + +def process_diffs(relative_test_file: str, diff_name, diffs, this_has_diffs, this_has_small_diffs): + # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" + if not diffs: + return this_has_diffs, this_has_small_diffs + if diffs.diff_type == 'Big Diffs': + this_has_diffs = True + print_warning(relative_test_file, f"{diff_name} Big Diffs") + elif diffs.diff_type == 'Small Diffs': + this_has_small_diffs = True + print_warning(relative_test_file, f"{diff_name} Small Diffs") + return this_has_diffs, this_has_small_diffs + + +def single_file_regressions(baseline: Path, modified: Path) -> [bool, bool, bool]: + + import energyplus_regressions + thresholds = Path(energyplus_regressions.__file__).parent / 'diffs' / 'math_diff.config' + + idf = baseline.name + + entry = TestEntry(idf, "") + entry, message = SuiteRunner.process_diffs_for_one_case( + entry, + str(baseline), + str(modified), + "", + str(thresholds), + ci_mode=True + ) # returns an updated entry + + with open('results.json', 'w') as f: + f.write(json.dumps(entry.to_dict(), indent=4)) + + success = True + has_diffs = False + has_small_diffs = False + + # Note, comment out any of the "has_diffs" below if you don't want + # it to generate an error condition + + if entry.aud_diffs and (entry.aud_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "AUD diffs.") + + if entry.bnd_diffs and (entry.bnd_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "BND diffs.") + + if entry.dl_in_diffs and (entry.dl_in_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "delightin diffs.") + + if entry.dl_out_diffs and (entry.dl_out_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "delightout diffs.") + + if entry.dxf_diffs and (entry.dxf_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "DXF diffs.") + + if entry.eio_diffs and (entry.eio_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "EIO diffs.") + + if entry.err_diffs and (entry.err_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "ERR diffs.") + + if entry.readvars_audit_diffs and (entry.readvars_audit_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "ReadvarsAudit diffs.") + + if entry.edd_diffs and (entry.edd_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "EDD diffs.") + + if entry.wrl_diffs and (entry.wrl_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "WRL diffs.") + + if entry.sln_diffs and (entry.sln_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "SLN diffs.") + + if entry.sci_diffs and (entry.sci_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "SCI diffs.") + + if entry.map_diffs and (entry.map_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "MAP diffs.") + + if entry.dfs_diffs and (entry.dfs_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "DFS diffs.") + + if entry.screen_diffs and (entry.screen_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "SCREEN diffs.") + + if entry.glhe_diffs and (entry.glhe_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "GLHE diffs") + + # numeric diff + if entry.eso_diffs: + has_diffs, has_small_diffs = process_diffs(idf, "ESO", entry.eso_diffs, has_diffs, has_small_diffs) + + if entry.mdd_diffs and (entry.mdd_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "MDD diffs.") + + if entry.mtd_diffs and (entry.mtd_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "MTD diffs.") + + # numeric diff + if entry.mtr_diffs: + has_diffs, has_small_diffs = process_diffs(idf, "MTR", entry.mtr_diffs, has_diffs, has_small_diffs) + + if entry.rdd_diffs and (entry.rdd_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "RDD diffs.") + + if entry.shd_diffs and (entry.shd_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "SHD diffs.") + + if entry.perf_log_diffs and (entry.perf_log_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "PERF_LOG diffs.") + + # numeric diff + if entry.ssz_diffs: + has_diffs, has_small_diffs = process_diffs(idf, "SSZ", entry.ssz_diffs, has_diffs, has_small_diffs) + + # numeric diff + if entry.zsz_diffs: + has_diffs, has_small_diffs = process_diffs(idf, "ZSZ", entry.zsz_diffs, has_diffs, has_small_diffs) + + # numeric diff + if entry.json_diffs: + has_diffs, has_small_diffs = process_diffs(idf, "JSON", entry.json_diffs, has_diffs, has_small_diffs) + + if entry.table_diffs: + if entry.table_diffs.big_diff_count > 0: + has_diffs = True + print_warning(idf, "Table big diffs.") + elif entry.table_diffs.small_diff_count > 0: + has_small_diffs = True + print_warning(idf, "Table small diffs.") + if entry.table_diffs.string_diff_count > 1: # There's always one...the time stamp + has_diffs = True + print_warning(idf, "Table string diffs.") + + if entry.idf_diffs and (entry.idf_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "IDF diffs.") + + if entry.stdout_diffs and (entry.stdout_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "StdOut diffs.") + + if entry.stderr_diffs and (entry.stderr_diffs.diff_type != TextDifferences.EQUAL): + has_small_diffs = True + print_warning(idf, "StdErr diffs.") + + return success, has_small_diffs, has_diffs + + +def check_all_regressions(base_testfiles: Path, mod_testfiles: Path) -> bool: + any_diffs = False + any_failures = False + for baseline in base_testfiles.iterdir(): + if not baseline.is_dir(): + continue + modified = mod_testfiles / baseline.name + if not modified.exists(): + continue # TODO: Should we warn that it is missing? + success, small_diffs, big_diffs = single_file_regressions(baseline, modified) + if small_diffs or big_diffs: + any_diffs = True + if not success: + any_failures = True + return any_diffs or any_failures + + +if __name__ == "__main__": # pragma: no cover - testing function, not the __main__ entry point + + if len(sys.argv) < 3: + print("syntax: %s base_dir mod_dir base_sha mod_sha device_id" % sys.argv[0]) + sys.exit(1) + arg_base_dir = Path(sys.argv[1]) + arg_mod_dir = Path(sys.argv[2]) + sys.exit(1 if check_all_regressions(arg_base_dir, arg_mod_dir) else 0) diff --git a/tst/EnergyPlus/unit/CommandLineInterface.unit.cc b/tst/EnergyPlus/unit/CommandLineInterface.unit.cc index d9d23566260..7fb32b7e246 100644 --- a/tst/EnergyPlus/unit/CommandLineInterface.unit.cc +++ b/tst/EnergyPlus/unit/CommandLineInterface.unit.cc @@ -385,7 +385,7 @@ TEST_F(CommandLineInterfaceFixture, runReadVars) } } -TEST_F(CommandLineInterfaceFixture, numThread) +TEST_F(CommandLineInterfaceFixture, DISABLED_numThread) { struct TestCase { diff --git a/tst/EnergyPlus/unit/Timer.unit.cc b/tst/EnergyPlus/unit/Timer.unit.cc index dae5b5da3ec..ac1878fc137 100644 --- a/tst/EnergyPlus/unit/Timer.unit.cc +++ b/tst/EnergyPlus/unit/Timer.unit.cc @@ -60,7 +60,7 @@ using namespace EnergyPlus; -TEST_F(EnergyPlusFixture, Timer_ticktock) +TEST_F(EnergyPlusFixture, DISABLED_Timer_ticktock) { constexpr std::chrono::milliseconds::rep sleep_time_ms = 100; From f33408f4ade18395b64bb76aa395eab093c40438 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Thu, 22 Aug 2024 12:40:13 -0500 Subject: [PATCH 06/78] Try removing braces --- .github/workflows/mac_build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/mac_build.yml b/.github/workflows/mac_build.yml index dd8e5e5f5b2..8afe20e87bf 100644 --- a/.github/workflows/mac_build.yml +++ b/.github/workflows/mac_build.yml @@ -61,16 +61,16 @@ jobs: # BUILD AND TEST INTEGRATION FILES ON THE BASELINE BRANCH - - name: [Baseline] Checkout + - name: Baseline Checkout uses: actions/checkout@v4 with: ref: develop path: baseline - - name: [Baseline] Create Build Directory + - name: Baseline Create Build Directory run: cmake -E make_directory ./baseline/build/ - - name: [Baseline] Configure CMake + - name: Baseline Configure CMake working-directory: ./baseline/build shell: bash run: | @@ -83,27 +83,27 @@ jobs: -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ ../ - - name: [Baseline] Build + - name: Baseline Build working-directory: ./baseline/build shell: bash run: ninja - - name: [Baseline] Test + - name: Baseline Test working-directory: ./baseline/build shell: bash run: ctest -E Basement -R integration -j 3 # TODO: Speed up basement so we don't have to skip it. # BUILD AND TEST EVERYTHING ON THE CURRENT BRANCH - - name: [Branch] Checkout + - name: Branch Checkout uses: actions/checkout@v4 with: path: branch - - name: [Branch] Create Build Directory + - name: Branch Create Build Directory run: cmake -E make_directory ./branch/build/ - - name: [Branch] Configure CMake + - name: Branch Configure CMake working-directory: ./branch/build shell: bash run: | @@ -116,12 +116,12 @@ jobs: -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ ../ - - name: [Branch] Build + - name: Branch Build working-directory: ./branch/build shell: bash run: ninja - - name: [Branch] Test + - name: Branch Test working-directory: ./branch/build shell: bash run: ctest -E Basement -j 3 From b3842b05b436cb9ba653b190fdb4efd32fb585b4 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Thu, 22 Aug 2024 14:47:38 -0500 Subject: [PATCH 07/78] Use correct arg for runtests --- scripts/dev/gha_regressions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index 653ec87f1f8..e556e24791a 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -44,8 +44,8 @@ def single_file_regressions(baseline: Path, modified: Path) -> [bool, bool, bool entry = TestEntry(idf, "") entry, message = SuiteRunner.process_diffs_for_one_case( entry, - str(baseline), - str(modified), + {'build_dir': str(baseline)}, + {'build_dir': str(modified)}, "", str(thresholds), ci_mode=True From c8119f7a0c8121d50118a33ebc731a3d37b1b29f Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Thu, 22 Aug 2024 16:38:58 -0500 Subject: [PATCH 08/78] Add a little debugging output --- scripts/dev/gha_regressions.py | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index e556e24791a..17cb6d80a8e 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -1,3 +1,59 @@ +#!/usr/bin/env python +# EnergyPlus, Copyright (c) 1996-2024, The Board of Trustees of the University +# of Illinois, The Regents of the University of California, through Lawrence +# Berkeley National Laboratory (subject to receipt of any required approvals +# from the U.S. Dept. of Energy), Oak Ridge National Laboratory, managed by UT- +# Battelle, Alliance for Sustainable Energy, LLC, and other contributors. All +# rights reserved. +# +# NOTICE: This Software was developed under funding from the U.S. Department of +# Energy and the U.S. Government consequently retains certain rights. As such, +# the U.S. Government has been granted for itself and others acting on its +# behalf a paid-up, nonexclusive, irrevocable, worldwide license in the +# Software to reproduce, distribute copies to the public, prepare derivative +# works, and perform publicly and display publicly, and to permit others to do +# so. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# (1) Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# (2) Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# (3) Neither the name of the University of California, Lawrence Berkeley +# National Laboratory, the University of Illinois, U.S. Dept. of Energy nor +# the names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# (4) Use of EnergyPlus(TM) Name. If Licensee (i) distributes the software in +# stand-alone form without changes from the version obtained under this +# License, or (ii) Licensee makes a reference solely to the software +# portion of its product, Licensee must refer to the software as +# "EnergyPlus version X" software, where "X" is the version number Licensee +# obtained under this License and may not use a different name for the +# software. Except as specifically required in this Section (4), Licensee +# shall not use in a company name, a product name, in advertising, +# publicity, or other promotional activities any name, trade name, +# trademark, logo, or other designation of "EnergyPlus", "E+", "e+" or +# confusingly similar designation, without the U.S. Department of Energy's +# prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + import json import sys from pathlib import Path @@ -188,6 +244,8 @@ def single_file_regressions(baseline: Path, modified: Path) -> [bool, bool, bool has_small_diffs = True print_warning(idf, "StdErr diffs.") + print(f"*** Regressions for file {idf}: {success=}, {has_small_diffs=}, {has_diffs=}") + print(f"{json.dumps(entry.to_dict(), indent=4)}\n") return success, has_small_diffs, has_diffs From 42f935a600eebfec5c6a6a85148e2f5d1588cdca Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 11:27:50 -0500 Subject: [PATCH 09/78] Rework mac build file, trying on all platforms just to see what happens --- .github/workflows/mac_build.yml | 92 +++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/.github/workflows/mac_build.yml b/.github/workflows/mac_build.yml index 8afe20e87bf..01e3fde8d90 100644 --- a/.github/workflows/mac_build.yml +++ b/.github/workflows/mac_build.yml @@ -13,6 +13,7 @@ env: FC: gfortran-13 SDKROOT: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk Python_REQUIRED_VERSION: 3.12.2 + shell: bash jobs: mac_build_and_test: @@ -27,37 +28,50 @@ jobs: os: macos-12 arch: x86_64 python-arch: x64 + generator: "Unix Makefiles" + nproc: 3 + targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - macos_dev_target: 13.0 os: macos-14 arch: arm64 python-arch: arm64 + generator: "Unix Makefiles" + nproc: 3 + targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor + - os: ubuntu-latest + generator: "Unix Makefiles" + nproc: 4 + targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor + - os: windows-2019 + generator: "Visual Studio 16 2019" + nproc: 4 + targets: energyplus ExpandObjects_build ReadVarsESO_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build + - os: windows-2022 + generator: "Visual Studio 17 2022" + nproc: 4 + targets: energyplus ExpandObjects_build ReadVarsESO_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build steps: - # TODO: I assume since we aren't doing any packaging here, we can just use actions/setup-python...but leaving this for now - - name: Remove python ${{ env.Python_REQUIRED_VERSION }} from the toolcache - run: | - ls $RUNNER_TOOL_CACHE/Python || true - rm -Rf "$RUNNER_TOOL_CACHE/Python/${{ env.Python_REQUIRED_VERSION }}" - rm -Rf "$RUNNER_TOOL_CACHE/Python/${{ env.Python_REQUIRED_VERSION }}*/" - name: Set up Python ${{ env.Python_REQUIRED_VERSION }} id: setup-python - uses: jmarrec/setup-python@v5 + uses: actions/setup-python@v5 with: python-version: ${{ env.Python_REQUIRED_VERSION }} - - name: Install System dependencies - shell: bash - run: | - set -x - brew update - # The MACOSX_DEPLOYMENT_TARGET environment variable sets the default value for the CMAKE_OSX_DEPLOYMENT_TARGET variable. - # We use cmake commands to build some subprojects, so setting it globally - echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV - echo "Installing gcc@13 for gfortran support of -static-libquadmath" - brew list gcc@13 || brew install gcc@13 - which gfortran-13 || echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV - brew install ninja + - name: Set MacOS Dev Target + run: echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV + +# - name: Install System dependencies +# run: | +# set -x +# brew update +# # The MACOSX_DEPLOYMENT_TARGET environment variable sets the default value for the CMAKE_OSX_DEPLOYMENT_TARGET variable. +# # We use cmake commands to build some subprojects, so setting it globally +# echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV +# echo "Installing gcc@13 for gfortran support of -static-libquadmath" +# brew list gcc@13 || brew install gcc@13 +# which gfortran-13 || echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV # BUILD AND TEST INTEGRATION FILES ON THE BASELINE BRANCH @@ -72,25 +86,28 @@ jobs: - name: Baseline Configure CMake working-directory: ./baseline/build - shell: bash run: | - cmake -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release \ + cmake -G "${{ matrix.generator }}" \ + -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ - -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ + -DLINK_WITH_PYTHON:BOOL=ON \ + -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ - -DBUILD_TESTING:BOOL=ON -DBUILD_FORTRAN:BOOL=ON -DBUILD_PACKAGE:BOOL=OFF \ + -DBUILD_TESTING:BOOL=ON \ + -DBUILD_FORTRAN:BOOL=ON \ + -DBUILD_PACKAGE:BOOL=OFF \ -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ - -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ + -DENABLE_OPENMP:BOOL=OFF \ + -DUSE_OpenMP:BOOL=OFF \ ../ - name: Baseline Build working-directory: ./baseline/build - shell: bash - run: ninja + # Building specific target list so that we don't waste time building the unit test binary + run: cmake --build . -j ${{ matrix.nproc }} --config Release --target ${{ matrix.targets }} - name: Baseline Test working-directory: ./baseline/build - shell: bash run: ctest -E Basement -R integration -j 3 # TODO: Speed up basement so we don't have to skip it. # BUILD AND TEST EVERYTHING ON THE CURRENT BRANCH @@ -105,25 +122,27 @@ jobs: - name: Branch Configure CMake working-directory: ./branch/build - shell: bash run: | - cmake -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release \ + cmake -G "${{ matrix.generator }}" \ + -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ - -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ + -DLINK_WITH_PYTHON:BOOL=ON \ + -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ - -DBUILD_TESTING:BOOL=ON -DBUILD_FORTRAN:BOOL=ON -DBUILD_PACKAGE:BOOL=OFF \ + -DBUILD_TESTING:BOOL=ON \ + -DBUILD_FORTRAN:BOOL=ON \ + -DBUILD_PACKAGE:BOOL=OFF \ -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ - -DENABLE_OPENMP:BOOL=OFF -DUSE_OpenMP:BOOL=OFF \ + -DENABLE_OPENMP:BOOL=OFF \ + -DUSE_OpenMP:BOOL=OFF \ ../ - name: Branch Build working-directory: ./branch/build - shell: bash - run: ninja + run: cmake --build . -j ${{ matrix.nproc }} --config Release - name: Branch Test working-directory: ./branch/build - shell: bash run: ctest -E Basement -j 3 # TODO: Allow this to fail temporarily but continue so that we also get regression results @@ -132,6 +151,3 @@ jobs: - name: Run Regressions run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ - - # NOW INSTALL THE REGRESSION TOOL AND DO COMPARISONS OF THE INTEGRATION RESULTS - # TODO: Add regression builds, as well as use GHA caching to store baseline branch results for efficiency From 2cec22231069de4def0d4d0baa986779bf04da8c Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 11:38:33 -0500 Subject: [PATCH 10/78] Try again --- .../{mac_build.yml => build_and_test.yml} | 80 ++++++++++--------- 1 file changed, 44 insertions(+), 36 deletions(-) rename .github/workflows/{mac_build.yml => build_and_test.yml} (70%) diff --git a/.github/workflows/mac_build.yml b/.github/workflows/build_and_test.yml similarity index 70% rename from .github/workflows/mac_build.yml rename to .github/workflows/build_and_test.yml index 01e3fde8d90..eca317fac6b 100644 --- a/.github/workflows/mac_build.yml +++ b/.github/workflows/build_and_test.yml @@ -1,4 +1,4 @@ -name: Mac Build +name: Build And Test on: push: @@ -16,37 +16,43 @@ env: shell: bash jobs: - mac_build_and_test: + build_and_test: runs-on: ${{ matrix.os }} continue-on-error: false strategy: fail-fast: false matrix: - macos_dev_target: [12.1, 13.0] + os: [macos-12, macos-14, ubuntu-latest, windows-2019, windows-latest] include: - - macos_dev_target: 12.1 - os: macos-12 + - os: macos-12 + macos_dev_target: 12.1 arch: x86_64 python-arch: x64 generator: "Unix Makefiles" nproc: 3 targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - - macos_dev_target: 13.0 - os: macos-14 + - os: macos-14 + macos_dev_target: 13.0 arch: arm64 python-arch: arm64 generator: "Unix Makefiles" nproc: 3 targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - os: ubuntu-latest + arch: x86_64 + python-arch: x64 generator: "Unix Makefiles" nproc: 4 targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - os: windows-2019 + arch: x86_64 + python-arch: x64 generator: "Visual Studio 16 2019" nproc: 4 targets: energyplus ExpandObjects_build ReadVarsESO_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build - os: windows-2022 + arch: x86_64 + python-arch: x64 generator: "Visual Studio 17 2022" nproc: 4 targets: energyplus ExpandObjects_build ReadVarsESO_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build @@ -86,20 +92,21 @@ jobs: - name: Baseline Configure CMake working-directory: ./baseline/build - run: | - cmake -G "${{ matrix.generator }}" \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ - -DLINK_WITH_PYTHON:BOOL=ON \ - -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ - -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ - -DBUILD_TESTING:BOOL=ON \ - -DBUILD_FORTRAN:BOOL=ON \ - -DBUILD_PACKAGE:BOOL=OFF \ - -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ - -DENABLE_OPENMP:BOOL=OFF \ - -DUSE_OpenMP:BOOL=OFF \ - ../ + run: > + cmake + -G "${{ matrix.generator }}" + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} + -DLINK_WITH_PYTHON:BOOL=ON + -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} + -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ + -DBUILD_TESTING:BOOL=ON + -DBUILD_FORTRAN:BOOL=ON + -DBUILD_PACKAGE:BOOL=OFF + -DDOCUMENTATION_BUILD:STRING=DoNotBuild + -DENABLE_OPENMP:BOOL=OFF + -DUSE_OpenMP:BOOL=OFF + ../ - name: Baseline Build working-directory: ./baseline/build @@ -122,20 +129,21 @@ jobs: - name: Branch Configure CMake working-directory: ./branch/build - run: | - cmake -G "${{ matrix.generator }}" \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} \ - -DLINK_WITH_PYTHON:BOOL=ON \ - -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} \ - -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ \ - -DBUILD_TESTING:BOOL=ON \ - -DBUILD_FORTRAN:BOOL=ON \ - -DBUILD_PACKAGE:BOOL=OFF \ - -DDOCUMENTATION_BUILD:STRING=DoNotBuild \ - -DENABLE_OPENMP:BOOL=OFF \ - -DUSE_OpenMP:BOOL=OFF \ - ../ + run: > + cmake + -G "${{ matrix.generator }}" + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} + -DLINK_WITH_PYTHON:BOOL=ON + -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} + -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ + -DBUILD_TESTING:BOOL=ON + -DBUILD_FORTRAN:BOOL=ON + -DBUILD_PACKAGE:BOOL=OFF + -DDOCUMENTATION_BUILD:STRING=DoNotBuild + -DENABLE_OPENMP:BOOL=OFF + -DUSE_OpenMP:BOOL=OFF + ../ - name: Branch Build working-directory: ./branch/build @@ -144,7 +152,7 @@ jobs: - name: Branch Test working-directory: ./branch/build run: ctest -E Basement -j 3 - # TODO: Allow this to fail temporarily but continue so that we also get regression results + continue-on-error: true # allow unit tests to fail but still try to run regressions next - name: Install Regression Tool run: pip install energyplus-regressions # TODO: Not sure if we need to use a different python path here, and we could pip it or clone it From 17138bbcedf1dd5b606b4cc271b6b5f386262089 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 11:47:55 -0500 Subject: [PATCH 11/78] Cleanup on aisle build_and_test --- .github/workflows/build_and_test.yml | 37 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index eca317fac6b..770921f55a7 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -11,7 +11,6 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FC: gfortran-13 - SDKROOT: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk Python_REQUIRED_VERSION: 3.12.2 shell: bash @@ -22,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-12, macos-14, ubuntu-latest, windows-2019, windows-latest] + os: [macos-12, macos-14, ubuntu-latest, windows-2019, windows-2022] include: - os: macos-12 macos_dev_target: 12.1 @@ -65,19 +64,27 @@ jobs: with: python-version: ${{ env.Python_REQUIRED_VERSION }} - - name: Set MacOS Dev Target - run: echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV - -# - name: Install System dependencies -# run: | -# set -x -# brew update -# # The MACOSX_DEPLOYMENT_TARGET environment variable sets the default value for the CMAKE_OSX_DEPLOYMENT_TARGET variable. -# # We use cmake commands to build some subprojects, so setting it globally -# echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV -# echo "Installing gcc@13 for gfortran support of -static-libquadmath" -# brew list gcc@13 || brew install gcc@13 -# which gfortran-13 || echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV + - name: Install Dependencies for Mac + if: ${{ runner.os == 'macOS' }} + run: > + brew update + brew install gcc@13 + echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV + echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV + + - name: Install Dependencies for Linux + if: ${{ runner.os == 'Linux' }} + run: > + sudo apt-get update + sudo apt-get install libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev + if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then + # https://github.com/actions/runner-images/issues/10025 + echo "FC=gfortran-13" >> $GITHUB_ENV + fi + +# - name: Install Dependencies for Windows +# if: ${{ runner.os == 'Windows' }} +# run: choco install cmake --version=3.28.4 --installargs 'ADD_CMAKE_TO_PATH=System' # BUILD AND TEST INTEGRATION FILES ON THE BASELINE BRANCH From 3872334681a1b37a006ab991b62f669807b9eb87 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 12:09:28 -0500 Subject: [PATCH 12/78] Be a little nicer with regression output --- scripts/dev/gha_regressions.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index 17cb6d80a8e..5bf18d08b3f 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -244,9 +244,7 @@ def single_file_regressions(baseline: Path, modified: Path) -> [bool, bool, bool has_small_diffs = True print_warning(idf, "StdErr diffs.") - print(f"*** Regressions for file {idf}: {success=}, {has_small_diffs=}, {has_diffs=}") - print(f"{json.dumps(entry.to_dict(), indent=4)}\n") - return success, has_small_diffs, has_diffs + return success, has_small_diffs, has_diffs, entry def check_all_regressions(base_testfiles: Path, mod_testfiles: Path) -> bool: @@ -258,11 +256,16 @@ def check_all_regressions(base_testfiles: Path, mod_testfiles: Path) -> bool: modified = mod_testfiles / baseline.name if not modified.exists(): continue # TODO: Should we warn that it is missing? - success, small_diffs, big_diffs = single_file_regressions(baseline, modified) + success, small_diffs, big_diffs, entry = single_file_regressions(baseline, modified) if small_diffs or big_diffs: any_diffs = True + print(f"*** Regressions for file {baseline.name}: {success=}, {small_diffs=}, {big_diffs=}") + print(f"{json.dumps(entry.to_dict(), indent=4)}\n") if not success: any_failures = True + print(f"*** FAILURE for file {baseline.name}\n") + if success and not any_diffs: + print(f"*** No regressions or failures found for {baseline.name}\n") return any_diffs or any_failures From 4fec0f280a160d4df746c954accce0e803ce761f Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 12:11:02 -0500 Subject: [PATCH 13/78] Use ReadVars project name on windows --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 770921f55a7..557faa77a85 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -48,13 +48,13 @@ jobs: python-arch: x64 generator: "Visual Studio 16 2019" nproc: 4 - targets: energyplus ExpandObjects_build ReadVarsESO_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build + targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build - os: windows-2022 arch: x86_64 python-arch: x64 generator: "Visual Studio 17 2022" nproc: 4 - targets: energyplus ExpandObjects_build ReadVarsESO_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build + targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build steps: From f371e53d2d36b4f5d9af9a9a75ffbac56ae6a31e Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 12:13:38 -0500 Subject: [PATCH 14/78] Use | multiline yaml --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 557faa77a85..a70bb53279f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -66,7 +66,7 @@ jobs: - name: Install Dependencies for Mac if: ${{ runner.os == 'macOS' }} - run: > + run: | brew update brew install gcc@13 echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV @@ -74,7 +74,7 @@ jobs: - name: Install Dependencies for Linux if: ${{ runner.os == 'Linux' }} - run: > + run: | sudo apt-get update sudo apt-get install libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then From 31592ae5851b9d816a9dcdd193a1dfb6d555f338 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 12:21:59 -0500 Subject: [PATCH 15/78] Use proper conditional for ubuntu FC assignment --- .github/workflows/build_and_test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a70bb53279f..6933c0b436d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -17,11 +17,9 @@ env: jobs: build_and_test: runs-on: ${{ matrix.os }} - continue-on-error: false strategy: fail-fast: false matrix: - os: [macos-12, macos-14, ubuntu-latest, windows-2019, windows-2022] include: - os: macos-12 macos_dev_target: 12.1 @@ -37,7 +35,7 @@ jobs: generator: "Unix Makefiles" nproc: 3 targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - - os: ubuntu-latest + - os: ubuntu-24.04 arch: x86_64 python-arch: x64 generator: "Unix Makefiles" From e68cf8dad16df2f92683575c6d56fdfc5fb0ddc1 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 12:39:33 -0500 Subject: [PATCH 16/78] Add -C release for ctest --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6933c0b436d..44061cd55d0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -120,7 +120,7 @@ jobs: - name: Baseline Test working-directory: ./baseline/build - run: ctest -E Basement -R integration -j 3 # TODO: Speed up basement so we don't have to skip it. + run: ctest -C Release -E Basement -R integration -j 3 # TODO: Speed up basement so we don't have to skip it. # BUILD AND TEST EVERYTHING ON THE CURRENT BRANCH @@ -156,7 +156,7 @@ jobs: - name: Branch Test working-directory: ./branch/build - run: ctest -E Basement -j 3 + run: ctest -C Release -E Basement -j 3 continue-on-error: true # allow unit tests to fail but still try to run regressions next - name: Install Regression Tool From 59c1c82e02850eb6aad5b80bb51f2bc7c6f34eaa Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 13:29:46 -0500 Subject: [PATCH 17/78] Bump python version for 24.04 support --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 44061cd55d0..2ae741df066 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -11,7 +11,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FC: gfortran-13 - Python_REQUIRED_VERSION: 3.12.2 + Python_REQUIRED_VERSION: 3.12.3 # 3.12.2 not available on Ubuntu 24 GHA shell: bash jobs: From d55edd01b008abe4f87e32856cc154dfb48437ad Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Fri, 23 Aug 2024 14:55:52 -0500 Subject: [PATCH 18/78] Minor clean in new yaml, delete unused yamls --- .github/disabled-workflows/mac_test.yml | 95 -------- .github/disabled-workflows/release.yml | 211 ------------------ .github/disabled-workflows/windows_test.yml | 36 --- .github/unfinished-workflows/linux_test.yml | 43 ---- .../unfinished-workflows/linux_test_debug.yml | 45 ---- .github/workflows/build_and_test.yml | 4 - 6 files changed, 434 deletions(-) delete mode 100644 .github/disabled-workflows/mac_test.yml delete mode 100644 .github/disabled-workflows/release.yml delete mode 100644 .github/disabled-workflows/windows_test.yml delete mode 100644 .github/unfinished-workflows/linux_test.yml delete mode 100644 .github/unfinished-workflows/linux_test_debug.yml diff --git a/.github/disabled-workflows/mac_test.yml b/.github/disabled-workflows/mac_test.yml deleted file mode 100644 index d0ea8050acc..00000000000 --- a/.github/disabled-workflows/mac_test.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Mac Testing - -on: [push] - -env: - FC: /usr/local/bin/gfortran-9 - DO_REGRESSIONS: false - INSTALL_DEPENDENCIES: true - -jobs: - build: - runs-on: macos-10.15 - - steps: - - name: Checkout Branch - uses: actions/checkout@v2 - with: - path: 'clone_branch' - - - name: Checkout Baseline - if: ${{ env.DO_REGRESSIONS == 'true' }} - uses: actions/checkout@v2 - with: - repository: 'NREL/EnergyPlus' - ref: 'develop' - path: 'clone_baseline' - fetch-depth: '1' - - - name: Checkout Regressions - if: ${{ env.DO_REGRESSIONS == 'true' }} - uses: actions/checkout@v2 - with: - repository: 'NREL/EnergyPlusRegressionTool' - ref: 'master' - path: 'clone_regressions' - fetch-depth: '1' - - - name: Set up Python 3.7 - if: ${{ env.INSTALL_DEPENDENCIES == 'true' }} - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Install Python dependencies - if: ${{ env.INSTALL_DEPENDENCIES == 'true' && env.DO_REGRESSIONS == 'true'}} - run: | - python -m pip install --upgrade pip - pip install beautifulsoup4 soupsieve boto - - - name: Create Baseline Build Directory - if: ${{ env.DO_REGRESSIONS == 'true' }} - run: cmake -E make_directory ${{runner.workspace}}/EnergyPlus/clone_baseline/build - - - name: Configure Baseline - if: ${{ env.DO_REGRESSIONS == 'true' }} - shell: bash - working-directory: ${{runner.workspace}}/EnergyPlus/clone_baseline/build - run: cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_FORTRAN=ON -DBUILD_TESTING:BOOL=ON -DCOMMIT_SHA=$COMMIT_SHA -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DLINK_WITH_PYTHON=ON $GITHUB_WORKSPACE/clone_baseline - - - name: Build Baseline - if: ${{ env.DO_REGRESSIONS == 'true' }} - working-directory: ${{runner.workspace}}/EnergyPlus/clone_baseline/build - shell: bash - run: cmake --build . -j 2 - - - name: Test Baseline - if: ${{ env.DO_REGRESSIONS == 'true' }} - working-directory: ${{runner.workspace}}/EnergyPlus/clone_baseline/build - shell: bash - run: ctest -R 1ZoneUncontrolled - - - name: Create Branch Build Directory - run: cmake -E make_directory ${{runner.workspace}}/EnergyPlus/clone_branch/build - - - name: Configure Branch without Regressions - if: ${{ env.DO_REGRESSIONS != 'true' }} - shell: bash - working-directory: ${{runner.workspace}}/EnergyPlus/clone_branch/build - run: cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_FORTRAN=ON -DBUILD_TESTING:BOOL=ON -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DLINK_WITH_PYTHON=ON .. - - - name: Configure Branch with Regressions - if: ${{ env.DO_REGRESSIONS == 'true' }} - shell: bash - working-directory: ${{runner.workspace}}/EnergyPlus/clone_branch/build - run: cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_FORTRAN=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=${{runner.workspace}}/clone_baseline/build -DREGRESSION_SCRIPT_PATH:PATH=${{runner.workspace}}/clone_regressions/build -DREGRESSION_BASELINE_SHA:STRING=UNNKOWN_SHA -DCOMMIT_SHA=${{github.sha}} -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DLINK_WITH_PYTHON=ON .. - - - name: Build Branch - working-directory: ${{runner.workspace}}/EnergyPlus/clone_branch/build - shell: bash - run: cmake --build . -j 2 - - - name: Test Branch - working-directory: ${{runner.workspace}}/EnergyPlus/clone_branch/build - shell: bash - run: ctest -j 2 diff --git a/.github/disabled-workflows/release.yml b/.github/disabled-workflows/release.yml deleted file mode 100644 index d61aa7beebd..00000000000 --- a/.github/disabled-workflows/release.yml +++ /dev/null @@ -1,211 +0,0 @@ -name: Releases - -on: - push: - tags: - - '*' - -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILD_TYPE: Release - -jobs: - release: - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.allow_failure }} - strategy: - # fail-fast: Default is true, switch to false to allow one platform to fail and still run others - fail-fast: false - matrix: - build_name: [Windows_x64, Windows_x86, Ubuntu_18.04, Ubuntu_20.04, macOS_10.15] - include: - - build_name: Windows_x64 - os: windows-2019 - arch: x64 - allow_failure: false - CMAKE_GENERATOR_PLATFORM: x64 - package-arch: x86_64 - BINARY_EXT: exe - COMPRESSED_EXT: zip - QT_OS_NAME: windows - - build_name: Windows_x86 - os: windows-2019 - arch: x86 - allow_failure: false - CMAKE_GENERATOR_PLATFORM: Win32 - package-arch: i386 - BINARY_EXT: exe - COMPRESSED_EXT: zip - QT_OS_NAME: windows - - build_name: Ubuntu_18.04 - os: ubuntu-18.04 - arch: x64 - allow_failure: false - package-arch: x86_64 - BINARY_EXT: run - COMPRESSED_EXT: tar.gz - SH_EXT: sh - QT_OS_NAME: linux - - build_name: Ubuntu_20.04 - os: ubuntu-20.04 - arch: x64 - allow_failure: false - package-arch: x86_64 - BINARY_EXT: run - COMPRESSED_EXT: tar.gz - SH_EXT: sh - QT_OS_NAME: linux - - build_name: macOS_10.15 - os: macos-10.15 - arch: x64 - allow_failure: false - package-arch: x86_64 - BINARY_EXT: dmg - COMPRESSED_EXT: tar.gz - SH_EXT: sh - QT_OS_NAME: mac - MACOSX_DEPLOYMENT_TARGET: 10.15 - SDKROOT: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - architecture: ${{ matrix.arch }} - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install aqtinstall - - - name: Install System dependencies and LaTeX - shell: bash - run: | - set -x - if [ "$RUNNER_OS" == "Linux" ]; then - echo "Using Apt to install dependencies" - sudo apt update - sudo apt install texlive texlive-xetex texlive-science libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev - - elif [ "$RUNNER_OS" == "macOS" ]; then - echo "Setting up MACOSX_DEPLOYMENT_TARGET and SDKROOT" - echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.MACOSX_DEPLOYMENT_TARGET }} >> $GITHUB_ENV - echo SDKROOT=${{ matrix.SDKROOT }} >> $GITHUB_ENV - # The MACOSX_DEPLOYMENT_TARGET environment variable sets the default value for the CMAKE_OSX_DEPLOYMENT_TARGET variable. - # echo CMAKE_MACOSX_DEPLOYMENT_TARGET='-DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET' >> $GITHUB_ENV - - echo "Using brew to install mactex and adding it to PATH" - brew cask install mactex - echo "/Library/TeX/texbin" >> $GITHUB_PATH - - echo "Setting FC (fortran compiler)" - echo FC=/usr/local/bin/gfortran-9 >> $GITHUB_ENV - - elif [ "$RUNNER_OS" == "Windows" ]; then - - echo "Setting CMAKE_GENERATOR options equivalent to ='-G \"Visual Studio 16 2019\" -A ${{ matrix.CMAKE_GENERATOR_PLATFORM }}'" - echo CMAKE_GENERATOR='Visual Studio 16 2019' >> $GITHUB_ENV - echo CMAKE_GENERATOR_PLATFORM=${{ matrix.CMAKE_GENERATOR_PLATFORM }} >> $GITHUB_ENV - - # echo FC="C:/msys64/mingw64/bin/x86_64-w64-mingw32-gfortran.exe" >> $GITHUB_ENV - - echo "Downloading MiKTeX CLI installer" - # We download from a specific miror already - curl -L -O https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64/miktexsetup-4.0-x64.zip - unzip miktexsetup-4.0-x64.zip - - echo "Setting up the local package directory via download" - ./miktexsetup --verbose \ - --local-package-repository=C:/ProgramData/MiKTeX-Repo \ - --remote-package-repository="https://ctan.math.illinois.edu/systems/win32/miktex/tm/packages/" \ - --package-set=basic \ - download - - echo "Installing from the local package directory previously set up" - ./miktexsetup --verbose \ - --local-package-repository=C:/ProgramData/MiKTeX-Repo \ - --package-set=basic \ - --shared \ - install - - echo "Adding MiKTeX bin folder to PATH and to GITHUB_PATH" - echo "C:/Program Files/MiKTeX/miktex/bin/x64/" >> $GITHUB_PATH - export PATH="/c/Program Files/MiKTeX/miktex/bin/x64/:$PATH" - - echo "Configuring MiKTeX to install missing packages on the fly" - initexmf --admin --verbose --set-config-value='[MPM]AutoInstall=1' - - echo "Configure default mirror for packages" - mpm --admin --set-repository="https://ctan.math.illinois.edu/systems/win32/miktex/tm/packages/" - # Avoid annoying warning: "xelatex: major issue: So far, you have not checked for updates as a MiKTeX user." - mpm --find-updates - mpm --admin --find-updates - fi; - - #- name: Build Test Document (will install missing packages) - #working-directory: ./doc/test - #shell: bash - #run: | - #set -x - #xelatex dependencies.tex - - - name: Install IFW - shell: bash - run: | - set -x - out_dir=${{ runner.workspace }}/Qt - if [ "$RUNNER_OS" == "Windows" ]; then - out_dir="C:/Qt" - fi; - - aqt tool ${{ matrix.QT_OS_NAME }} tools_ifw 3.2.2 qt.tools.ifw.32 --outputdir="$out_dir" - echo "$out_dir/Tools/QtInstallerFramework/3.2/bin" >> $GITHUB_PATH - - - name: Create Build Directory - run: cmake -E make_directory ./build/ - - - name: Configure CMake - working-directory: ./build - shell: bash - run: | - set -x - cmake -DLINK_WITH_PYTHON=ON -DBUILD_FORTRAN=ON -DBUILD_PACKAGE:BOOL=ON \ - -DDOCUMENTATION_BUILD="BuildWithAll" -DTEX_INTERACTION="batchmode" \ - ../ - - - name: Build Package - working-directory: ./build - shell: bash - run: cmake --build . --target package -j 2 --config $BUILD_TYPE - - - name: Upload Zip to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: build/EnergyPlus-*-${{ matrix.package-arch }}.${{ matrix.COMPRESSED_EXT }} - tag: ${{ github.ref }} - overwrite: true - file_glob: true - - - name: Upload IFW to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: build/EnergyPlus-*-${{ matrix.package-arch }}.${{ matrix.BINARY_EXT }} - tag: ${{ github.ref }} - overwrite: true - file_glob: true - - - name: Upload SH to release - if: runner.os == 'Linux' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: build/EnergyPlus-*-${{ matrix.package-arch }}.${{ matrix.SH_EXT }} - tag: ${{ github.ref }} - overwrite: true - file_glob: true - diff --git a/.github/disabled-workflows/windows_test.yml b/.github/disabled-workflows/windows_test.yml deleted file mode 100644 index 8d31333d916..00000000000 --- a/.github/disabled-workflows/windows_test.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Windows 64-bit Testing - -on: [push] - -env: - CMAKE_Fortran_COMPILER: "/c/msys64/mingw64/bin/x86_64-w64-mingw32-gfortran.exe" - -jobs: - windows_test_64: - runs-on: windows-2019 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - architecture: 'x64' - - - name: Create Build Directory - run: cmake -E make_directory ${{runner.workspace}}/EnergyPlus/build - - - name: Configure CMake - shell: bash - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: cmake -G "Visual Studio 16 2019" -A x64 -DLINK_WITH_PYTHON=ON -DBUILD_TESTING=ON -DBUILD_FORTRAN=ON .. - - - name: Build EnergyPlus - working-directory: ${{runner.workspace}}/EnergyPlus/build - shell: bash - run: cmake --build . -j 2 --config Release - - - name: Run Tests - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: ctest -j 2 -C Release diff --git a/.github/unfinished-workflows/linux_test.yml b/.github/unfinished-workflows/linux_test.yml deleted file mode 100644 index 869b13c94c7..00000000000 --- a/.github/unfinished-workflows/linux_test.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Linux Test - Release Mode - -on: [push] - -jobs: - build: - runs-on: ubuntu-18.04 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Install APT dependencies - # install valgrind and performance test stuff - run: sudo apt-get update && sudo apt-get install libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev - - - name: Create Build Directory - run: cmake -E make_directory ${{runner.workspace}}/EnergyPlus/build - - - name: Configure CMake - working-directory: ${{runner.workspace}}/EnergyPlus/build - # turn on performance testing - run: cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DLINK_WITH_PYTHON=ON -DDOCUMENTATION_BUILD="BuildWithAll" -DTEX_INTERACTION="batchmode" -DBUILD_FORTRAN=ON -DBUILD_PACKAGE:BOOL=ON .. - - - name: Build EnergyPlus - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: cmake --build . -j 2 - - - name: Run EnergyPlus Tests - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: ctest -j 2 - - # collect performance results here, upload them - - - name: Run EnergyPlus Integration Tests - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: ctest -j 2 -R "integration.*" - - # get coverage results here, upload them diff --git a/.github/unfinished-workflows/linux_test_debug.yml b/.github/unfinished-workflows/linux_test_debug.yml deleted file mode 100644 index 23e90e2e8c8..00000000000 --- a/.github/unfinished-workflows/linux_test_debug.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Linux Test - Debug Mode - -on: [push] - -jobs: - build: - runs-on: ubuntu-18.04 - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - # install boto to upload coverage results - - - name: Install APT dependencies - # install gcovr and coverage stuff - run: sudo apt-get update && sudo apt-get install libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev - - - name: Create Build Directory - run: cmake -E make_directory ${{runner.workspace}}/EnergyPlus/build - - - name: Configure CMake - working-directory: ${{runner.workspace}}/EnergyPlus/build - # turn on ENABLE_COVERAGE - run: cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DLINK_WITH_PYTHON=ON -DDOCUMENTATION_BUILD="BuildWithAll" -DTEX_INTERACTION="batchmode" -DBUILD_FORTRAN=ON -DBUILD_PACKAGE:BOOL=ON .. - - - name: Build EnergyPlus - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: cmake --build . -j 2 - - - name: Run EnergyPlus Unit Tests - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: ctest -j 2 -E "integration.*" - - # get coverage results here, upload them, then clear them - - - name: Run EnergyPlus Integration Tests - working-directory: ${{runner.workspace}}/EnergyPlus/build - run: ctest -j 2 -R "integration.*" - - # get coverage results here, upload them diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 2ae741df066..ee75e6e6fa1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -80,10 +80,6 @@ jobs: echo "FC=gfortran-13" >> $GITHUB_ENV fi -# - name: Install Dependencies for Windows -# if: ${{ runner.os == 'Windows' }} -# run: choco install cmake --version=3.28.4 --installargs 'ADD_CMAKE_TO_PATH=System' - # BUILD AND TEST INTEGRATION FILES ON THE BASELINE BRANCH - name: Baseline Checkout From 4fedbab57d133c1f592550b3987df12a4bd46fd4 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Mon, 26 Aug 2024 16:11:27 -0500 Subject: [PATCH 19/78] Test only for now, lots of changes --- .github/workflows/build_and_test.yml | 21 +- scripts/dev/gha_regressions.py | 502 ++++++++++++++++----------- 2 files changed, 305 insertions(+), 218 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ee75e6e6fa1..d60e7a53a5c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1,12 +1,8 @@ name: Build And Test on: - push: - branches: [ develop ] - tags: - - 'v*' # Push events matching v*, i.e. v1.0, v20.15.10 pull_request: - branches: [ develop ] + branches: [ develop ] # run this on any PR pointing to develop env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -147,16 +143,25 @@ jobs: ../ - name: Branch Build + id: branch_build working-directory: ./branch/build run: cmake --build . -j ${{ matrix.nproc }} --config Release - name: Branch Test working-directory: ./branch/build run: ctest -C Release -E Basement -j 3 - continue-on-error: true # allow unit tests to fail but still try to run regressions next - name: Install Regression Tool - run: pip install energyplus-regressions # TODO: Not sure if we need to use a different python path here, and we could pip it or clone it + if: always() && steps.branch_build.outcome != 'failure' # always run this step as long as we actually built + run: pip install energyplus-regressions - name: Run Regressions - run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ + if: always() && steps.branch_build.outcome != 'failure' # always run this step as long as we actually built + id: regressions + run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ ./branch/build/regressions + + - uses: actions/upload-artifact@v4 + if: always() && steps.regressions.outcome == 'failure' # only run this if regressions were encountered "failed" + with: + name: "regressions-${{ matrix.os }}" + path: "${{ github.workspace }}/build/regressions" diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index 5bf18d08b3f..590c3f77e49 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -53,227 +53,309 @@ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. - +from collections import defaultdict import json -import sys +from shutil import copy from pathlib import Path +import sys +import tempfile from energyplus_regressions.runtests import SuiteRunner from energyplus_regressions.structures import TextDifferences, TestEntry -def print_notice(relative_test_file: str, message: str) -> None: - # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" - print(f"::notice file=testfiles/{relative_test_file}::{message}") - - -def print_warning(relative_test_file: str, message: str) -> None: - # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" - print(f"::warning file=testfiles/{relative_test_file}::{message}") - - -def print_error(relative_test_file: str, message: str) -> None: - # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" - print(f"::error file=testfiles/{relative_test_file}::{message}") - - -def process_diffs(relative_test_file: str, diff_name, diffs, this_has_diffs, this_has_small_diffs): - # relative_test_file should be relative to the testfiles/ directory, like "1Zone.idf" or "advanced/2a.idf" - if not diffs: - return this_has_diffs, this_has_small_diffs - if diffs.diff_type == 'Big Diffs': - this_has_diffs = True - print_warning(relative_test_file, f"{diff_name} Big Diffs") - elif diffs.diff_type == 'Small Diffs': - this_has_small_diffs = True - print_warning(relative_test_file, f"{diff_name} Small Diffs") - return this_has_diffs, this_has_small_diffs - - -def single_file_regressions(baseline: Path, modified: Path) -> [bool, bool, bool]: - - import energyplus_regressions - thresholds = Path(energyplus_regressions.__file__).parent / 'diffs' / 'math_diff.config' - - idf = baseline.name - - entry = TestEntry(idf, "") - entry, message = SuiteRunner.process_diffs_for_one_case( - entry, - {'build_dir': str(baseline)}, - {'build_dir': str(modified)}, - "", - str(thresholds), - ci_mode=True - ) # returns an updated entry - - with open('results.json', 'w') as f: - f.write(json.dumps(entry.to_dict(), indent=4)) - - success = True - has_diffs = False - has_small_diffs = False - - # Note, comment out any of the "has_diffs" below if you don't want - # it to generate an error condition - - if entry.aud_diffs and (entry.aud_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "AUD diffs.") - - if entry.bnd_diffs and (entry.bnd_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "BND diffs.") - - if entry.dl_in_diffs and (entry.dl_in_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "delightin diffs.") - - if entry.dl_out_diffs and (entry.dl_out_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "delightout diffs.") - - if entry.dxf_diffs and (entry.dxf_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "DXF diffs.") - - if entry.eio_diffs and (entry.eio_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "EIO diffs.") - - if entry.err_diffs and (entry.err_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "ERR diffs.") - - if entry.readvars_audit_diffs and (entry.readvars_audit_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "ReadvarsAudit diffs.") - - if entry.edd_diffs and (entry.edd_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "EDD diffs.") - - if entry.wrl_diffs and (entry.wrl_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "WRL diffs.") - - if entry.sln_diffs and (entry.sln_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "SLN diffs.") - - if entry.sci_diffs and (entry.sci_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "SCI diffs.") - - if entry.map_diffs and (entry.map_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "MAP diffs.") - - if entry.dfs_diffs and (entry.dfs_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "DFS diffs.") - - if entry.screen_diffs and (entry.screen_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "SCREEN diffs.") - - if entry.glhe_diffs and (entry.glhe_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "GLHE diffs") - - # numeric diff - if entry.eso_diffs: - has_diffs, has_small_diffs = process_diffs(idf, "ESO", entry.eso_diffs, has_diffs, has_small_diffs) - - if entry.mdd_diffs and (entry.mdd_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "MDD diffs.") - - if entry.mtd_diffs and (entry.mtd_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "MTD diffs.") - - # numeric diff - if entry.mtr_diffs: - has_diffs, has_small_diffs = process_diffs(idf, "MTR", entry.mtr_diffs, has_diffs, has_small_diffs) - - if entry.rdd_diffs and (entry.rdd_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "RDD diffs.") - - if entry.shd_diffs and (entry.shd_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "SHD diffs.") - - if entry.perf_log_diffs and (entry.perf_log_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "PERF_LOG diffs.") - - # numeric diff - if entry.ssz_diffs: - has_diffs, has_small_diffs = process_diffs(idf, "SSZ", entry.ssz_diffs, has_diffs, has_small_diffs) - - # numeric diff - if entry.zsz_diffs: - has_diffs, has_small_diffs = process_diffs(idf, "ZSZ", entry.zsz_diffs, has_diffs, has_small_diffs) - - # numeric diff - if entry.json_diffs: - has_diffs, has_small_diffs = process_diffs(idf, "JSON", entry.json_diffs, has_diffs, has_small_diffs) - - if entry.table_diffs: - if entry.table_diffs.big_diff_count > 0: - has_diffs = True - print_warning(idf, "Table big diffs.") - elif entry.table_diffs.small_diff_count > 0: - has_small_diffs = True - print_warning(idf, "Table small diffs.") - if entry.table_diffs.string_diff_count > 1: # There's always one...the time stamp - has_diffs = True - print_warning(idf, "Table string diffs.") - - if entry.idf_diffs and (entry.idf_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "IDF diffs.") - - if entry.stdout_diffs and (entry.stdout_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "StdOut diffs.") - - if entry.stderr_diffs and (entry.stderr_diffs.diff_type != TextDifferences.EQUAL): - has_small_diffs = True - print_warning(idf, "StdErr diffs.") - - return success, has_small_diffs, has_diffs, entry - - -def check_all_regressions(base_testfiles: Path, mod_testfiles: Path) -> bool: - any_diffs = False - any_failures = False - for baseline in base_testfiles.iterdir(): - if not baseline.is_dir(): - continue - modified = mod_testfiles / baseline.name - if not modified.exists(): - continue # TODO: Should we warn that it is missing? - success, small_diffs, big_diffs, entry = single_file_regressions(baseline, modified) - if small_diffs or big_diffs: - any_diffs = True - print(f"*** Regressions for file {baseline.name}: {success=}, {small_diffs=}, {big_diffs=}") - print(f"{json.dumps(entry.to_dict(), indent=4)}\n") - if not success: - any_failures = True - print(f"*** FAILURE for file {baseline.name}\n") - if success and not any_diffs: - print(f"*** No regressions or failures found for {baseline.name}\n") - return any_diffs or any_failures +class RegressionManager: + + def __init__(self): + self.diffs_by_idf = [] + self.diffs_by_type = defaultdict(list) + self.all_files_compared = [] + import energyplus_regressions + self.threshold_file = str(Path(energyplus_regressions.__file__).parent / 'diffs' / 'math_diff.config') + + def single_file_regressions(self, baseline: Path, modified: Path) -> [TestEntry, bool]: + + idf = baseline.name + this_file_diffs = [] + + entry = TestEntry(idf, "") + entry, message = SuiteRunner.process_diffs_for_one_case( + entry, + {'build_dir': str(baseline)}, + {'build_dir': str(modified)}, + "", + self.threshold_file, + ci_mode=True + ) # returns an updated entry + + has_diffs = False + + text_diff_results = { + "Audit": entry.aud_diffs, + "BND": entry.bnd_diffs, + "DELightIn": entry.dl_in_diffs, + "DELightOut": entry.dl_out_diffs, + "DXF": entry.dxf_diffs, + "EIO": entry.eio_diffs, + "ERR": entry.err_diffs, + "Readvars_Audit": entry.readvars_audit_diffs, + "EDD": entry.edd_diffs, + "WRL": entry.wrl_diffs, + "SLN": entry.sln_diffs, + "SCI": entry.sci_diffs, + "MAP": entry.map_diffs, + "DFS": entry.dfs_diffs, + "SCREEN": entry.screen_diffs, + "GLHE": entry.glhe_diffs, + "MDD": entry.mdd_diffs, + "MTD": entry.mtd_diffs, + "RDD": entry.rdd_diffs, + "SHD": entry.shd_diffs, + "PERF_LOG": entry.perf_log_diffs, + "IDF": entry.idf_diffs, + "StdOut": entry.stdout_diffs, + "StdErr": entry.stderr_diffs, + } + for diff_type, diffs in text_diff_results.items(): + if diffs is None: + continue + if diffs.diff_type != TextDifferences.EQUAL: + has_diffs = True + this_file_diffs.append(diff_type) + self.diffs_by_type[diff_type].append(idf) + + numeric_diff_results = { + "ESO": entry.eso_diffs, + "MTR": entry.mtr_diffs, + "SSZ": entry.ssz_diffs, + "ZSZ": entry.zsz_diffs, + "JSON": entry.json_diffs, + } + for diff_type, diffs in numeric_diff_results.items(): + if diffs is None: + continue + if diffs.diff_type == 'Big Diffs': + has_diffs = True + this_file_diffs.append(f"{diff_type} Big Diffs") + self.diffs_by_type[f"{diff_type} Big Diffs"].append(idf) + elif diffs.diff_type == 'Small Diffs': + has_diffs = True + this_file_diffs.append(f"{diff_type} Small Diffs") + self.diffs_by_type[f"{diff_type} Big Diffs"].append(idf) + + if entry.table_diffs: + if entry.table_diffs.big_diff_count > 0: + has_diffs = True + this_file_diffs.append(f"Table Big Diffs") + self.diffs_by_type[f"Table Big Diffs"].append(idf) + elif entry.table_diffs.small_diff_count > 0: + has_diffs = True + this_file_diffs.append(f"Table Small Diffs") + self.diffs_by_type[f"Table Small Diffs"].append(idf) + if entry.table_diffs.string_diff_count > 1: # There's always one...the time stamp + has_diffs = True + this_file_diffs.append(f"Table String Diffs") + self.diffs_by_type[f"Table String Diffs"].append(idf) + + return entry, has_diffs + + @staticmethod + def single_diff_html(contents: str) -> str: + return f""" + + + + + + + + + 
+   
+    {contents}
+   
+
+ + +""" + + @staticmethod + def regression_row_in_single_test_case_html(diff_file_name: str) -> str: + return f""" + + {diff_file_name} + download + view + """ + + @staticmethod + def single_test_case_html(contents: str) -> str: + return f""" + + + + + + + + + + + + + + + + +{contents} +
filename
+ +""" + + @staticmethod + def bundle_root_index_html(header_info: list[str], no_diffs: list[str], diffs: list[str]) -> str: + header_content = "" + for hi in header_info: + header_content += f"""
  • {hi}
    • \n""" + # gather some plural "s" as needed and form up the html entries + num_no_diff = len(no_diffs) + nds = 's' if num_no_diff == 0 or num_no_diff > 1 else '' + no_diff_content = "" + for nd in no_diffs: + no_diff_content += f"""
  • {nd}
    • \n""" + num_diff = len(diffs) + ds = 's' if num_diff == 0 or num_diff > 1 else '' + diff_content = "" + for d in diffs: + diff_content += f"""{d}\n""" + return f""" + + + + + + + + + +
    +
    +
    +
    +

    + Header Metadata +

    +
    + +
    +
    + +
    +
    +
    +

    + {num_no_diff} File{nds} with No Diffs +

    +
    +
    +
    +
      + {no_diff_content} +
    +
    +
    +
    +
    + +
    +
    +
    +

    + {num_diff} File{ds} with Diffs +

    +
    +
    +
    +
      + {diff_content} +
    +
    +
    +
    +
    +
    + +""" + + def check_all_regressions(self, base_testfiles: Path, mod_testfiles: Path, bundle_root: Path) -> bool: + any_diffs = False + root_index_files_no_diff = [] + root_index_files_diffs = [] + for baseline in base_testfiles.iterdir(): + if not baseline.is_dir(): + continue + if baseline.name == 'CMakeFiles': # add more ignore dirs here + continue + modified = mod_testfiles / baseline.name + if not modified.exists(): + continue # TODO: Should we warn that it is missing? + entry, diffs = self.single_file_regressions(baseline, modified) + if diffs: + root_index_files_diffs.append(baseline.name) + any_diffs = True + potential_diff_files = baseline.glob("*.*.*") # TODO: Could try to get this from the regression tool + target_dir_for_this_file_diffs = bundle_root / baseline.name + if potential_diff_files: + target_dir_for_this_file_diffs.mkdir() + index_contents_this_file = "" + for potential_diff_file in potential_diff_files: + copy(potential_diff_file, target_dir_for_this_file_diffs) + diff_file_with_html = target_dir_for_this_file_diffs / (potential_diff_file.name + '.html') + if potential_diff_file.name.endswith('.htm'): + # already a html file, just upload the raw contents but renamed as ...htm.html + copy(potential_diff_file, diff_file_with_html) + else: + # it's not an HTML file, wrap it inside an HTML wrapper in a temp file and send it + contents = potential_diff_file.read_text() + wrapped_contents = self.single_diff_html(contents) + diff_file_with_html.write_text(wrapped_contents) + index_contents_this_file += self.regression_row_in_single_test_case_html(potential_diff_file.name) + index_file = target_dir_for_this_file_diffs / 'index.html' + index_this_file = self.single_test_case_html(index_contents_this_file) + index_file.write_text(index_this_file) + so_far = ' Diffs! ' if any_diffs else 'No diffs' + print(f"Diff status so far: {so_far}, this file HAZ DIFFS: {baseline.name}") + else: + root_index_files_no_diff.append(baseline.name) + so_far = ' Diffs! ' if any_diffs else 'No diffs' + print(f"Diff status so far: {so_far}, this file has no diffs: {baseline.name}") + bundle_root_index_file_path = bundle_root / 'index.html' + bundle_root_index_content = self.bundle_root_index_html( + ['hello', 'world'], root_index_files_no_diff, root_index_files_diffs + ) + bundle_root_index_file_path.write_text(bundle_root_index_content) + + # print(f"*** Regressions for file {baseline.name}: {diffs=}") + # print(f"{json.dumps(entry.to_dict(), indent=4)}\n") + # if not any_diffs: + # print(f"*** No regressions or failures found for {baseline.name}\n") + return any_diffs if __name__ == "__main__": # pragma: no cover - testing function, not the __main__ entry point - if len(sys.argv) < 3: - print("syntax: %s base_dir mod_dir base_sha mod_sha device_id" % sys.argv[0]) + if len(sys.argv) < 4: + print("syntax: %s base_dir mod_dir regression_dir" % sys.argv[0]) sys.exit(1) arg_base_dir = Path(sys.argv[1]) arg_mod_dir = Path(sys.argv[2]) - sys.exit(1 if check_all_regressions(arg_base_dir, arg_mod_dir) else 0) + arg_regression_dir = Path(sys.argv[3]) + rm = RegressionManager() + response = rm.check_all_regressions(arg_base_dir, arg_mod_dir, arg_regression_dir) + sys.exit(1 if response else 0) From 0bff02c8eb09657ae40d13209f71f9020831c138 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Mon, 26 Aug 2024 16:52:53 -0500 Subject: [PATCH 20/78] Nicer messaging, add a diff! --- .github/workflows/build_and_test.yml | 12 ++++++++++++ src/EnergyPlus/ConvectionCoefficients.cc | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d60e7a53a5c..d751943e548 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -161,7 +161,19 @@ jobs: run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ ./branch/build/regressions - uses: actions/upload-artifact@v4 + id: upload_regressions if: always() && steps.regressions.outcome == 'failure' # only run this if regressions were encountered "failed" with: name: "regressions-${{ matrix.os }}" path: "${{ github.workspace }}/build/regressions" + + - uses: actions/github-script@v7 + if: always() && steps.regressions.outcome == 'failure' + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: "# :warning: Regressions Detected\n - Build: ${{ matrix.os }}\n - [View Results](https://github.com/NREL/EnergyPlus/actions/runs/${{ github.run_id }})\n - [Download Regressions](${{ steps.upload_regressions.outputs.artifact-url }})" + }) diff --git a/src/EnergyPlus/ConvectionCoefficients.cc b/src/EnergyPlus/ConvectionCoefficients.cc index 7c33104830f..0fe7fce48ea 100644 --- a/src/EnergyPlus/ConvectionCoefficients.cc +++ b/src/EnergyPlus/ConvectionCoefficients.cc @@ -6299,7 +6299,7 @@ Real64 CalcClearRoof(EnergyPlusData &state, Real64 Rf = RoughnessMultiplier[(int)RoughnessIndex]; if (Rex > 0.1) { // avoid zero and crazy small denominators - Real64 tmp = std::log(1.0 + GrLn / pow_2(Rex)); + Real64 tmp = std::log1p(GrLn / pow_2(Rex)); // std::log(1.0 + GrLn / pow_2(Rex)); eta = tmp / (1.0 + tmp); } else { eta = 1.0; // forced convection gone because no wind From 0871148497482bfcad644ee5739f4411aae09c05 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Mon, 26 Aug 2024 17:02:10 -0500 Subject: [PATCH 21/78] Fix path issue, just trying on Mac14/U24 --- .github/workflows/build_and_test.yml | 42 ++++++++++++++-------------- scripts/dev/gha_regressions.py | 1 + 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d751943e548..f0520e7e749 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -17,13 +17,13 @@ jobs: fail-fast: false matrix: include: - - os: macos-12 - macos_dev_target: 12.1 - arch: x86_64 - python-arch: x64 - generator: "Unix Makefiles" - nproc: 3 - targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor +# - os: macos-12 +# macos_dev_target: 12.1 +# arch: x86_64 +# python-arch: x64 +# generator: "Unix Makefiles" +# nproc: 3 +# targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - os: macos-14 macos_dev_target: 13.0 arch: arm64 @@ -37,18 +37,18 @@ jobs: generator: "Unix Makefiles" nproc: 4 targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - - os: windows-2019 - arch: x86_64 - python-arch: x64 - generator: "Visual Studio 16 2019" - nproc: 4 - targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build - - os: windows-2022 - arch: x86_64 - python-arch: x64 - generator: "Visual Studio 17 2022" - nproc: 4 - targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build +# - os: windows-2019 +# arch: x86_64 +# python-arch: x64 +# generator: "Visual Studio 16 2019" +# nproc: 4 +# targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build +# - os: windows-2022 +# arch: x86_64 +# python-arch: x64 +# generator: "Visual Studio 17 2022" +# nproc: 4 +# targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build steps: @@ -158,14 +158,14 @@ jobs: - name: Run Regressions if: always() && steps.branch_build.outcome != 'failure' # always run this step as long as we actually built id: regressions - run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ ./branch/build/regressions + run: python ./branch/scripts/dev/gha_regressions.py ./baseline/build/testfiles ./branch/build/testfiles/ ${{ github.workspace }}/regressions - uses: actions/upload-artifact@v4 id: upload_regressions if: always() && steps.regressions.outcome == 'failure' # only run this if regressions were encountered "failed" with: name: "regressions-${{ matrix.os }}" - path: "${{ github.workspace }}/build/regressions" + path: "${{ github.workspace }}/regressions" - uses: actions/github-script@v7 if: always() && steps.regressions.outcome == 'failure' diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index 590c3f77e49..ddf2d460eb1 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -297,6 +297,7 @@ def check_all_regressions(self, base_testfiles: Path, mod_testfiles: Path, bundl any_diffs = False root_index_files_no_diff = [] root_index_files_diffs = [] + bundle_root.mkdir(exist_ok=True) for baseline in base_testfiles.iterdir(): if not baseline.is_dir(): continue From 2b258345dacb785cf5e9917aa6ec0f4c3821f9bf Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Mon, 26 Aug 2024 17:50:48 -0500 Subject: [PATCH 22/78] Allow PR permission --- .github/workflows/build_and_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f0520e7e749..3855bee2a1f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -13,6 +13,8 @@ env: jobs: build_and_test: runs-on: ${{ matrix.os }} + permissions: + pull-requests: write strategy: fail-fast: false matrix: From 1a85cdd91be564f55a98300fe0cce5b56f64c6ca Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Mon, 26 Aug 2024 17:51:00 -0500 Subject: [PATCH 23/78] Re-enable all platforms --- .github/workflows/build_and_test.yml | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3855bee2a1f..ca1536ff020 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -19,13 +19,13 @@ jobs: fail-fast: false matrix: include: -# - os: macos-12 -# macos_dev_target: 12.1 -# arch: x86_64 -# python-arch: x64 -# generator: "Unix Makefiles" -# nproc: 3 -# targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor + - os: macos-12 + macos_dev_target: 12.1 + arch: x86_64 + python-arch: x64 + generator: "Unix Makefiles" + nproc: 3 + targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor - os: macos-14 macos_dev_target: 13.0 arch: arm64 @@ -39,18 +39,18 @@ jobs: generator: "Unix Makefiles" nproc: 4 targets: energyplus ExpandObjects ReadVarsESO Slab Basement AppGPostProcess ParametricPreprocessor -# - os: windows-2019 -# arch: x86_64 -# python-arch: x64 -# generator: "Visual Studio 16 2019" -# nproc: 4 -# targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build -# - os: windows-2022 -# arch: x86_64 -# python-arch: x64 -# generator: "Visual Studio 17 2022" -# nproc: 4 -# targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build + - os: windows-2019 + arch: x86_64 + python-arch: x64 + generator: "Visual Studio 16 2019" + nproc: 4 + targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build + - os: windows-2022 + arch: x86_64 + python-arch: x64 + generator: "Visual Studio 17 2022" + nproc: 4 + targets: energyplus ExpandObjects_build ReadVars_build Slab_build Basement_build AppGPostProcess_build ParametricPreprocessor_build steps: From c4e6ed2876d761e01fc714bbfddc9e0c45362b72 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Tue, 27 Aug 2024 09:56:46 -0500 Subject: [PATCH 24/78] Much improved regression page --- .github/workflows/build_and_test.yml | 2 +- scripts/dev/gha_regressions.py | 253 ++++++++++++++++++++------- 2 files changed, 193 insertions(+), 62 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ca1536ff020..c3853d08b8b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -177,5 +177,5 @@ jobs: issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: "# :warning: Regressions Detected\n - Build: ${{ matrix.os }}\n - [View Results](https://github.com/NREL/EnergyPlus/actions/runs/${{ github.run_id }})\n - [Download Regressions](${{ steps.upload_regressions.outputs.artifact-url }})" + body: "# :warning: Regressions Detected\n - Build: ${{ matrix.os }}\n - Commit: ${{ github.action_ref }}\n - [View Results](https://github.com/NREL/EnergyPlus/actions/runs/${{ github.run_id }})\n - [Download Regressions](${{ steps.upload_regressions.outputs.artifact-url }})" }) diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index ddf2d460eb1..bdb6ff0f245 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -54,28 +54,33 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from collections import defaultdict +from datetime import datetime, UTC import json from shutil import copy from pathlib import Path import sys -import tempfile +from shutil import rmtree +from zoneinfo import ZoneInfo from energyplus_regressions.runtests import SuiteRunner -from energyplus_regressions.structures import TextDifferences, TestEntry +from energyplus_regressions.structures import TextDifferences, TestEntry, EndErrSummary class RegressionManager: def __init__(self): - self.diffs_by_idf = [] + self.diffs_by_idf = defaultdict(list) self.diffs_by_type = defaultdict(list) - self.all_files_compared = [] + self.summary_results = {} + self.num_idf_inspected = 0 + # self.all_files_compared = [] TODO: need to get this from regression runner import energyplus_regressions self.threshold_file = str(Path(energyplus_regressions.__file__).parent / 'diffs' / 'math_diff.config') def single_file_regressions(self, baseline: Path, modified: Path) -> [TestEntry, bool]: idf = baseline.name + self.num_idf_inspected += 1 this_file_diffs = [] entry = TestEntry(idf, "") @@ -87,6 +92,7 @@ def single_file_regressions(self, baseline: Path, modified: Path) -> [TestEntry, self.threshold_file, ci_mode=True ) # returns an updated entry + self.summary_results[idf] = entry.summary_result has_diffs = False @@ -123,6 +129,7 @@ def single_file_regressions(self, baseline: Path, modified: Path) -> [TestEntry, has_diffs = True this_file_diffs.append(diff_type) self.diffs_by_type[diff_type].append(idf) + self.diffs_by_idf[idf].append(diff_type) numeric_diff_results = { "ESO": entry.eso_diffs, @@ -138,24 +145,29 @@ def single_file_regressions(self, baseline: Path, modified: Path) -> [TestEntry, has_diffs = True this_file_diffs.append(f"{diff_type} Big Diffs") self.diffs_by_type[f"{diff_type} Big Diffs"].append(idf) + self.diffs_by_idf[idf].append(f"{diff_type} Big Diffs") elif diffs.diff_type == 'Small Diffs': has_diffs = True this_file_diffs.append(f"{diff_type} Small Diffs") - self.diffs_by_type[f"{diff_type} Big Diffs"].append(idf) + self.diffs_by_type[f"{diff_type} Small Diffs"].append(idf) + self.diffs_by_idf[idf].append(f"{diff_type} Small Diffs") if entry.table_diffs: if entry.table_diffs.big_diff_count > 0: has_diffs = True - this_file_diffs.append(f"Table Big Diffs") - self.diffs_by_type[f"Table Big Diffs"].append(idf) + this_file_diffs.append("Table Big Diffs") + self.diffs_by_type["Table Big Diffs"].append(idf) + self.diffs_by_idf[idf].append("Table Big Diffs") elif entry.table_diffs.small_diff_count > 0: has_diffs = True - this_file_diffs.append(f"Table Small Diffs") - self.diffs_by_type[f"Table Small Diffs"].append(idf) + this_file_diffs.append("Table Small Diffs") + self.diffs_by_type["Table Small Diffs"].append(idf) + self.diffs_by_idf[idf].append("Table Small Diffs") if entry.table_diffs.string_diff_count > 1: # There's always one...the time stamp has_diffs = True - this_file_diffs.append(f"Table String Diffs") - self.diffs_by_type[f"Table String Diffs"].append(idf) + this_file_diffs.append("Table String Diffs") + self.diffs_by_type["Table String Diffs"].append(idf) + self.diffs_by_idf[idf].append("Table String Diffs") return entry, has_diffs @@ -212,12 +224,13 @@ def single_test_case_html(contents: str) -> str: """ - @staticmethod - def bundle_root_index_html(header_info: list[str], no_diffs: list[str], diffs: list[str]) -> str: + def bundle_root_index_html(self, header_info: list[str], no_diffs: list[str], diffs: list[str]) -> str: + # set up header table header_content = "" for hi in header_info: header_content += f"""
  • {hi}
    • \n""" - # gather some plural "s" as needed and form up the html entries + + # set up diff summary listings num_no_diff = len(no_diffs) nds = 's' if num_no_diff == 0 or num_no_diff > 1 else '' no_diff_content = "" @@ -228,6 +241,61 @@ def bundle_root_index_html(header_info: list[str], no_diffs: list[str], diffs: l diff_content = "" for d in diffs: diff_content += f"""{d}\n""" + + # set up diff type listing + diff_type_keys = self.diffs_by_type.keys() + num_diff_types = len(diff_type_keys) + dt = 's' if num_diff_types == 0 or num_diff_types > 1 else '' + diff_type_content = "" + if num_diff_types > 0: + for k in diff_type_keys: + nice_type_key = k.lower().replace(' ', '') + diffs_this_type = self.diffs_by_type[k] + num_files_this_type = len(diffs_this_type) + dtt = 's' if num_diff_types == 0 or num_diff_types > 1 else '' + this_diff_type_list = "" + for idf in diffs_this_type: + this_diff_type_list += f"""{idf}\n""" + diff_type_content += f""" +
    +
    +
    +

    + {k}: {num_files_this_type} File{dtt} +

    +
    +
    +
    +
      +{this_diff_type_list} +
    +
    +
    +
    +
    """ + + # set up runtime results table + run_time_rows_text = "" + sum_base_seconds = 0 + sum_branch_seconds = 0 + for idf, summary in self.summary_results.items(): + case_1_success = summary.simulation_status_case1 == EndErrSummary.STATUS_SUCCESS + case_2_success = summary.simulation_status_case2 == EndErrSummary.STATUS_SUCCESS + if case_1_success: + base_time = summary.run_time_seconds_case1 + else: + base_time = "N/A" + if case_1_success: + branch_time = summary.run_time_seconds_case2 + else: + branch_time = "N/A" + if case_1_success and case_2_success: + sum_base_seconds += base_time + sum_branch_seconds += branch_time + + run_time_rows_text += f"""{idf}{base_time}{branch_time}""" + run_time_rows_text += f"""Runtime Total (Successes){sum_base_seconds:.1f}{sum_branch_seconds:.1f}""" + return f""" @@ -238,58 +306,115 @@ def bundle_root_index_html(header_info: list[str], no_diffs: list[str], diffs: l -
    -
    -
    -
    -

    - Header Metadata -

    -
    -