From 02d9d2e647d227bc6938318ddfe779bd9180aab7 Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 08:00:45 -0700 Subject: [PATCH 1/8] use variable subst in case OPT_PREFIX key not set --- eol_scons/tools/inilib.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/eol_scons/tools/inilib.py b/eol_scons/tools/inilib.py index 320cab0..df5957d 100644 --- a/eol_scons/tools/inilib.py +++ b/eol_scons/tools/inilib.py @@ -2,15 +2,12 @@ # # This source code is licensed under the MIT license found in the LICENSE # file in the root directory of this source tree. -import os def generate(env): - env.AppendUnique(CPPPATH=[os.path.join(env['OPT_PREFIX'],'include'),]) - env.AppendUnique(LIBPATH=[os.path.join(env['OPT_PREFIX'],'lib')]) - env.Append(LIBS=['ini',]) - + env.AppendUnique(CPPPATH=['$OPT_PREFIX/include']) + env.AppendUnique(LIBPATH=['$OPT_PREFIX/lib']) + env.Append(LIBS=['ini',]) def exists(env): return True - From e857a1b0a56ff2f92f8af8828f52dc9b0db28243 Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 08:01:57 -0700 Subject: [PATCH 2/8] ensure QT_VERSION is an int 4, 5, or 6 --- eol_scons/tool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eol_scons/tool.py b/eol_scons/tool.py index ede71d4..ba6094c 100644 --- a/eol_scons/tool.py +++ b/eol_scons/tool.py @@ -406,6 +406,10 @@ def qtmtool(env): env.Require('qt5') elif qtversion == 4: env.Require('qt4') + else: + raise SCons.Errors.StopError( + "QT_VERSION (%s) must be integer 4, 5, or 6" % + (repr(qtversion))) env.EnableQtModules(modules) kw = {} kw[module.lower()] = qtmtool From 1c86edfff43b63348bb95346a213cc319a10855a Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 08:03:02 -0700 Subject: [PATCH 3/8] parameterize qt_version for tool loading test it is now possible to run the SConstruct load_all_tools tests against either qt5 or qt6, but only qt5 is included for automated tests at the moment. --- tests/SConstruct | 24 ++++++++++++++++-------- tests/runtests | 1 + 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 91398e8..9e4bea5 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -10,7 +10,7 @@ import SCons import glob from pathlib import Path -from SCons.Script import Environment, SConscript +from SCons.Script import Environment, SConscript, ARGUMENTS tooldir = eol_scons.tools_dir @@ -36,7 +36,9 @@ ignore_tools = ['gcc', 'msvc', 'windriver', 'netcdf', 'netcdfcxx', - 'netcdfcxx4' + 'netcdfcxx4', + # No point testing this tool since it is defunct. + 'qt4' ] for it in ignore_tools: toolnames.remove(it) @@ -132,6 +134,8 @@ def test_5_xercesc(): def test_7_load_all_tools(): + qt_version = ARGUMENTS.get('qt_version', None) + qt_version = int(qt_version) if qt_version else 5 # Test each tool individually, so we can catch SCons.Errors.StopError # from the tools which abort if their component is not found. tut = dict([(name, None) for name in toolnames]) @@ -151,16 +155,20 @@ def test_7_load_all_tools(): tut['netcdfcxx'] = stop_or_none tut['cppunit'] = stop_or_none - # No point testing these directly since they conflict. Tools which - # do require qt modules will be tested with QT_VERSION=5. - del tut['qt4'] - # del tut['qt5'] + # Tools which require a specific QT_VERSION should fail if the other + # version is set. + if (qt_version == 5): + tut['qt6'] = stop_or_none + elif (qt_version == 6): + tut['qt5'] = stop_or_none + for toolname, error in tut.items(): result = None try: print("Loading tool %s..." % (toolname)) - x = Environment(tools=['default'] + [toolname], QT_VERSION=5) - assert x['QT_VERSION'] == 5 + x = Environment(tools=['default'] + [toolname], + QT_VERSION=qt_version) + assert x['QT_VERSION'] == qt_version except Exception as ex: result = ex # If no error expected but got one, raise it again to get a diff --git a/tests/runtests b/tests/runtests index e9e956f..ecf0fa6 100755 --- a/tests/runtests +++ b/tests/runtests @@ -33,6 +33,7 @@ test_sconstruct() # sfile options } test_sconstruct SConstruct +# test_sconstruct SConstruct qt_version=6 pytest=pytest which pytest-3 1> /dev/null 2>&1 && pytest=pytest-3 From 153429721d311b50446dd00db0f626c3e68ab154 Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 11:56:12 -0700 Subject: [PATCH 4/8] Create codeql.yml Remove language c-cpp from the default codeql.yml since the sole C file is helloworld.c used only for testing the python. --- .github/workflows/codeql.yml | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..265206c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,92 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: '39 3 * * 2' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From d32d6363c5dd42025f3d5b3cd583be13ed0c6a96 Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 13:51:12 -0700 Subject: [PATCH 5/8] add lib to load path even if lib64 exists in case it might help transition from lib64 to lib. remove obsolete function. --- eol_scons/tools/nidas.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/eol_scons/tools/nidas.py b/eol_scons/tools/nidas.py index e798a0c..2b4ad3f 100644 --- a/eol_scons/tools/nidas.py +++ b/eol_scons/tools/nidas.py @@ -204,28 +204,6 @@ def _NidasRuntimeENV(env): env.PrependENVPath('LD_LIBRARY_PATH', '/opt/nc_server/lib') -def _NidasAppFindFile(env, name): - # Look for a program with the given name in either the build dir for - # the active arch in the source tree, or else in the installed path. - vdir = '#/build/build' - if 'ARCH' in env and env['ARCH'] not in ['host', 'x86', '']: - arch = env['ARCH'] # empty string for native builds - vdir = vdir + '_' + arch - vdir = env.Dir(vdir) - eol_scons.Debug("Looking up app %s under %s..." % (name, vdir)) - nodes = env.arg2nodes([vdir], env.fs.Dir) - app = SCons.Node.FS.find_file(name, tuple(nodes), verbose=True) - # app = env.FindFile(name, [vdir]) - if not app: - # Default to install bin using the prefix, which already contains - # the arch distinction. - vdir = env.Dir(env['PREFIX']) - eol_scons.Debug("Looking up app %s under %s..." % (name, vdir)) - app = env.FindFile(name, [vdir]) - eol_scons.Debug("Found app: %s" % (str(app))) - return app - - def _check_nc_server(env, lib): lddcmd = ["ldd", lib] lddprocess = sp.Popen(lddcmd, stdout=sp.PIPE, env=env['ENV']) @@ -241,7 +219,7 @@ def _resolve_libpaths(env, paths): plib = os.path.join(p, 'lib') if os.path.exists(parch): libpaths.append(parch) - elif os.path.exists(plib): + if os.path.exists(plib): libpaths.append(plib) return libpaths From 21a0668114aefdcdd0a06cb1a9e3f8631bbc4b0f Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 13:52:46 -0700 Subject: [PATCH 6/8] filter password from print output I don't think the password has actually been used anywhere, but filter it just in case and to satisfy security scans. --- eol_scons/postgres/testdb.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/eol_scons/postgres/testdb.py b/eol_scons/postgres/testdb.py index f2a8098..1d6c79e 100644 --- a/eol_scons/postgres/testdb.py +++ b/eol_scons/postgres/testdb.py @@ -127,6 +127,8 @@ def __init__(self, cwd=None, personality="postgrestestdb"): self.settingsfile = None self.setupTempConnection() self.debug = False + # cache the password so it can be removed from log output + self.password = None def _log(self, msg): if self.debug: @@ -238,8 +240,14 @@ def stop(self): """ self._run(["pg_ctl", "-m", "fast", "-w", "stop"]) - def _popen(self, cmd, env=None, **args): + def _sanitized_cmd(self, cmd: list) -> str: scmd = " ".join(cmd) + if self.password: + scmd = scmd.replace(self.password, "*****") + return scmd + + def _popen(self, cmd, env=None, **args): + scmd = self._sanitized_cmd(cmd) print("Running: %s" % (scmd)) if not env: env = self.getEnvironment() @@ -251,7 +259,7 @@ def _popen(self, cmd, env=None, **args): raise def _run(self, cmd, env=None): - scmd = " ".join(cmd) + scmd = self._sanitized_cmd(cmd) p = self._popen(cmd, env=env) retcode = p.wait() if retcode: @@ -320,6 +328,7 @@ def createUser(self, user, password=None): self.PGUSER = None pwd = "" if password: + self.password = password pwd = "PASSWORD '%s'" % (password) self._psql("template1", "CREATE USER \"%s\" " "WITH %s CREATEDB;" % (user, pwd)) From fa62415d1b783b879e70b372138ee6ebd889a484 Mon Sep 17 00:00:00 2001 From: Gary Granger Date: Mon, 30 Dec 2024 14:41:41 -0700 Subject: [PATCH 7/8] test password sanitization --- eol_scons/postgres/testdb.py | 7 +++++++ tests/test_testdb.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/test_testdb.py diff --git a/eol_scons/postgres/testdb.py b/eol_scons/postgres/testdb.py index 1d6c79e..d0b98e3 100644 --- a/eol_scons/postgres/testdb.py +++ b/eol_scons/postgres/testdb.py @@ -129,6 +129,9 @@ def __init__(self, cwd=None, personality="postgrestestdb"): self.debug = False # cache the password so it can be removed from log output self.password = None + # cache and echo the command but do not run it + self.dryrun = False + self.last_command = None def _log(self, msg): if self.debug: @@ -260,6 +263,10 @@ def _popen(self, cmd, env=None, **args): def _run(self, cmd, env=None): scmd = self._sanitized_cmd(cmd) + if self.dryrun: + self.last_command = cmd + print("Dry run: %s" % (scmd)) + return p = self._popen(cmd, env=env) retcode = p.wait() if retcode: diff --git a/tests/test_testdb.py b/tests/test_testdb.py new file mode 100644 index 0000000..ac59cc1 --- /dev/null +++ b/tests/test_testdb.py @@ -0,0 +1,17 @@ + +import eol_scons.postgres.testdb as testdb + + +def test_testdb_sanitize(): + tdb = testdb.PostgresTestDB() + tdb.dryrun = True + tdb.createUser('ads', 'password') + assert tdb.PGUSER == 'ads' + xcmd = ["psql", "template1", "-c", + 'CREATE USER "ads" WITH PASSWORD \'password\' CREATEDB;' + ] + assert tdb.last_command == xcmd + assert tdb.password == 'password' + xs = ("psql template1 -c CREATE USER \"ads\" " + "WITH PASSWORD '*****' CREATEDB;") + assert tdb._sanitized_cmd(tdb.last_command) == xs From fb7fc76a15fad8ae56c853ee0e0e351fbfddcaa8 Mon Sep 17 00:00:00 2001 From: Chris Webster Date: Mon, 30 Dec 2024 16:38:54 -0700 Subject: [PATCH 8/8] It appears homebrew dropped the -mt from the boost lib names. Boost 1.87 was released a couple weeks ago. Aeros & RIC stopped linking as they could not find boost libraries with -mt extensions. According to boost documentation, this -mt was more a formality. The msys/ucrt64 environmnt still has the -mt extensions. --- eol_scons/tools/boost.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eol_scons/tools/boost.py b/eol_scons/tools/boost.py index cfa3d82..6640ddc 100644 --- a/eol_scons/tools/boost.py +++ b/eol_scons/tools/boost.py @@ -30,10 +30,10 @@ def boost_libflags(env): def _append_boost_library(env, libname): - if env['PLATFORM'] != 'darwin' and env['PLATFORM'] != 'msys': - env.Append(LIBS=[libname]) - else: + if env['PLATFORM'] == 'msys': env.Append(LIBS=[libname + "-mt"]) + else: + env.Append(LIBS=[libname]) def boost_version(env):