From 751b97364d8617bf419880edba810b6275ff45da Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Tue, 14 Jul 2020 23:25:14 -0600 Subject: [PATCH 01/36] Tweaking pipeline to be multi-stage --- Makefile | 4 +- azure-deploy.yml | 27 +++++++++ azure-pipelines.yml | 138 ++++++++++++++++++++++++++++++-------------- 3 files changed, 126 insertions(+), 43 deletions(-) create mode 100644 azure-deploy.yml diff --git a/Makefile b/Makefile index bac9f8c9..ed3a9a1c 100644 --- a/Makefile +++ b/Makefile @@ -39,13 +39,15 @@ macos-ci: python3 -u tools/ci.py \ -B download \ -T tools/gcc-9.jsonc \ - -T2 tools/gcc-9.next.jsonc \ + -T2 tools/gcc-9.next.jsonc + mv _build/dds _build/dds-macos-x64 linux-ci: python3 -u tools/ci.py \ -B download \ -T tools/gcc-9.jsonc \ -T2 tools/gcc-9-static.jsonc + mv _build/dds _build/dds-linux-x64 nix-ci: python3 -u tools/ci.py \ diff --git a/azure-deploy.yml b/azure-deploy.yml new file mode 100644 index 00000000..14fbde90 --- /dev/null +++ b/azure-deploy.yml @@ -0,0 +1,27 @@ +parameters: + - name: branch + type: string + displayName: Branch + - name: targetPath + type: string + displayName: Target Path + +jobs: + - ${{ if eq(variables.build.SourceBranchName, parameters.branch) }}: + - job: deploy_${{ parameters.branch }} + displayName: Deploy for ${{parameters.branch}} + condition: succeeded() + steps: + - task: DownloadPipelineArtifact@2 + displayName: Download builds + inputs: + targetPath: art/ + - task: CopyFilesOverSSH@0 + displayName: Post builds + inputs: + sshEndpoint: dds.pizza + sourceFolder: art/ + targetFolder: ${{ parameters.targetPath }} + failOnEmptySource: true + overwrite: true + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a64d2fd3..6ffd6a7e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,47 +1,101 @@ # Refer: https://aka.ms/yaml -jobs: +variables: + shouldDeploy: > + ${{ or( + eq(variables.Build.SourceBranch, 'refs/heads/develop'), + eq(variables.Build.SourceBranch, 'refs/heads/master') + ) }} - - job: Windows_MSVC_VS2019 - pool: - vmImage: windows-2019 - steps: - - script: | - echo Loading VS environment - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1 - echo Executing Build and Tests - reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 - python -m pip install pytest pytest-xdist || exit 1 - python -u tools/ci.py -B download -T tools\msvc.jsonc -T2 tools\msvc.next.jsonc || exit 1 - displayName: Full CI - - publish: _build/dds.exe - artifact: DDS Executable - Windows VS2019 +stages: + - stage: build_test + displayName: Build and Test + jobs: + - job: win_vs2019 + displayName: Windows - VS 2019 + pool: + vmImage: windows-2019 + steps: + - script: | + echo Loading VS environment + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1 + echo Executing Build and Tests + reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 + python -m pip install pytest pytest-xdist || exit 1 + python -u tools/ci.py -B download -T tools\msvc.jsonc -T2 tools\msvc.next.jsonc || exit 1 + move _build\dds.exe _build\dds-win-x64.exe || exit 1 + displayName: Build and Test + - publish: _build\dds-win-x64.exe + displayName: Publish + artifact: dds-win-x64 - - job: Linux_GCC9 - pool: - vmImage: ubuntu-18.04 - steps: - - script: | - set -eu - sudo apt update -y - sudo apt install -y python3-minimal g++-9 ccache - python3 -m pip install pytest pytest-xdist - displayName: Prepare System - - script: make linux-ci - displayName: Full CI - - publish: _build/dds - artifact: DDS Executable - Linux + - job: linux_gcc9 + displayName: Linux - GCC 9 + pool: + vmImage: ubuntu-18.04 + steps: + - script: | + set -eu + sudo apt update -y + sudo apt install -y python3-minimal g++-9 ccache + python3 -m pip install pytest pytest-xdist + displayName: Prepare System + - script: make linux-ci + displayName: Build and Test + - publish: _build/dds-linux-x64 + displayName: Publish + artifact: dds-linux-x64 - - job: macOS_GCC9 - pool: - vmImage: macOS-10.14 - steps: - - script: brew install gcc@9 ccache - displayName: Prepare System - - script: | - set -eu - python3 -m pip install pytest pytest-xdist - make macos-ci - displayName: Build and Run Unit Tests - - publish: _build/dds - artifact: DDS Executable - macOS + - job: macos_gcc9 + displayName: macOS - GCC 9 + pool: + vmImage: macOS-10.14 + steps: + - script: brew install gcc@9 ccache + displayName: Prepare System + - script: | + set -eu + python3 -m pip install pytest pytest-xdist + make macos-ci + displayName: Build and Test + - publish: _build/dds-macos-x64 + displayName: Publish + artifact: dds-macos-x64 + + - stage: deploy_build + displayName: Deploy + condition: eq(variables.shouldDeploy, true) + jobs: + - job: deploy + displayName: Deploy for ${{variables.Build.SourceBranchName}} + condition: succeeded() + steps: + - task: DownloadPipelineArtifact@2 + displayName: Download builds + inputs: + targetPath: art/ + - task: CopyFilesOverSSH@0 + displayName: Post builds + inputs: + sshEndpoint: dds.pizza + sourceFolder: art/ + targetFolder: ~/${{variables.Build.SourceBranchName}}/ + failOnEmptySource: true + overwrite: true + # - job: deploy + # displayName: Deploy Canary + # # Only deploy the "canary" build when we are on the 'develop' branch + # condition: and(succeeded(), eq(variables.Build.SourceBranchName, 'develop')) + # steps: + # - task: DownloadPipelineArtifact@2 + # displayName: Download builds + # inputs: + # targetPath: art/ + # - task: CopyFilesOverSSH@0 + # displayName: Post builds + # inputs: + # sshEndpoint: dds.pizza + # sourceFolder: art/ + # targetFolder: ~/canary/ + # failOnEmptySource: true + # overwrite: true From f2739101aafde53f92e2da2409cd58c8bd06176c Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Wed, 15 Jul 2020 23:56:11 -0600 Subject: [PATCH 02/36] Update to use alpha.4 to build ourself --- Makefile | 6 ++---- azure-pipelines.yml | 2 +- tools/bootstrap.py | 1 + tools/ci.py | 15 ++++----------- tools/freebsd-gcc-9.jsonc | 5 ++--- tools/freebsd-gcc-9.next.jsonc | 20 -------------------- tools/gcc-9.jsonc | 8 ++------ tools/gcc-9.next.jsonc | 20 -------------------- tools/msvc.jsonc | 3 +-- tools/msvc.next.jsonc | 13 ------------- 10 files changed, 13 insertions(+), 80 deletions(-) delete mode 100644 tools/freebsd-gcc-9.next.jsonc delete mode 100644 tools/gcc-9.next.jsonc delete mode 100644 tools/msvc.next.jsonc diff --git a/Makefile b/Makefile index ed3a9a1c..d92e437b 100644 --- a/Makefile +++ b/Makefile @@ -38,15 +38,13 @@ docs-sync-server: macos-ci: python3 -u tools/ci.py \ -B download \ - -T tools/gcc-9.jsonc \ - -T2 tools/gcc-9.next.jsonc + -T tools/gcc-9.jsonc mv _build/dds _build/dds-macos-x64 linux-ci: python3 -u tools/ci.py \ -B download \ - -T tools/gcc-9.jsonc \ - -T2 tools/gcc-9-static.jsonc + -T tools/gcc-9.jsonc mv _build/dds _build/dds-linux-x64 nix-ci: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ffd6a7e..e499013e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ stages: echo Executing Build and Tests reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 python -m pip install pytest pytest-xdist || exit 1 - python -u tools/ci.py -B download -T tools\msvc.jsonc -T2 tools\msvc.next.jsonc || exit 1 + python -u tools/ci.py -B download -T tools\msvc.jsonc || exit 1 move _build\dds.exe _build\dds-win-x64.exe || exit 1 displayName: Build and Test - publish: _build\dds-win-x64.exe diff --git a/tools/bootstrap.py b/tools/bootstrap.py index 925558c6..0763977a 100644 --- a/tools/bootstrap.py +++ b/tools/bootstrap.py @@ -25,6 +25,7 @@ def platform_compiler(self): BootstrapPhase('bootstrap-p4.2', 'g++-8', 'cl.exe'), BootstrapPhase('bootstrap-p5.2', 'g++-9', 'cl.exe'), BootstrapPhase('0.1.0-alpha.3', 'g++-9', 'cl.exe'), + BootstrapPhase('0.1.0-alpha.4', 'g++-9', 'cl.exe'), ] HERE = Path(__file__).parent.absolute() diff --git a/tools/ci.py b/tools/ci.py index ff1511c6..0c2c9518 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -15,7 +15,6 @@ class CIOptions(NamedTuple): toolchain: str - toolchain_2: str def _do_bootstrap_build(opts: CIOptions) -> None: @@ -38,7 +37,7 @@ def _do_bootstrap_download() -> None: if filename is None: raise RuntimeError(f'We do not have a prebuilt DDS binary for ' f'the "{sys.platform}" platform') - url = f'https://github.com/vector-of-bool/dds/releases/download/0.1.0-alpha.3/{filename}' + url = f'https://github.com/vector-of-bool/dds/releases/download/0.1.0-alpha.4/{filename}' print(f'Downloading prebuilt DDS executable: {url}') stream = urllib.request.urlopen(url) @@ -72,19 +71,13 @@ def main(argv: Sequence[str]) -> int: help='The toolchain to use for the CI process', required=True, ) - parser.add_argument( - '--toolchain-2', - '-T2', - help='The toolchain to use for the self-build', - required=True, - ) parser.add_argument( '--build-only', action='store_true', help='Only build the `dds` executable. Skip second-phase and tests.') args = parser.parse_args(argv) - opts = CIOptions(toolchain=args.toolchain, toolchain_2=args.toolchain_2) + opts = CIOptions(toolchain=args.toolchain) if args.bootstrap_with == 'build': _do_bootstrap_build(opts) @@ -107,7 +100,7 @@ def main(argv: Sequence[str]) -> int: paths.PREBUILT_DDS, toolchain=opts.toolchain, cat_path=old_cat_path, - cat_json_path=Path('catalog.old.json'), + cat_json_path=Path('catalog.json'), dds_flags=[('--repo-dir', ci_repo_dir)]) print('Main build PASSED!') print(f'A `dds` executable has been generated: {paths.CUR_BUILT_DDS}') @@ -123,7 +116,7 @@ def main(argv: Sequence[str]) -> int: new_repo_dir = paths.BUILD_DIR / 'ci-repo' self_build( paths.CUR_BUILT_DDS, - toolchain=opts.toolchain_2, + toolchain=opts.toolchain, cat_path=new_cat_path, dds_flags=[f'--repo-dir={new_repo_dir}']) print('Bootstrap test PASSED!') diff --git a/tools/freebsd-gcc-9.jsonc b/tools/freebsd-gcc-9.jsonc index 58cfc753..783f502c 100644 --- a/tools/freebsd-gcc-9.jsonc +++ b/tools/freebsd-gcc-9.jsonc @@ -3,9 +3,8 @@ "compiler_id": "gnu", "c_compiler": "gcc9", "cxx_compiler": "g++9", - "flags": [ - "-DSPDLOG_COMPILED_LIB", // Required to use a compiled spdlog - "-Werror=return-type", + "warning_flags": [ + "-Werror", ], "cxx_flags": [ "-fconcepts", diff --git a/tools/freebsd-gcc-9.next.jsonc b/tools/freebsd-gcc-9.next.jsonc deleted file mode 100644 index 783f502c..00000000 --- a/tools/freebsd-gcc-9.next.jsonc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "../res/toolchain-schema.json", - "compiler_id": "gnu", - "c_compiler": "gcc9", - "cxx_compiler": "g++9", - "warning_flags": [ - "-Werror", - ], - "cxx_flags": [ - "-fconcepts", - "-std=c++2a", - ], - "link_flags": [ - "-static-libgcc", - "-static-libstdc++", - ], - // "debug": true, - "optimize": true, - "compiler_launcher": "ccache" -} \ No newline at end of file diff --git a/tools/gcc-9.jsonc b/tools/gcc-9.jsonc index dbfd6201..d5aaf373 100644 --- a/tools/gcc-9.jsonc +++ b/tools/gcc-9.jsonc @@ -3,10 +3,8 @@ "compiler_id": "gnu", "c_compiler": "gcc-9", "cxx_compiler": "g++-9", - "flags": [ - "-DSPDLOG_COMPILED_LIB", // Required to use a compiled spdlog - "-Werror=return-type", - // "-fsanitize=address", + "warning_flags": [ + "-Werror", ], "cxx_flags": [ "-fconcepts", @@ -15,8 +13,6 @@ "link_flags": [ "-static-libgcc", "-static-libstdc++" - // "-fsanitize=address", - // "-fuse-ld=lld", ], // "debug": true, "optimize": true, diff --git a/tools/gcc-9.next.jsonc b/tools/gcc-9.next.jsonc deleted file mode 100644 index d5aaf373..00000000 --- a/tools/gcc-9.next.jsonc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "../res/toolchain-schema.json", - "compiler_id": "gnu", - "c_compiler": "gcc-9", - "cxx_compiler": "g++-9", - "warning_flags": [ - "-Werror", - ], - "cxx_flags": [ - "-fconcepts", - "-std=c++2a", - ], - "link_flags": [ - "-static-libgcc", - "-static-libstdc++" - ], - // "debug": true, - "optimize": true, - "compiler_launcher": "ccache" -} \ No newline at end of file diff --git a/tools/msvc.jsonc b/tools/msvc.jsonc index 7db2877a..b452ee21 100644 --- a/tools/msvc.jsonc +++ b/tools/msvc.jsonc @@ -2,8 +2,7 @@ "$schema": "../res/toolchain-schema.json", "compiler_id": "msvc", "flags": [ - "/Zc:preprocessor", // Required for range-v3 - "/DSPDLOG_COMPILED_LIB", // Required to use spdlog as a compiled lib + "/Zc:preprocessor", "/std:c++latest", ], "link_flags": [ diff --git a/tools/msvc.next.jsonc b/tools/msvc.next.jsonc deleted file mode 100644 index b452ee21..00000000 --- a/tools/msvc.next.jsonc +++ /dev/null @@ -1,13 +0,0 @@ -{ - "$schema": "../res/toolchain-schema.json", - "compiler_id": "msvc", - "flags": [ - "/Zc:preprocessor", - "/std:c++latest", - ], - "link_flags": [ - "rpcrt4.lib", - ], - // "debug": true, - "optimize": true -} \ No newline at end of file From e4ea3ac3364795329f2b7a5cb0b28f4e587c7c18 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 12:36:35 -0600 Subject: [PATCH 03/36] Cleanup and path fixes --- azure-deploy.yml | 27 --------------------------- azure-pipelines.yml | 32 ++++++++------------------------ 2 files changed, 8 insertions(+), 51 deletions(-) delete mode 100644 azure-deploy.yml diff --git a/azure-deploy.yml b/azure-deploy.yml deleted file mode 100644 index 14fbde90..00000000 --- a/azure-deploy.yml +++ /dev/null @@ -1,27 +0,0 @@ -parameters: - - name: branch - type: string - displayName: Branch - - name: targetPath - type: string - displayName: Target Path - -jobs: - - ${{ if eq(variables.build.SourceBranchName, parameters.branch) }}: - - job: deploy_${{ parameters.branch }} - displayName: Deploy for ${{parameters.branch}} - condition: succeeded() - steps: - - task: DownloadPipelineArtifact@2 - displayName: Download builds - inputs: - targetPath: art/ - - task: CopyFilesOverSSH@0 - displayName: Post builds - inputs: - sshEndpoint: dds.pizza - sourceFolder: art/ - targetFolder: ${{ parameters.targetPath }} - failOnEmptySource: true - overwrite: true - diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e499013e..1a67e40b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,11 +1,12 @@ # Refer: https://aka.ms/yaml variables: - shouldDeploy: > + shouldDeploy: >- ${{ or( - eq(variables.Build.SourceBranch, 'refs/heads/develop'), - eq(variables.Build.SourceBranch, 'refs/heads/master') + eq(variables['Build.SourceBranch'], 'refs/heads/develop'), + eq(variables['Build.SourceBranch'], 'refs/heads/master') ) }} + deployDest: ${{ format('~/{0}/', variables['Build.SourceBranchName']) }} stages: - stage: build_test @@ -64,12 +65,12 @@ stages: - stage: deploy_build displayName: Deploy - condition: eq(variables.shouldDeploy, true) + condition: and(succeeded(), eq(variables.shouldDeploy, 'true')) jobs: - job: deploy - displayName: Deploy for ${{variables.Build.SourceBranchName}} - condition: succeeded() + displayName: Deploy (${{variables['Build.SourceBranch']}}) steps: + - checkout: none - task: DownloadPipelineArtifact@2 displayName: Download builds inputs: @@ -79,23 +80,6 @@ stages: inputs: sshEndpoint: dds.pizza sourceFolder: art/ - targetFolder: ~/${{variables.Build.SourceBranchName}}/ + targetFolder: ${{ variables.deployDest }} failOnEmptySource: true overwrite: true - # - job: deploy - # displayName: Deploy Canary - # # Only deploy the "canary" build when we are on the 'develop' branch - # condition: and(succeeded(), eq(variables.Build.SourceBranchName, 'develop')) - # steps: - # - task: DownloadPipelineArtifact@2 - # displayName: Download builds - # inputs: - # targetPath: art/ - # - task: CopyFilesOverSSH@0 - # displayName: Post builds - # inputs: - # sshEndpoint: dds.pizza - # sourceFolder: art/ - # targetFolder: ~/canary/ - # failOnEmptySource: true - # overwrite: true From 7ff8a3ebb148f41864ab5551e01adfa6b8f612c8 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 15:01:47 -0600 Subject: [PATCH 04/36] Tweak deployment paths --- azure-pipelines.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1a67e40b..cf988950 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,7 +6,7 @@ variables: eq(variables['Build.SourceBranch'], 'refs/heads/develop'), eq(variables['Build.SourceBranch'], 'refs/heads/master') ) }} - deployDest: ${{ format('~/{0}/', variables['Build.SourceBranchName']) }} + deployDest: ${{ format('~/web/{0}/', variables['Build.SourceBranchName']) }} stages: - stage: build_test @@ -74,7 +74,12 @@ stages: - task: DownloadPipelineArtifact@2 displayName: Download builds inputs: - targetPath: art/ + targetPath: art-dirs/ + - bash: | + set -eu + mkdir -p art/ + mv -- $(find art-dirs/ -type f) art/ + displayName: Rearrange - task: CopyFilesOverSSH@0 displayName: Post builds inputs: From fae2776f66fc3a66473e57c26a153e223d355709 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 15:47:53 -0600 Subject: [PATCH 05/36] spdlog is slow to compile. Wrap in a shim --- package.jsonc | 3 +- src/dds.main.cpp | 29 ++++++------ src/dds/build/builder.cpp | 21 +++++---- src/dds/build/file_deps.cpp | 6 +-- src/dds/build/plan/archive.cpp | 12 +++-- src/dds/build/plan/compile_exec.cpp | 38 ++++++++-------- src/dds/build/plan/compile_file.cpp | 2 - src/dds/build/plan/exe.cpp | 13 +++--- src/dds/build/plan/full.cpp | 2 - src/dds/catalog/catalog.cpp | 9 ++-- src/dds/catalog/get.cpp | 10 ++--- src/dds/db/database.cpp | 6 +-- src/dds/library/manifest.cpp | 1 - src/dds/library/root.cpp | 5 +-- src/dds/package/manifest.cpp | 4 +- src/dds/proc.nix.cpp | 7 ++- src/dds/proc.win.cpp | 1 - src/dds/repo/repo.cpp | 21 +++++---- src/dds/solve/solve.cpp | 10 ++--- src/dds/source/dist.cpp | 11 +++-- src/dds/util/log.cpp | 33 ++++++++++++++ src/dds/util/log.hpp | 68 +++++++++++++++++++++++++++++ src/dds/util/parallel.cpp | 4 +- src/dds/util/paths.linux_fbsd.cpp | 4 +- src/dds/util/paths.macos.cpp | 4 +- src/dds/util/paths.win.cpp | 4 +- src/libman/library.cpp | 2 +- 27 files changed, 209 insertions(+), 121 deletions(-) create mode 100644 src/dds/util/log.cpp create mode 100644 src/dds/util/log.hpp diff --git a/package.jsonc b/package.jsonc index 7b8096c3..0a1a22a9 100644 --- a/package.jsonc +++ b/package.jsonc @@ -4,7 +4,7 @@ "version": "0.1.0-alpha.4", "namespace": "dds", "depends": { - "spdlog": "1.4.2", + "spdlog": "1.7.0", "ms-wil": "2020.3.16", "range-v3": "0.10.0", "nlohmann-json": "3.7.1", @@ -15,6 +15,7 @@ "vob-json5": "0.1.5", "vob-semester": "0.2.1", "ctre": "2.8.1", + "fmt": "^7.0.0" }, "test_driver": "Catch-Main" } \ No newline at end of file diff --git a/src/dds.main.cpp b/src/dds.main.cpp index e54297ff..7dc5748f 100644 --- a/src/dds.main.cpp +++ b/src/dds.main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -14,9 +15,9 @@ #include #include #include +#include #include -#include #include #include @@ -226,7 +227,7 @@ struct cli_catalog { auto tsd = dds::get_package_sdist(*info); auto out_path = out.Get(); auto dest = out_path / id.to_string(); - spdlog::info("Create sdist at {}", dest.string()); + dds::log::info("Create sdist at {}", dest.string()); dds::fs::remove_all(dest); dds::safe_rename(tsd.sdist.path, dest); } @@ -341,7 +342,7 @@ struct cli_catalog { auto cat = cat_path.open(); auto pkg = cat.get(pk_id); if (!pkg) { - spdlog::error("No package '{}' in the catalog", pk_id.to_string()); + dds::log::error("No package '{}' in the catalog", pk_id.to_string()); return 1; } std::cout << "Name: " << pkg->ident.name << '\n' @@ -418,9 +419,9 @@ struct cli_repo { }); for (const auto& [name, grp] : grp_by_name) { - spdlog::info("{}:", name); + dds::log::info("{}:", name); for (const dds::sdist& sd : grp) { - spdlog::info(" - {}", sd.manifest.pkg_id.version.to_string()); + dds::log::info(" - {}", sd.manifest.pkg_id.version.to_string()); } } @@ -691,7 +692,7 @@ struct cli_build_deps { auto all_file_deps = deps_files.Get() // | ranges::views::transform([&](auto dep_fpath) { - spdlog::info("Reading deps from {}", dep_fpath.string()); + dds::log::info("Reading deps from {}", dep_fpath.string()); return dds::dependency_manifest::from_file(dep_fpath).dependencies; }) | ranges::actions::join; @@ -708,7 +709,7 @@ struct cli_build_deps { dds::repo_flags::write_lock | dds::repo_flags::create_if_absent, [&](dds::repository repo) { // Download dependencies - spdlog::info("Loading {} dependencies", all_deps.size()); + dds::log::info("Loading {} dependencies", all_deps.size()); auto deps = repo.solve(all_deps, cat); dds::get_all(deps, repo, cat); for (const dds::package_id& pk : deps) { @@ -716,7 +717,7 @@ struct cli_build_deps { assert(sdist_ptr); dds::sdist_build_params deps_params; deps_params.subdir = sdist_ptr->manifest.pkg_id.to_string(); - spdlog::info("Dependency: {}", sdist_ptr->manifest.pkg_id.to_string()); + dds::log::info("Dependency: {}", sdist_ptr->manifest.pkg_id.to_string()); bd.add(*sdist_ptr, deps_params); } }); @@ -740,7 +741,7 @@ struct cli_build_deps { int main(int argc, char** argv) { #if DDS_DEBUG - spdlog::set_level(spdlog::level::debug); + dds::log::current_log_level = dds::log::level::debug; #endif spdlog::set_pattern("[%H:%M:%S] [%^%-5l%$] %v"); args::ArgumentParser parser("DDS - The drop-dead-simple library manager"); @@ -783,15 +784,15 @@ int main(int argc, char** argv) { std::terminate(); } } catch (const dds::user_cancelled&) { - spdlog::critical("Operation cancelled by user"); + dds::log::critical("Operation cancelled by user"); return 2; } catch (const dds::error_base& e) { - spdlog::error("{}", e.what()); - spdlog::error("{}", e.explanation()); - spdlog::error("Refer: {}", e.error_reference()); + dds::log::error("{}", e.what()); + dds::log::error("{}", e.explanation()); + dds::log::error("Refer: {}", e.error_reference()); return 1; } catch (const std::exception& e) { - spdlog::critical(e.what()); + dds::log::critical(e.what()); return 2; } } diff --git a/src/dds/build/builder.cpp b/src/dds/build/builder.cpp index 509e1532..91cf99d2 100644 --- a/src/dds/build/builder.cpp +++ b/src/dds/build/builder.cpp @@ -6,11 +6,10 @@ #include #include #include +#include #include #include -#include - #include #include @@ -24,14 +23,14 @@ struct state { }; void log_failure(const test_failure& fail) { - spdlog::error("Test '{}' failed! [exited {}]", fail.executable_path.string(), fail.retc); + log::error("Test '{}' failed! [exited {}]", fail.executable_path.string(), fail.retc); if (fail.signal) { - spdlog::error("Test execution received signal {}", fail.signal); + log::error("Test execution received signal {}", fail.signal); } if (trim_view(fail.output).empty()) { - spdlog::error("(Test executable produced no output"); + log::error("(Test executable produced no output"); } else { - spdlog::error("Test output:\n{}[dds - test output end]", fail.output); + log::error("Test output:\n{}[dds - test output end]", fail.output); } } @@ -84,7 +83,7 @@ prepare_catch2_driver(test_lib test_driver, const build_params& params, build_en auto obj_file = plan.calc_object_file_path(env2); if (!fs::exists(obj_file)) { - spdlog::info("Compiling Catch2 test driver (This will only happen once)..."); + log::info("Compiling Catch2 test driver (This will only happen once)..."); compile_all(std::array{plan}, env2, 1); } @@ -242,19 +241,19 @@ void builder::build(const build_params& params) const { dds::stopwatch sw; plan.compile_all(env, params.parallel_jobs); - spdlog::info("Compilation completed in {:n}ms", sw.elapsed_ms().count()); + log::info("Compilation completed in {:n}ms", sw.elapsed_ms().count()); sw.reset(); plan.archive_all(env, params.parallel_jobs); - spdlog::info("Archiving completed in {:n}ms", sw.elapsed_ms().count()); + log::info("Archiving completed in {:n}ms", sw.elapsed_ms().count()); sw.reset(); plan.link_all(env, params.parallel_jobs); - spdlog::info("Runtime binary linking completed in {:n}ms", sw.elapsed_ms().count()); + log::info("Runtime binary linking completed in {:n}ms", sw.elapsed_ms().count()); sw.reset(); auto test_failures = plan.run_all_tests(env, params.parallel_jobs); - spdlog::info("Test execution finished in {:n}ms", sw.elapsed_ms().count()); + log::info("Test execution finished in {:n}ms", sw.elapsed_ms().count()); for (auto& fail : test_failures) { log_failure(fail); diff --git a/src/dds/build/file_deps.cpp b/src/dds/build/file_deps.cpp index 7f33817b..8bb2cd2f 100644 --- a/src/dds/build/file_deps.cpp +++ b/src/dds/build/file_deps.cpp @@ -2,12 +2,12 @@ #include #include +#include #include #include #include #include -#include using namespace dds; @@ -26,14 +26,14 @@ file_deps_info dds::parse_mkfile_deps_str(std::string_view str) { auto iter = split.begin(); auto stop = split.end(); if (iter == stop) { - spdlog::critical( + log::critical( "Invalid deps listing. Shell split was empty. This is almost certainly a bug."); return ret; } auto& head = *iter; ++iter; if (!ends_with(head, ":")) { - spdlog::critical( + log::critical( "Invalid deps listing. Leader item is not colon-terminated. This is probably a bug. " "(Are you trying to use C++ Modules? That's not ready yet, sorry. Set `Deps-Mode` to " "`None` in your toolchain file.)"); diff --git a/src/dds/build/plan/archive.cpp b/src/dds/build/plan/archive.cpp index 4d5af0aa..e6123917 100644 --- a/src/dds/build/plan/archive.cpp +++ b/src/dds/build/plan/archive.cpp @@ -2,10 +2,10 @@ #include #include +#include #include #include -#include using namespace dds; @@ -40,16 +40,14 @@ void create_archive_plan::archive(const build_env& env) const { fs::create_directories(ar.out_path.parent_path()); // Do it! - spdlog::info("[{}] Archive: {}", _qual_name, out_relpath); + log::info("[{}] Archive: {}", _qual_name, out_relpath); auto&& [dur_ms, ar_res] = timed([&] { return run_proc(ar_cmd); }); - spdlog::info("[{}] Archive: {} - {:n}ms", _qual_name, out_relpath, dur_ms.count()); + log::info("[{}] Archive: {} - {:n}ms", _qual_name, out_relpath, dur_ms.count()); // Check, log, and throw if (!ar_res.okay()) { - spdlog::error("Creating static library archive [{}] failed for '{}'", - out_relpath, - _qual_name); - spdlog::error("Subcommand FAILED: {}\n{}", quote_command(ar_cmd), ar_res.output); + log::error("Creating static library archive [{}] failed for '{}'", out_relpath, _qual_name); + log::error("Subcommand FAILED: {}\n{}", quote_command(ar_cmd), ar_res.output); throw_external_error< errc::archive_failure>("Creating static library archive [{}] failed for '{}'", out_relpath, diff --git a/src/dds/build/plan/compile_exec.cpp b/src/dds/build/plan/compile_exec.cpp index e1baef3b..31e295af 100644 --- a/src/dds/build/plan/compile_exec.cpp +++ b/src/dds/build/plan/compile_exec.cpp @@ -3,13 +3,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -54,16 +54,16 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun fs::relative(source_path, cf.plan.source().basis_path).string()); // Do it! - spdlog::info(msg); + log::info(msg); auto&& [dur_ms, proc_res] = timed([&] { return run_proc(cf.cmd_info.command); }); auto nth = counter.n.fetch_add(1); - spdlog::info("{:60} - {:>7n}ms [{:{}}/{}]", - msg, - dur_ms.count(), - nth, - counter.max_digits, - counter.max); + log::info("{:60} - {:>7n}ms [{:{}}/{}]", + msg, + dur_ms.count(), + nth, + counter.max_digits, + counter.max); const bool compiled_okay = proc_res.okay(); const auto compile_retc = proc_res.retc; @@ -78,7 +78,7 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun assert(cf.cmd_info.gnu_depfile_path.has_value()); auto& df_path = *cf.cmd_info.gnu_depfile_path; if (!fs::is_regular_file(df_path)) { - spdlog::critical( + log::critical( "The expected Makefile deps were not generated on disk. This is a bug! " "(Expected file to exist: [{}])", df_path.string()); @@ -121,23 +121,23 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun // Log a compiler failure if (!compiled_okay) { - spdlog::error("Compilation failed: {}", source_path.string()); - spdlog::error("Subcommand FAILED [Exitted {}]: {}\n{}", - compile_retc, - quote_command(cf.cmd_info.command), - compiler_output); + log::error("Compilation failed: {}", source_path.string()); + log::error("Subcommand FAILED [Exitted {}]: {}\n{}", + compile_retc, + quote_command(cf.cmd_info.command), + compiler_output); if (compile_signal) { - spdlog::error("Process exited via signal {}", compile_signal); + log::error("Process exited via signal {}", compile_signal); } throw_user_error("Compilation failed [{}]", source_path.string()); } // Print any compiler output, sans whitespace if (!dds::trim_view(compiler_output).empty()) { - spdlog::warn("While compiling file {} [{}]:\n{}", - source_path.string(), - quote_command(cf.cmd_info.command), - compiler_output); + log::warn("While compiling file {} [{}]:\n{}", + source_path.string(), + quote_command(cf.cmd_info.command), + compiler_output); } // We'll only get here if the compilation was successful, otherwise we throw diff --git a/src/dds/build/plan/compile_file.cpp b/src/dds/build/plan/compile_file.cpp index 39b95eea..f1f31b54 100644 --- a/src/dds/build/plan/compile_file.cpp +++ b/src/dds/build/plan/compile_file.cpp @@ -5,8 +5,6 @@ #include #include -#include - #include #include diff --git a/src/dds/build/plan/exe.cpp b/src/dds/build/plan/exe.cpp index 0c0e6c5a..0bd644d8 100644 --- a/src/dds/build/plan/exe.cpp +++ b/src/dds/build/plan/exe.cpp @@ -4,10 +4,9 @@ #include #include #include +#include #include -#include - #include #include @@ -49,10 +48,10 @@ void link_executable_plan::link(build_env_ref env, const library_plan& lib) cons auto msg = fmt::format("[{}] Link: {:30}", lib.qualified_name(), fs::relative(spec.output, env.output_root).string()); - spdlog::info(msg); + log::info(msg); auto [dur_ms, proc_res] = timed([&] { return run_proc(link_command); }); - spdlog::info("{} - {:>6n}ms", msg, dur_ms.count()); + log::info("{} - {:>6n}ms", msg, dur_ms.count()); // Check and throw if errant if (!proc_res.okay()) { @@ -77,19 +76,19 @@ bool link_executable_plan::is_test() const noexcept { std::optional link_executable_plan::run_test(build_env_ref env) const { auto exe_path = calc_executable_path(env); auto msg = fmt::format("Run test: {:30}", fs::relative(exe_path, env.output_root).string()); - spdlog::info(msg); + log::info(msg); using namespace std::chrono_literals; auto&& [dur, res] = timed( [&] { return run_proc({.command = {exe_path.string()}, .timeout = 10s}); }); if (res.okay()) { - spdlog::info("{} - PASSED - {:>9n}μs", msg, dur.count()); + log::info("{} - PASSED - {:>9n}μs", msg, dur.count()); return std::nullopt; } else { auto exit_msg = fmt::format(res.signal ? "signalled {}" : "exited {}", res.signal ? res.signal : res.retc); auto fail_str = res.timed_out ? "TIMEOUT" : "FAILED "; - spdlog::error("{} - {} - {:>9n}μs [{}]", msg, fail_str, dur.count(), exit_msg); + log::error("{} - {} - {:>9n}μs [{}]", msg, fail_str, dur.count(), exit_msg); test_failure f; f.executable_path = exe_path; f.output = res.output; diff --git a/src/dds/build/plan/full.cpp b/src/dds/build/plan/full.cpp index f000b7e6..f1eb6f21 100644 --- a/src/dds/build/plan/full.cpp +++ b/src/dds/build/plan/full.cpp @@ -12,8 +12,6 @@ #include #include -#include - #include #include diff --git a/src/dds/catalog/catalog.cpp b/src/dds/catalog/catalog.cpp index 3fb62b44..fb553872 100644 --- a/src/dds/catalog/catalog.cpp +++ b/src/dds/catalog/catalog.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -18,8 +19,6 @@ #include #include -#include - using namespace dds; namespace sqlite3 = neo::sqlite3; @@ -218,7 +217,7 @@ void ensure_migrated(sqlite3::database& db) { exec(db, "UPDATE dds_cat_meta SET meta=?", std::forward_as_tuple(meta.dump())); if (import_init_packages) { - spdlog::info( + log::info( "A new catalog database case been created, and has been populated with some initial " "contents."); neo::sqlite3::statement_cache stmts{db}; @@ -242,7 +241,7 @@ catalog catalog::open(const std::string& db_path) { try { ensure_migrated(db); } catch (const sqlite3::sqlite3_error& e) { - spdlog::critical( + log::critical( "Failed to load the repository database. It appears to be invalid/corrupted. The " "exception message is: {}", e.what()); @@ -412,6 +411,6 @@ void catalog::import_json_str(std::string_view content) { void catalog::import_initial() { sqlite3::transaction_guard tr{_db}; - spdlog::info("Restoring built-in initial catalog contents"); + log::info("Restoring built-in initial catalog contents"); store_init_packages(_db, _stmt_cache); } diff --git a/src/dds/catalog/get.cpp b/src/dds/catalog/get.cpp index 445f8d3c..ccf063d3 100644 --- a/src/dds/catalog/get.cpp +++ b/src/dds/catalog/get.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -13,7 +14,6 @@ #include #include #include -#include using namespace dds; @@ -32,16 +32,16 @@ temporary_sdist do_pull_sdist(const package_info& listing, std::monostate) { temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) { auto tmpdir = dds::temporary_dir::create(); - spdlog::info("Cloning Git repository: {} [{}] ...", git.url, git.ref); + log::info("Cloning Git repository: {} [{}] ...", git.url, git.ref); git.clone(tmpdir.path()); for (const auto& tr : git.transforms) { tr.apply_to(tmpdir.path()); } - spdlog::info("Create sdist from clone ..."); + log::info("Create sdist from clone ..."); if (git.auto_lib.has_value()) { - spdlog::info("Generating library data automatically"); + log::info("Generating library data automatically"); auto pkg_strm = dds::open(tmpdir.path() / "package.json5", std::ios::binary | std::ios::out); @@ -99,7 +99,7 @@ void dds::get_all(const std::vector& pkgs, repository& repo, const c }); auto okay = parallel_run(absent_pkg_infos, 8, [&](package_info inf) { - spdlog::info("Download package: {}", inf.ident.to_string()); + log::info("Download package: {}", inf.ident.to_string()); auto tsd = get_package_sdist(inf); std::scoped_lock lk{repo_mut}; repo.add_sdist(tsd.sdist, if_exists::throw_exc); diff --git a/src/dds/db/database.cpp b/src/dds/db/database.cpp index 09f95999..01b5aa1e 100644 --- a/src/dds/db/database.cpp +++ b/src/dds/db/database.cpp @@ -1,6 +1,7 @@ #include "./database.hpp" #include +#include #include #include @@ -10,7 +11,6 @@ #include #include #include -#include using namespace dds; @@ -87,7 +87,7 @@ database database::open(const std::string& db_path) { try { ensure_migrated(db); } catch (const sqlite3::sqlite3_error& e) { - spdlog::error( + log::error( "Failed to load the databsae. It appears to be invalid/corrupted. We'll delete it and " "create a new one. The exception message is: {}", e.what()); @@ -96,7 +96,7 @@ database database::open(const std::string& db_path) { try { ensure_migrated(db); } catch (const sqlite3::sqlite3_error& e) { - spdlog::critical( + log::critical( "Failed to apply database migrations to recovery database. This is a critical " "error. The exception message is: {}", e.what()); diff --git a/src/dds/library/manifest.cpp b/src/dds/library/manifest.cpp index 2b604c00..39f37757 100644 --- a/src/dds/library/manifest.cpp +++ b/src/dds/library/manifest.cpp @@ -7,7 +7,6 @@ #include #include #include -#include using namespace dds; diff --git a/src/dds/library/root.cpp b/src/dds/library/root.cpp index ba170425..e1f70ac5 100644 --- a/src/dds/library/root.cpp +++ b/src/dds/library/root.cpp @@ -4,10 +4,10 @@ #include #include #include +#include #include #include -#include using namespace dds; @@ -28,8 +28,7 @@ auto collect_pf_sources(path_ref path) { // Drop any source files we found within `include/` erase_if(sources, [&](auto& info) { if (info.kind != source_kind::header) { - spdlog::warn("Source file in `include` will not be compiled: {}", - info.path.string()); + log::warn("Source file in `include` will not be compiled: {}", info.path.string()); return true; } return false; diff --git a/src/dds/package/manifest.cpp b/src/dds/package/manifest.cpp index 7f1c8b11..1b144042 100644 --- a/src/dds/package/manifest.cpp +++ b/src/dds/package/manifest.cpp @@ -2,13 +2,13 @@ #include #include +#include #include #include #include #include #include -#include #include @@ -63,7 +63,7 @@ package_manifest parse_json(const json5::data& data, std::string_view fpath) { if_key{"depends", [&](auto&& dat) { if (dat.is_object()) { - spdlog::warn( + log::warn( "{}: Using a JSON object for 'depends' is deprecated. Use an " "array of strings instead.", fpath); diff --git a/src/dds/proc.nix.cpp b/src/dds/proc.nix.cpp index 95d857c9..144b21d8 100644 --- a/src/dds/proc.nix.cpp +++ b/src/dds/proc.nix.cpp @@ -1,10 +1,9 @@ #ifndef _WIN32 #include "./proc.hpp" +#include #include -#include - #include #include #include @@ -63,7 +62,7 @@ spawn_child(const std::vector& command, int stdout_pipe, int close_ } // namespace proc_result dds::run_proc(const proc_options& opts) { - spdlog::debug("Spawning subprocess: {}", quote_command(opts.command)); + log::debug("Spawning subprocess: {}", quote_command(opts.command)); int stdio_pipe[2] = {}; auto rc = ::pipe(stdio_pipe); check_rc(rc == 0, "Create stdio pipe for subprocess"); @@ -101,7 +100,7 @@ proc_result dds::run_proc(const proc_options& opts) { ::kill(child, SIGINT); timeout = -1ms; res.timed_out = true; - spdlog::debug("Subprocess [{}] timed out", quote_command(opts.command)); + log::debug("Subprocess [{}] timed out", quote_command(opts.command)); continue; } std::string buffer; diff --git a/src/dds/proc.win.cpp b/src/dds/proc.win.cpp index fb2f2bc3..c9c71e80 100644 --- a/src/dds/proc.win.cpp +++ b/src/dds/proc.win.cpp @@ -2,7 +2,6 @@ #include "./proc.hpp" #include -#include #include #include diff --git a/src/dds/repo/repo.cpp b/src/dds/repo/repo.cpp index 0779fb05..177c2cda 100644 --- a/src/dds/repo/repo.cpp +++ b/src/dds/repo/repo.cpp @@ -4,11 +4,10 @@ #include #include #include +#include #include #include -#include - #include #include #include @@ -33,9 +32,9 @@ auto load_sdists(path_ref root) { try { return sdist::from_directory(p); } catch (const std::runtime_error& e) { - spdlog::error("Failed to load source distribution from directory '{}': {}", - p.string(), - e.what()); + log::error("Failed to load source distribution from directory '{}': {}", + p.string(), + e.what()); return std::nullopt; } }; @@ -54,8 +53,8 @@ auto load_sdists(path_ref root) { } // namespace void repository::_log_blocking(path_ref dirpath) noexcept { - spdlog::warn("Another process has the repository directory locked [{}]", dirpath.string()); - spdlog::warn("Waiting for repository to be released..."); + log::warn("Another process has the repository directory locked [{}]", dirpath.string()); + log::warn("Waiting for repository to be released..."); } void repository::_init_repo_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); } @@ -69,7 +68,7 @@ repository repository::_open_for_directory(bool writeable, path_ref dirpath) { void repository::add_sdist(const sdist& sd, if_exists ife_action) { if (!_write_enabled) { - spdlog::critical( + log::critical( "DDS attempted to write into a repository that wasn't opened with a write-lock. This " "is a hard bug and should be reported. For the safety and integrity of the local " "repository, we'll hard-exit immediately."); @@ -82,10 +81,10 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) { if (ife_action == if_exists::throw_exc) { throw_user_error(msg); } else if (ife_action == if_exists::ignore) { - spdlog::warn(msg); + log::warn(msg); return; } else { - spdlog::info(msg + " - Replacing"); + log::info(msg + " - Replacing"); } } auto tmp_copy = sd_dest; @@ -100,7 +99,7 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) { } fs::rename(tmp_copy, sd_dest); _sdists.insert(sdist::from_directory(sd_dest)); - spdlog::info("Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); + log::info("Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); } const sdist* repository::find(const package_id& pkg) const noexcept { diff --git a/src/dds/solve/solve.cpp b/src/dds/solve/solve.cpp index c62c00c2..ffc77df3 100644 --- a/src/dds/solve/solve.cpp +++ b/src/dds/solve/solve.cpp @@ -1,12 +1,12 @@ #include "./solve.hpp" #include +#include #include #include #include -#include #include @@ -129,7 +129,7 @@ struct explainer { void operator()(pubgrub::explain::premise pr) { strm.str(""); put(pr.value); - spdlog::error("{} {},", at_head ? "┌─ Given that" : "│ and", strm.str()); + log::error("{} {},", at_head ? "┌─ Given that" : "│ and", strm.str()); at_head = false; } @@ -138,10 +138,10 @@ struct explainer { at_head = true; strm.str(""); put(cncl.value); - spdlog::error("╘═ then {}.", strm.str()); + log::error("╘═ then {}.", strm.str()); } - void operator()(pubgrub::explain::separator) { spdlog::error(""); } + void operator()(pubgrub::explain::separator) { log::error(""); } }; } // namespace @@ -156,7 +156,7 @@ std::vector dds::solve(const std::vector& deps, auto solution = pubgrub::solve(wrap_req, solver_provider{pkgs_prov, deps_prov}); return solution | ranges::views::transform(as_pkg_id) | ranges::to_vector; } catch (const solve_fail_exc& failure) { - spdlog::error("Dependency resolution has failed! Explanation:"); + log::error("Dependency resolution has failed! Explanation:"); pubgrub::generate_explaination(failure, explainer()); throw_user_error(); } diff --git a/src/dds/source/dist.cpp b/src/dds/source/dist.cpp index 2ed34884..7ebe41bd 100644 --- a/src/dds/source/dist.cpp +++ b/src/dds/source/dist.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -11,15 +12,13 @@ #include #include -#include - using namespace dds; namespace { void sdist_export_file(path_ref out_root, path_ref in_root, path_ref filepath) { auto relpath = fs::relative(filepath, in_root); - spdlog::debug("Export file {}", relpath.string()); + log::debug("Export file {}", relpath.string()); auto dest = out_root / relpath; fs::create_directories(dest.parent_path()); fs::copy(filepath, dest); @@ -53,7 +52,7 @@ void sdist_copy_library(path_ref out_root, const library_root& lib, const sdist_ } sdist_export_file(out_root, params.project_dir, *lib_man_path); - spdlog::info("sdist: Export library from {}", lib.path().string()); + log::info("sdist: Export library from {}", lib.path().string()); fs::create_directories(out_root); for (const auto& source : sources_to_keep) { sdist_export_file(out_root, params.project_dir, source.path); @@ -78,7 +77,7 @@ sdist dds::create_sdist(const sdist_params& params) { } fs::create_directories(dest.parent_path()); safe_rename(tempdir.path(), dest); - spdlog::info("Source distribution created in {}", dest.string()); + log::info("Source distribution created in {}", dest.string()); return sdist::from_directory(dest); } @@ -99,7 +98,7 @@ sdist dds::create_sdist_in_dir(path_ref out, const sdist_params& params) { auto pkg_man = package_manifest::load_from_file(*man_path); sdist_export_file(out, params.project_dir, *man_path); - spdlog::info("Generated export as {}", pkg_man.pkg_id.to_string()); + log::info("Generated export as {}", pkg_man.pkg_id.to_string()); return sdist::from_directory(out); } diff --git a/src/dds/util/log.cpp b/src/dds/util/log.cpp new file mode 100644 index 00000000..e92b552a --- /dev/null +++ b/src/dds/util/log.cpp @@ -0,0 +1,33 @@ +#include "./log.hpp" + +#include + +#include + +void dds::log::log_print(dds::log::level l, std::string_view msg) noexcept { + static auto logger = [] { + auto logger = spdlog::default_logger_raw(); + logger->set_level(spdlog::level::trace); + return logger; + }(); + + const auto lvl = [&] { + switch (l) { + case level::trace: + return spdlog::level::trace; + case level::debug: + return spdlog::level::debug; + case level::info: + return spdlog::level::info; + case level::warn: + return spdlog::level::warn; + case level::error: + return spdlog::level::err; + case level::critical: + return spdlog::level::critical; + } + neo_assert_always(invariant, false, "Invalid log level", msg, int(l)); + }(); + + logger->log(lvl, msg); +} diff --git a/src/dds/util/log.hpp b/src/dds/util/log.hpp new file mode 100644 index 00000000..948b1894 --- /dev/null +++ b/src/dds/util/log.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include + +#include + +namespace dds::log { + +enum class level : int { + trace, + debug, + info, + warn, + error, + critical, +}; + +inline level current_log_level = level::info; + +void log_print(level l, std::string_view s) noexcept; + +// clang-format off +template +concept formattable = requires (const T item) { + fmt::format("{}", item); +}; + +template +void log(level l, std::string_view s, const Args&... args) { + if (int(l) >= int(current_log_level)) { + auto message = fmt::format(s, args...); + log_print(l, message); + } +} + +template +void trace(std::string_view s, const Args&... args) { + log(level::trace, s, args...); +} + +template +void debug(std::string_view s, const Args&... args) { + log(level::debug, s, args...); +} + +template +void info(std::string_view s, const Args&... args) { + log(level::info, s, args...); +} + +template +void warn(std::string_view s, const Args&... args) { + log(level::warn, s, args...); +} + +template +void error(std::string_view s, const Args&... args) { + log(level::error, s, args...); +} + +template +void critical(std::string_view s, const Args&&... args) { + log(level::critical, s, args...); +} + +// clang-format on + +} // namespace dds::log \ No newline at end of file diff --git a/src/dds/util/parallel.cpp b/src/dds/util/parallel.cpp index b30efd10..db8986f2 100644 --- a/src/dds/util/parallel.cpp +++ b/src/dds/util/parallel.cpp @@ -1,6 +1,6 @@ #include "./parallel.hpp" -#include +#include using namespace dds; @@ -8,6 +8,6 @@ void dds::log_exception(std::exception_ptr eptr) noexcept { try { std::rethrow_exception(eptr); } catch (const std::exception& e) { - spdlog::error(e.what()); + log::error(e.what()); } } diff --git a/src/dds/util/paths.linux_fbsd.cpp b/src/dds/util/paths.linux_fbsd.cpp index 0be907c7..7ebfe41d 100644 --- a/src/dds/util/paths.linux_fbsd.cpp +++ b/src/dds/util/paths.linux_fbsd.cpp @@ -2,7 +2,7 @@ #include "./paths.hpp" -#include +#include #include @@ -12,7 +12,7 @@ fs::path dds::user_home_dir() { static auto ret = []() -> fs::path { auto home_env = std::getenv("HOME"); if (!home_env) { - spdlog::warn("No HOME environment variable set!"); + log::error("No HOME environment variable set!"); return "/"; } return fs::absolute(fs::path(home_env)); diff --git a/src/dds/util/paths.macos.cpp b/src/dds/util/paths.macos.cpp index 7ded6485..e81613f2 100644 --- a/src/dds/util/paths.macos.cpp +++ b/src/dds/util/paths.macos.cpp @@ -2,7 +2,7 @@ #include "./paths.hpp" -#include +#include #include @@ -12,7 +12,7 @@ fs::path dds::user_home_dir() { static auto ret = []() -> fs::path { auto home_env = std::getenv("HOME"); if (!home_env) { - spdlog::warn("No HOME environment variable set!"); + log::warn("No HOME environment variable set!"); return "/"; } return fs::absolute(fs::path(home_env)); diff --git a/src/dds/util/paths.win.cpp b/src/dds/util/paths.win.cpp index 077c8838..2ad897c2 100644 --- a/src/dds/util/paths.win.cpp +++ b/src/dds/util/paths.win.cpp @@ -2,7 +2,7 @@ #include "./paths.hpp" -#include +#include #include @@ -12,7 +12,7 @@ fs::path dds::user_home_dir() { static auto ret = []() -> fs::path { auto userprofile_env = std::getenv("USERPROFILE"); if (!userprofile_env) { - spdlog::warn("No USERPROFILE environment variable set!"); + log::warn("No USERPROFILE environment variable set!"); return "/"; } return fs::absolute(fs::path(userprofile_env)); diff --git a/src/libman/library.cpp b/src/libman/library.cpp index 423d0941..101f1ccb 100644 --- a/src/libman/library.cpp +++ b/src/libman/library.cpp @@ -2,7 +2,7 @@ #include -#include +#include using namespace lm; From 007d5dfd43aa7c58da56e48ac519d51841f4b4a5 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 15:54:57 -0600 Subject: [PATCH 06/36] Win32 NOMINMAX --- src/dds/proc.win.cpp | 1 + src/dds/util/log.cpp | 2 ++ src/dds/util/paths.win.cpp | 1 + tools/msvc.jsonc | 1 + 4 files changed, 5 insertions(+) diff --git a/src/dds/proc.win.cpp b/src/dds/proc.win.cpp index c9c71e80..066be53c 100644 --- a/src/dds/proc.win.cpp +++ b/src/dds/proc.win.cpp @@ -1,6 +1,7 @@ #ifdef _WIN32 #include "./proc.hpp" +#include #include #include diff --git a/src/dds/util/log.cpp b/src/dds/util/log.cpp index e92b552a..85c56082 100644 --- a/src/dds/util/log.cpp +++ b/src/dds/util/log.cpp @@ -4,6 +4,8 @@ #include +#include + void dds::log::log_print(dds::log::level l, std::string_view msg) noexcept { static auto logger = [] { auto logger = spdlog::default_logger_raw(); diff --git a/src/dds/util/paths.win.cpp b/src/dds/util/paths.win.cpp index 2ad897c2..0c91b6c6 100644 --- a/src/dds/util/paths.win.cpp +++ b/src/dds/util/paths.win.cpp @@ -4,6 +4,7 @@ #include +#include #include using namespace dds; diff --git a/tools/msvc.jsonc b/tools/msvc.jsonc index b452ee21..43f1fa94 100644 --- a/tools/msvc.jsonc +++ b/tools/msvc.jsonc @@ -4,6 +4,7 @@ "flags": [ "/Zc:preprocessor", "/std:c++latest", + "/DNOMINMAX", ], "link_flags": [ "rpcrt4.lib", From ef887e644bf2cd60db6cbca17c17079bac19baef Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 22:27:35 -0600 Subject: [PATCH 07/36] Complete migration to fmt 7.0 --- src/dds/build/builder.cpp | 8 ++++---- src/dds/build/plan/archive.cpp | 2 +- src/dds/build/plan/compile_exec.cpp | 2 +- src/dds/build/plan/exe.cpp | 6 +++--- src/dds/catalog/import.cpp | 4 ++-- src/dds/deps.cpp | 2 +- src/dds/error/errors.hpp | 2 +- src/dds/package/id.cpp | 2 +- src/dds/toolchain/from_json.cpp | 2 +- src/dds/usage_reqs.cpp | 2 +- src/dds/util/flock.nix.cpp | 2 +- src/dds/util/flock.win.cpp | 4 +++- src/dds/util/fs.cpp | 2 +- src/dds/util/log.hpp | 2 +- src/libman/index.cpp | 2 +- src/libman/package.cpp | 2 +- src/libman/parse.cpp | 2 +- 17 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/dds/build/builder.cpp b/src/dds/build/builder.cpp index 91cf99d2..832b5809 100644 --- a/src/dds/build/builder.cpp +++ b/src/dds/build/builder.cpp @@ -241,19 +241,19 @@ void builder::build(const build_params& params) const { dds::stopwatch sw; plan.compile_all(env, params.parallel_jobs); - log::info("Compilation completed in {:n}ms", sw.elapsed_ms().count()); + log::info("Compilation completed in {:L}ms", sw.elapsed_ms().count()); sw.reset(); plan.archive_all(env, params.parallel_jobs); - log::info("Archiving completed in {:n}ms", sw.elapsed_ms().count()); + log::info("Archiving completed in {:L}ms", sw.elapsed_ms().count()); sw.reset(); plan.link_all(env, params.parallel_jobs); - log::info("Runtime binary linking completed in {:n}ms", sw.elapsed_ms().count()); + log::info("Runtime binary linking completed in {:L}ms", sw.elapsed_ms().count()); sw.reset(); auto test_failures = plan.run_all_tests(env, params.parallel_jobs); - log::info("Test execution finished in {:n}ms", sw.elapsed_ms().count()); + log::info("Test execution finished in {:L}ms", sw.elapsed_ms().count()); for (auto& fail : test_failures) { log_failure(fail); diff --git a/src/dds/build/plan/archive.cpp b/src/dds/build/plan/archive.cpp index e6123917..41568c26 100644 --- a/src/dds/build/plan/archive.cpp +++ b/src/dds/build/plan/archive.cpp @@ -42,7 +42,7 @@ void create_archive_plan::archive(const build_env& env) const { // Do it! log::info("[{}] Archive: {}", _qual_name, out_relpath); auto&& [dur_ms, ar_res] = timed([&] { return run_proc(ar_cmd); }); - log::info("[{}] Archive: {} - {:n}ms", _qual_name, out_relpath, dur_ms.count()); + log::info("[{}] Archive: {} - {:L}ms", _qual_name, out_relpath, dur_ms.count()); // Check, log, and throw if (!ar_res.okay()) { diff --git a/src/dds/build/plan/compile_exec.cpp b/src/dds/build/plan/compile_exec.cpp index 31e295af..03a1b43f 100644 --- a/src/dds/build/plan/compile_exec.cpp +++ b/src/dds/build/plan/compile_exec.cpp @@ -58,7 +58,7 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun auto&& [dur_ms, proc_res] = timed([&] { return run_proc(cf.cmd_info.command); }); auto nth = counter.n.fetch_add(1); - log::info("{:60} - {:>7n}ms [{:{}}/{}]", + log::info("{:60} - {:>7L}ms [{:{}}/{}]", msg, dur_ms.count(), nth, diff --git a/src/dds/build/plan/exe.cpp b/src/dds/build/plan/exe.cpp index 0bd644d8..d272f182 100644 --- a/src/dds/build/plan/exe.cpp +++ b/src/dds/build/plan/exe.cpp @@ -51,7 +51,7 @@ void link_executable_plan::link(build_env_ref env, const library_plan& lib) cons log::info(msg); auto [dur_ms, proc_res] = timed([&] { return run_proc(link_command); }); - log::info("{} - {:>6n}ms", msg, dur_ms.count()); + log::info("{} - {:>6L}ms", msg, dur_ms.count()); // Check and throw if errant if (!proc_res.okay()) { @@ -82,13 +82,13 @@ std::optional link_executable_plan::run_test(build_env_ref env) co [&] { return run_proc({.command = {exe_path.string()}, .timeout = 10s}); }); if (res.okay()) { - log::info("{} - PASSED - {:>9n}μs", msg, dur.count()); + log::info("{} - PASSED - {:>9L}μs", msg, dur.count()); return std::nullopt; } else { auto exit_msg = fmt::format(res.signal ? "signalled {}" : "exited {}", res.signal ? res.signal : res.retc); auto fail_str = res.timed_out ? "TIMEOUT" : "FAILED "; - log::error("{} - {} - {:>9n}μs [{}]", msg, fail_str, dur.count(), exit_msg); + log::error("{} - {} - {:>9L}μs [{}]", msg, fail_str, dur.count(), exit_msg); test_failure f; f.executable_path = exe_path; f.output = res.output; diff --git a/src/dds/catalog/import.cpp b/src/dds/catalog/import.cpp index 219780c3..81b10509 100644 --- a/src/dds/catalog/import.cpp +++ b/src/dds/catalog/import.cpp @@ -2,10 +2,10 @@ #include +#include #include #include #include -#include #include @@ -31,7 +31,7 @@ struct any_key { }; template -any_key(KF&&, Args&&...) -> any_key; +any_key(KF&&, Args&&...)->any_key; namespace { diff --git a/src/dds/deps.cpp b/src/dds/deps.cpp index a30f094e..1f0c5bf4 100644 --- a/src/dds/deps.cpp +++ b/src/dds/deps.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include diff --git a/src/dds/error/errors.hpp b/src/dds/error/errors.hpp index 2ea0fd50..1f60fabb 100644 --- a/src/dds/error/errors.hpp +++ b/src/dds/error/errors.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/src/dds/package/id.cpp b/src/dds/package/id.cpp index eb200e60..a22b5fba 100644 --- a/src/dds/package/id.cpp +++ b/src/dds/package/id.cpp @@ -2,7 +2,7 @@ #include -#include +#include #include diff --git a/src/dds/toolchain/from_json.cpp b/src/dds/toolchain/from_json.cpp index e5d1050c..efc91cae 100644 --- a/src/dds/toolchain/from_json.cpp +++ b/src/dds/toolchain/from_json.cpp @@ -6,9 +6,9 @@ #include #include +#include #include #include -#include #include diff --git a/src/dds/usage_reqs.cpp b/src/dds/usage_reqs.cpp index 2c856a41..f9864ea9 100644 --- a/src/dds/usage_reqs.cpp +++ b/src/dds/usage_reqs.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include diff --git a/src/dds/util/flock.nix.cpp b/src/dds/util/flock.nix.cpp index dced491f..e1d5d5d9 100644 --- a/src/dds/util/flock.nix.cpp +++ b/src/dds/util/flock.nix.cpp @@ -4,7 +4,7 @@ #include -#include +#include #include #include diff --git a/src/dds/util/flock.win.cpp b/src/dds/util/flock.win.cpp index 0692a4ad..97b21153 100644 --- a/src/dds/util/flock.win.cpp +++ b/src/dds/util/flock.win.cpp @@ -4,9 +4,11 @@ #include -#include +#include #include +#include + using namespace dds; namespace { diff --git a/src/dds/util/fs.cpp b/src/dds/util/fs.cpp index fa220bdd..a2224289 100644 --- a/src/dds/util/fs.cpp +++ b/src/dds/util/fs.cpp @@ -1,6 +1,6 @@ #include "./fs.hpp" -#include +#include #include diff --git a/src/dds/util/log.hpp b/src/dds/util/log.hpp index 948b1894..71410d71 100644 --- a/src/dds/util/log.hpp +++ b/src/dds/util/log.hpp @@ -26,7 +26,7 @@ concept formattable = requires (const T item) { }; template -void log(level l, std::string_view s, const Args&... args) { +void log(level l, std::string_view s, const Args&... args) noexcept { if (int(l) >= int(current_log_level)) { auto message = fmt::format(s, args...); log_print(l, message); diff --git a/src/libman/index.cpp b/src/libman/index.cpp index 4552df28..60aaea41 100644 --- a/src/libman/index.cpp +++ b/src/libman/index.cpp @@ -2,7 +2,7 @@ #include -#include +#include using namespace lm; diff --git a/src/libman/package.cpp b/src/libman/package.cpp index e6a888e2..de3c669c 100644 --- a/src/libman/package.cpp +++ b/src/libman/package.cpp @@ -2,7 +2,7 @@ #include -#include +#include using namespace lm; diff --git a/src/libman/parse.cpp b/src/libman/parse.cpp index 045990e2..80ec27d8 100644 --- a/src/libman/parse.cpp +++ b/src/libman/parse.cpp @@ -2,7 +2,7 @@ #include -#include +#include #include #include From 88634e0be0e82202f13eaf5d33ca3904ce76398e Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 22:28:26 -0600 Subject: [PATCH 08/36] Set Windows console to UTF-8, and prevent unexpected formatting --- src/dds/util/log.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/dds/util/log.cpp b/src/dds/util/log.cpp index 85c56082..fad62830 100644 --- a/src/dds/util/log.cpp +++ b/src/dds/util/log.cpp @@ -6,10 +6,23 @@ #include +#if _WIN32 +#include +static void set_utf8_output() { + // 65'001 is the codepage id for UTF-8 output + ::SetConsoleOutputCP(65'001); +} +#else +static void set_utf8_output() { + // Nothing on other platforms +} +#endif + void dds::log::log_print(dds::log::level l, std::string_view msg) noexcept { - static auto logger = [] { + static auto logger_inst = [] { auto logger = spdlog::default_logger_raw(); logger->set_level(spdlog::level::trace); + set_utf8_output(); return logger; }(); @@ -31,5 +44,5 @@ void dds::log::log_print(dds::log::level l, std::string_view msg) noexcept { neo_assert_always(invariant, false, "Invalid log level", msg, int(l)); }(); - logger->log(lvl, msg); + logger_inst->log(lvl, "{}", msg); } From 21edf271079aa0156b1b3d1f2c8c0cc5e601259f Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Thu, 16 Jul 2020 22:31:17 -0600 Subject: [PATCH 09/36] Missing #include for assert() --- src/dds/util/flock.nix.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dds/util/flock.nix.cpp b/src/dds/util/flock.nix.cpp index e1d5d5d9..52a5b6cd 100644 --- a/src/dds/util/flock.nix.cpp +++ b/src/dds/util/flock.nix.cpp @@ -10,6 +10,8 @@ #include #include +#include + using namespace dds; namespace { From e259992656e802b9602b62b59e2d362a5dbefe6f Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Fri, 17 Jul 2020 15:12:00 -0600 Subject: [PATCH 10/36] More flexible control over debug/runtime flag selection --- res/toolchain-schema.json | 33 +++++++- src/dds/toolchain/from_json.cpp | 122 ++++++++++++++++++++------- src/dds/toolchain/from_json.test.cpp | 47 ++++++++++- 3 files changed, 166 insertions(+), 36 deletions(-) diff --git a/res/toolchain-schema.json b/res/toolchain-schema.json index ced6469b..81f4b777 100644 --- a/res/toolchain-schema.json +++ b/res/toolchain-schema.json @@ -5,6 +5,7 @@ "patternProperties": { "^\\$": {} }, + "$schema": "https://json-schema.org/draft/2019-09/schema", "definitions": { "command_line_flags": { "anyOf": [ @@ -98,15 +99,41 @@ "$ref": "#/definitions/command_line_flags" }, "debug": { - "description": "Enable the generation of debug information", - "type": "boolean", - "default": true + "description": "Tweak the generation of debug information", + "default": true, + "oneOf": [ + { + "type": "string", + "enum": [ + "embedded", + "split" + ] + }, + { + "type": "boolean" + } + ] }, "optimize": { "description": "Optimize generated code", "type": "boolean", "default": true }, + "runtime": { + "description": "Select the runtime/stdlib modes", + "type": "object", + "additionalProperties": false, + "properties": { + "static": { + "type": "boolean", + "description": "If `true`, enable static stdlib/runtime linking" + }, + "debug": { + "type": "boolean", + "description": "If 'true', enable debug for the stdlib/runtime" + } + } + }, "advanced": { "type": "object", "additionalProperties": false, diff --git a/src/dds/toolchain/from_json.cpp b/src/dds/toolchain/from_json.cpp index efc91cae..583413dc 100644 --- a/src/dds/toolchain/from_json.cpp +++ b/src/dds/toolchain/from_json.cpp @@ -61,8 +61,11 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie opt_string_seq link_flags; opt_string_seq warning_flags; - optional do_debug; + optional debug_bool; + opt_string debug_str; optional do_optimize; + optional runtime_static; + optional runtime_debug; // Advanced-mode: opt_string deps_mode_str; @@ -130,12 +133,25 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie KEY_EXTEND_FLAGS(link_flags), KEY_EXTEND_FLAGS(compiler_launcher), if_key{"debug", - require_type("`debug` must be a boolean value"), - put_into{do_debug}}, + if_type(put_into{debug_bool}), + if_type(put_into{debug_str}), + reject_with{"'debug' must be a bool or string"}}, if_key{"optimize", require_type("`optimize` must be a boolean value"), put_into{do_optimize}}, if_key{"flags", extend_flags("flags", common_flags)}, + if_key{"runtime", + require_type("'runtime' must be a JSON object"), + mapping{if_key{"static", + require_type("'/runtime/static' should be a boolean"), + put_into(runtime_static)}, + if_key{"debug", + require_type("'/runtime/debug' should be a boolean"), + put_into(runtime_debug)}, + [](auto&& key, auto&&) { + fail("Unknown 'runtime' key '{}'", key); + return dc_reject_t(); + }}}, if_key{ "advanced", require_type("`advanced` must be a mapping"), @@ -202,6 +218,7 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie "flags", "debug", "optimize", + "runtime", }); fail(context, "Unknown toolchain config key ‘{}’ (Did you mean ‘{}’?)", @@ -217,6 +234,11 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie fail(context, rej_opt->message); } + if (debug_str.has_value() && debug_str != "embedded" && debug_str != "split" + && debug_str != "none") { + fail(context, "'debug' string must be one of 'none', 'embedded', or 'split'"); + } + enum compiler_id_e_t { no_comp_id, msvc, @@ -404,26 +426,78 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie return cxx_ver_iter->second; }; - auto get_link_flags = [&]() -> string_seq { + auto get_runtime_flags = [&]() -> string_seq { string_seq ret; if (is_msvc) { - strv rt_lib = "/MT"; - if (do_optimize.value_or(false)) { - extend(ret, {"/O2"}); + std::string rt_flag = "/M"; + // Select debug/release runtime flag. Default is 'true' + if (runtime_static.value_or(true)) { + rt_flag.push_back('T'); + } else { + rt_flag.push_back('D'); } - if (do_debug.value_or(false)) { - extend(ret, {"/Z7", "/DEBUG"}); - rt_lib = "/MTd"; + if (runtime_debug.value_or(debug_bool.value_or(false) || debug_str == "embedded" + || debug_str == "split")) { + rt_flag.push_back('d'); } - ret.emplace_back(rt_lib); + ret.push_back(rt_flag); } else if (is_gnu_like) { - if (do_optimize.value_or(false)) { - extend(ret, {"-O2"}); + if (runtime_static.value_or(false)) { + extend(ret, {"-static-libgcc", "-static-libstdc++"}); } - if (do_debug.value_or(false)) { - extend(ret, {"-g"}); + if (runtime_debug.value_or(false)) { + extend(ret, {"-D_GLIBCXX_DEBUG"}); } + } else { + // No flags if we don't know the compiler + } + + return ret; + }; + + auto get_optim_flags = [&]() -> string_seq { + if (do_optimize != true) { + return {}; } + if (is_msvc) { + return {"/O2"}; + } else if (is_gnu_like) { + return {"-O2"}; + } else { + return {}; + } + }; + + auto get_debug_flags = [&]() -> string_seq { + if (is_msvc) { + if (debug_bool == true || debug_str == "embedded") { + return {"/Z7"}; + } else if (debug_str == "split") { + return {"/Zi"}; + } else { + // Do not generate any debug infro + return {}; + } + } else if (is_gnu_like) { + if (debug_bool == true || debug_str == "embedded") { + return {"-g"}; + } else if (debug_str == "split") { + return {"-g", "-gsplit-dwarf"}; + } else { + // Do not generate debug info + return {}; + } + } else { + // Cannont deduce the debug flags + return {}; + } + }; + + auto get_link_flags = [&]() -> string_seq { + string_seq ret; + extend(ret, get_runtime_flags()); + extend(ret, get_optim_flags()); + extend(ret, get_debug_flags()); if (link_flags) { extend(ret, *link_flags); } @@ -432,27 +506,15 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie auto get_flags = [&](language lang) -> string_seq { string_seq ret; + extend(ret, get_runtime_flags()); + extend(ret, get_optim_flags()); + extend(ret, get_debug_flags()); if (is_msvc) { - strv rt_lib = "/MT"; - if (do_optimize.has_value() && *do_optimize) { - extend(ret, {"/O2"}); - } - if (do_debug.has_value() && *do_debug) { - extend(ret, {"/Z7", "/DEBUG"}); - rt_lib = "/MTd"; - } - ret.emplace_back(rt_lib); if (lang == language::cxx) { extend(ret, {"/EHsc"}); } extend(ret, {"/nologo", "/permissive-", "[flags]", "/c", "[in]", "/Fo[out]"}); } else if (is_gnu_like) { - if (do_optimize.has_value() && *do_optimize) { - extend(ret, {"-O2"}); - } - if (do_debug.has_value() && *do_debug) { - extend(ret, {"-g"}); - } extend(ret, {"-fPIC", "-pthread", "[flags]", "-c", "[in]", "-o[out]"}); } if (common_flags) { diff --git a/src/dds/toolchain/from_json.test.cpp b/src/dds/toolchain/from_json.test.cpp index 98fd8cca..9362e5af 100644 --- a/src/dds/toolchain/from_json.test.cpp +++ b/src/dds/toolchain/from_json.test.cpp @@ -69,6 +69,14 @@ TEST_CASE("Generating toolchain commands") { "ar rcs stuff.a foo.o bar.o", "g++ -fPIC foo.o bar.a -pthread -omeow.exe -O2 -g"); + check_tc_compile( + "{compiler_id: 'gnu', debug: 'split', optimize: true}", + "g++ -O2 -g -gsplit-dwarf -fPIC -pthread -MD -MF foo.o.d -MT foo.o -c foo.cpp -ofoo.o", + "g++ -O2 -g -gsplit-dwarf -fPIC -pthread -Wall -Wextra -Wpedantic -Wconversion -MD -MF " + "foo.o.d -MT foo.o -c foo.cpp -ofoo.o", + "ar rcs stuff.a foo.o bar.o", + "g++ -fPIC foo.o bar.a -pthread -omeow.exe -O2 -g -gsplit-dwarf"); + check_tc_compile("{compiler_id: 'msvc'}", "cl.exe /MT /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", "cl.exe /MT /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", @@ -77,10 +85,24 @@ TEST_CASE("Generating toolchain commands") { check_tc_compile( "{compiler_id: 'msvc', debug: true}", - "cl.exe /Z7 /DEBUG /MTd /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", - "cl.exe /Z7 /DEBUG /MTd /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MTd /Z7 /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MTd /Z7 /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "lib /nologo /OUT:stuff.a foo.o bar.o", + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MTd /Z7"); + + check_tc_compile( + "{compiler_id: 'msvc', debug: 'embedded'}", + "cl.exe /MTd /Z7 /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MTd /Z7 /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "lib /nologo /OUT:stuff.a foo.o bar.o", + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MTd /Z7"); + + check_tc_compile( + "{compiler_id: 'msvc', debug: 'split'}", + "cl.exe /MTd /Zi /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MTd /Zi /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", "lib /nologo /OUT:stuff.a foo.o bar.o", - "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /Z7 /DEBUG /MTd"); + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MTd /Zi"); check_tc_compile( "{compiler_id: 'msvc', flags: '-DFOO'}", @@ -88,6 +110,25 @@ TEST_CASE("Generating toolchain commands") { "cl.exe /MT /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o -DFOO", "lib /nologo /OUT:stuff.a foo.o bar.o", "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MT"); + + check_tc_compile("{compiler_id: 'msvc', runtime: {static: false}}", + "cl.exe /MD /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MD /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "lib /nologo /OUT:stuff.a foo.o bar.o", + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MD"); + + check_tc_compile( + "{compiler_id: 'msvc', runtime: {static: false}, debug: true}", + "cl.exe /MDd /Z7 /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MDd /Z7 /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "lib /nologo /OUT:stuff.a foo.o bar.o", + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MDd /Z7"); + + check_tc_compile("{compiler_id: 'msvc', runtime: {static: false, debug: true}}", + "cl.exe /MDd /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MDd /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "lib /nologo /OUT:stuff.a foo.o bar.o", + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MDd"); } TEST_CASE("Manipulate a toolchain and file compilation") { From 7f3dad4dfc780a3808675228134a7165a86a54d0 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Fri, 17 Jul 2020 16:07:11 -0600 Subject: [PATCH 11/36] /FS is required for parallel access to a separate .pdb file --- src/dds/toolchain/from_json.cpp | 2 +- src/dds/toolchain/from_json.test.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dds/toolchain/from_json.cpp b/src/dds/toolchain/from_json.cpp index 583413dc..c3239e18 100644 --- a/src/dds/toolchain/from_json.cpp +++ b/src/dds/toolchain/from_json.cpp @@ -473,7 +473,7 @@ toolchain dds::parse_toolchain_json_data(const json5::data& dat, std::string_vie if (debug_bool == true || debug_str == "embedded") { return {"/Z7"}; } else if (debug_str == "split") { - return {"/Zi"}; + return {"/Zi", "/FS"}; } else { // Do not generate any debug infro return {}; diff --git a/src/dds/toolchain/from_json.test.cpp b/src/dds/toolchain/from_json.test.cpp index 9362e5af..d8c89b55 100644 --- a/src/dds/toolchain/from_json.test.cpp +++ b/src/dds/toolchain/from_json.test.cpp @@ -99,10 +99,10 @@ TEST_CASE("Generating toolchain commands") { check_tc_compile( "{compiler_id: 'msvc', debug: 'split'}", - "cl.exe /MTd /Zi /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", - "cl.exe /MTd /Zi /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MTd /Zi /FS /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o", + "cl.exe /MTd /Zi /FS /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o", "lib /nologo /OUT:stuff.a foo.o bar.o", - "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MTd /Zi"); + "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MTd /Zi /FS"); check_tc_compile( "{compiler_id: 'msvc', flags: '-DFOO'}", From cdc9b6d6204f680398d7fa5128abefb568589def Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Fri, 17 Jul 2020 21:41:21 -0600 Subject: [PATCH 12/36] Significantly improved logging, and log-level cli param --- src/dds.main.cpp | 47 +++++++++++++------- src/dds/build/builder.cpp | 18 ++++---- src/dds/build/file_deps.cpp | 7 +-- src/dds/build/plan/archive.cpp | 12 +++-- src/dds/build/plan/compile_exec.cpp | 69 ++++++++++++++++++++--------- src/dds/build/plan/exe.cpp | 16 ++++--- src/dds/build/plan/library.cpp | 6 +++ src/dds/catalog/catalog.cpp | 69 +++++++++++++++++++---------- src/dds/catalog/get.cpp | 34 ++------------ src/dds/catalog/git.hpp | 20 --------- src/dds/catalog/import.cpp | 5 ++- src/dds/catalog/remote/git.cpp | 26 ++++++++++- src/dds/catalog/remote/git.hpp | 2 +- src/dds/db/database.cpp | 11 ++--- src/dds/library/root.cpp | 4 +- src/dds/package/manifest.cpp | 8 ++-- src/dds/proc.nix.cpp | 4 +- src/dds/proc.win.cpp | 1 + src/dds/repo/repo.cpp | 20 +++++---- src/dds/solve/solve.cpp | 24 +++++++--- src/dds/source/dist.cpp | 8 ++-- src/dds/toolchain/toolchain.cpp | 28 ++++++++++-- src/dds/util/log.hpp | 37 ++++------------ src/dds/util/parallel.cpp | 2 +- src/dds/util/paths.linux_fbsd.cpp | 2 +- src/dds/util/paths.macos.cpp | 2 +- src/dds/util/paths.win.cpp | 2 +- 27 files changed, 280 insertions(+), 204 deletions(-) delete mode 100644 src/dds/catalog/git.hpp diff --git a/src/dds.main.cpp b/src/dds.main.cpp index 7dc5748f..8544f607 100644 --- a/src/dds.main.cpp +++ b/src/dds.main.cpp @@ -107,6 +107,22 @@ struct cli_base { "Print `yes` and exit 0. Useful for scripting.", {"are-you-the-real-dds?"}}; + args::MapFlag log_level{ + parser, + "log-level", + "Set the logging level", + {"log-level", 'l'}, + { + {"trace", dds::log::level::trace}, + {"debug", dds::log::level::debug}, + {"info", dds::log::level::info}, + {"warn", dds::log::level::warn}, + {"error", dds::log::level::error}, + {"critical", dds::log::level::critical}, + }, + dds::log::level::info, + }; + args::Group cmd_group{parser, "Available Commands"}; }; @@ -227,7 +243,7 @@ struct cli_catalog { auto tsd = dds::get_package_sdist(*info); auto out_path = out.Get(); auto dest = out_path / id.to_string(); - dds::log::info("Create sdist at {}", dest.string()); + dds_log(info, "Create sdist at {}", dest.string()); dds::fs::remove_all(dest); dds::safe_rename(tsd.sdist.path, dest); } @@ -342,14 +358,14 @@ struct cli_catalog { auto cat = cat_path.open(); auto pkg = cat.get(pk_id); if (!pkg) { - dds::log::error("No package '{}' in the catalog", pk_id.to_string()); + dds_log(error, "No package '{}' in the catalog", pk_id.to_string()); return 1; } std::cout << "Name: " << pkg->ident.name << '\n' << "Version: " << pkg->ident.version << '\n'; for (const auto& dep : pkg->deps) { - std::cout << "Depends: " << dep.to_string() << '\n'; + std::cout << "Depends: " << dep.to_string() << '\n'; } std::visit([&](const auto& remote) { print_remote_info(remote); }, pkg->remote); @@ -419,9 +435,9 @@ struct cli_repo { }); for (const auto& [name, grp] : grp_by_name) { - dds::log::info("{}:", name); + dds_log(info, "{}:", name); for (const dds::sdist& sd : grp) { - dds::log::info(" - {}", sd.manifest.pkg_id.version.to_string()); + dds_log(info, " - {}", sd.manifest.pkg_id.version.to_string()); } } @@ -692,7 +708,7 @@ struct cli_build_deps { auto all_file_deps = deps_files.Get() // | ranges::views::transform([&](auto dep_fpath) { - dds::log::info("Reading deps from {}", dep_fpath.string()); + dds_log(info, "Reading deps from {}", dep_fpath.string()); return dds::dependency_manifest::from_file(dep_fpath).dependencies; }) | ranges::actions::join; @@ -709,7 +725,7 @@ struct cli_build_deps { dds::repo_flags::write_lock | dds::repo_flags::create_if_absent, [&](dds::repository repo) { // Download dependencies - dds::log::info("Loading {} dependencies", all_deps.size()); + dds_log(info, "Loading {} dependencies", all_deps.size()); auto deps = repo.solve(all_deps, cat); dds::get_all(deps, repo, cat); for (const dds::package_id& pk : deps) { @@ -717,7 +733,7 @@ struct cli_build_deps { assert(sdist_ptr); dds::sdist_build_params deps_params; deps_params.subdir = sdist_ptr->manifest.pkg_id.to_string(); - dds::log::info("Dependency: {}", sdist_ptr->manifest.pkg_id.to_string()); + dds_log(info, "Dependency: {}", sdist_ptr->manifest.pkg_id.to_string()); bd.add(*sdist_ptr, deps_params); } }); @@ -740,9 +756,6 @@ struct cli_build_deps { */ int main(int argc, char** argv) { -#if DDS_DEBUG - dds::log::current_log_level = dds::log::level::debug; -#endif spdlog::set_pattern("[%H:%M:%S] [%^%-5l%$] %v"); args::ArgumentParser parser("DDS - The drop-dead-simple library manager"); @@ -752,6 +765,7 @@ int main(int argc, char** argv) { cli_repo repo{cli}; cli_catalog catalog{cli}; cli_build_deps build_deps{cli}; + try { parser.ParseCLI(argc, argv); } catch (const args::Help&) { @@ -764,6 +778,7 @@ int main(int argc, char** argv) { } dds::install_signal_handlers(); + dds::log::current_log_level = cli.log_level.Get(); try { if (cli._verify_ident) { @@ -784,15 +799,15 @@ int main(int argc, char** argv) { std::terminate(); } } catch (const dds::user_cancelled&) { - dds::log::critical("Operation cancelled by user"); + dds_log(critical, "Operation cancelled by user"); return 2; } catch (const dds::error_base& e) { - dds::log::error("{}", e.what()); - dds::log::error("{}", e.explanation()); - dds::log::error("Refer: {}", e.error_reference()); + dds_log(error, "{}", e.what()); + dds_log(error, "{}", e.explanation()); + dds_log(error, "Refer: {}", e.error_reference()); return 1; } catch (const std::exception& e) { - dds::log::critical(e.what()); + dds_log(critical, e.what()); return 2; } } diff --git a/src/dds/build/builder.cpp b/src/dds/build/builder.cpp index 832b5809..b7f46026 100644 --- a/src/dds/build/builder.cpp +++ b/src/dds/build/builder.cpp @@ -23,14 +23,14 @@ struct state { }; void log_failure(const test_failure& fail) { - log::error("Test '{}' failed! [exited {}]", fail.executable_path.string(), fail.retc); + dds_log(error, "Test '{}' failed! [exited {}]", fail.executable_path.string(), fail.retc); if (fail.signal) { - log::error("Test execution received signal {}", fail.signal); + dds_log(error, "Test execution received signal {}", fail.signal); } if (trim_view(fail.output).empty()) { - log::error("(Test executable produced no output"); + dds_log(error, "(Test executable produced no output"); } else { - log::error("Test output:\n{}[dds - test output end]", fail.output); + dds_log(error, "Test output:\n{}[dds - test output end]", fail.output); } } @@ -83,7 +83,7 @@ prepare_catch2_driver(test_lib test_driver, const build_params& params, build_en auto obj_file = plan.calc_object_file_path(env2); if (!fs::exists(obj_file)) { - log::info("Compiling Catch2 test driver (This will only happen once)..."); + dds_log(info, "Compiling Catch2 test driver (This will only happen once)..."); compile_all(std::array{plan}, env2, 1); } @@ -241,19 +241,19 @@ void builder::build(const build_params& params) const { dds::stopwatch sw; plan.compile_all(env, params.parallel_jobs); - log::info("Compilation completed in {:L}ms", sw.elapsed_ms().count()); + dds_log(info, "Compilation completed in {:L}ms", sw.elapsed_ms().count()); sw.reset(); plan.archive_all(env, params.parallel_jobs); - log::info("Archiving completed in {:L}ms", sw.elapsed_ms().count()); + dds_log(info, "Archiving completed in {:L}ms", sw.elapsed_ms().count()); sw.reset(); plan.link_all(env, params.parallel_jobs); - log::info("Runtime binary linking completed in {:L}ms", sw.elapsed_ms().count()); + dds_log(info, "Runtime binary linking completed in {:L}ms", sw.elapsed_ms().count()); sw.reset(); auto test_failures = plan.run_all_tests(env, params.parallel_jobs); - log::info("Test execution finished in {:L}ms", sw.elapsed_ms().count()); + dds_log(info, "Test execution finished in {:L}ms", sw.elapsed_ms().count()); for (auto& fail : test_failures) { log_failure(fail); diff --git a/src/dds/build/file_deps.cpp b/src/dds/build/file_deps.cpp index 8bb2cd2f..d26de512 100644 --- a/src/dds/build/file_deps.cpp +++ b/src/dds/build/file_deps.cpp @@ -26,14 +26,15 @@ file_deps_info dds::parse_mkfile_deps_str(std::string_view str) { auto iter = split.begin(); auto stop = split.end(); if (iter == stop) { - log::critical( - "Invalid deps listing. Shell split was empty. This is almost certainly a bug."); + dds_log(critical, + "Invalid deps listing. Shell split was empty. This is almost certainly a bug."); return ret; } auto& head = *iter; ++iter; if (!ends_with(head, ":")) { - log::critical( + dds_log( + critical, "Invalid deps listing. Leader item is not colon-terminated. This is probably a bug. " "(Are you trying to use C++ Modules? That's not ready yet, sorry. Set `Deps-Mode` to " "`None` in your toolchain file.)"); diff --git a/src/dds/build/plan/archive.cpp b/src/dds/build/plan/archive.cpp index 41568c26..17cbabe8 100644 --- a/src/dds/build/plan/archive.cpp +++ b/src/dds/build/plan/archive.cpp @@ -33,6 +33,7 @@ void create_archive_plan::archive(const build_env& env) const { // Different archiving tools behave differently between platforms depending on whether the // archive file exists. Make it uniform by simply removing the prior copy. if (fs::exists(ar.out_path)) { + dds_log(debug, "Remove prior archive file [{}]", ar.out_path.string()); fs::remove(ar.out_path); } @@ -40,14 +41,17 @@ void create_archive_plan::archive(const build_env& env) const { fs::create_directories(ar.out_path.parent_path()); // Do it! - log::info("[{}] Archive: {}", _qual_name, out_relpath); + dds_log(info, "[{}] Archive: {}", _qual_name, out_relpath); auto&& [dur_ms, ar_res] = timed([&] { return run_proc(ar_cmd); }); - log::info("[{}] Archive: {} - {:L}ms", _qual_name, out_relpath, dur_ms.count()); + dds_log(info, "[{}] Archive: {} - {:L}ms", _qual_name, out_relpath, dur_ms.count()); // Check, log, and throw if (!ar_res.okay()) { - log::error("Creating static library archive [{}] failed for '{}'", out_relpath, _qual_name); - log::error("Subcommand FAILED: {}\n{}", quote_command(ar_cmd), ar_res.output); + dds_log(error, + "Creating static library archive [{}] failed for '{}'", + out_relpath, + _qual_name); + dds_log(error, "Subcommand FAILED: {}\n{}", quote_command(ar_cmd), ar_res.output); throw_external_error< errc::archive_failure>("Creating static library archive [{}] failed for '{}'", out_relpath, diff --git a/src/dds/build/plan/compile_exec.cpp b/src/dds/build/plan/compile_exec.cpp index 03a1b43f..74c93b09 100644 --- a/src/dds/build/plan/compile_exec.cpp +++ b/src/dds/build/plan/compile_exec.cpp @@ -54,16 +54,17 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun fs::relative(source_path, cf.plan.source().basis_path).string()); // Do it! - log::info(msg); + dds_log(info, msg); auto&& [dur_ms, proc_res] = timed([&] { return run_proc(cf.cmd_info.command); }); auto nth = counter.n.fetch_add(1); - log::info("{:60} - {:>7L}ms [{:{}}/{}]", - msg, - dur_ms.count(), - nth, - counter.max_digits, - counter.max); + dds_log(info, + "{:60} - {:>7L}ms [{:{}}/{}]", + msg, + dur_ms.count(), + nth, + counter.max_digits, + counter.max); const bool compiled_okay = proc_res.okay(); const auto compile_retc = proc_res.retc; @@ -73,16 +74,21 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun // Build dependency information, if applicable to the toolchain std::optional ret_deps_info; - if (env.toolchain.deps_mode() == file_deps_mode::gnu) { + if (!compiled_okay) { + /** + * Do nothing: We failed to compile, so updating deps would be wasteful, and possibly wrong + */ + } else if (env.toolchain.deps_mode() == file_deps_mode::gnu) { // GNU-style deps using Makefile generation assert(cf.cmd_info.gnu_depfile_path.has_value()); auto& df_path = *cf.cmd_info.gnu_depfile_path; if (!fs::is_regular_file(df_path)) { - log::critical( - "The expected Makefile deps were not generated on disk. This is a bug! " - "(Expected file to exist: [{}])", - df_path.string()); + dds_log(critical, + "The expected Makefile deps were not generated on disk. This is a bug! " + "(Expected file to exist: [{}])", + df_path.string()); } else { + dds_log(trace, "Loading compilation dependencies from {}", df_path.string()); auto dep_info = dds::parse_mkfile_deps_file(df_path); assert(dep_info.output == cf.object_file_path); dep_info.command = quote_command(cf.cmd_info.command); @@ -91,6 +97,7 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun } } else if (env.toolchain.deps_mode() == file_deps_mode::msvc) { // Uglier deps generation by parsing the output from cl.exe + dds_log(trace, "Parsing compilation dependencies from MSVC output"); /// TODO: Handle different #include Note: prefixes, since those are localized auto msvc_deps = parse_msvc_output_for_deps(compiler_output, "Note: including file:"); // parse_msvc_output_for_deps will return the compile output without the /showIncludes notes @@ -106,6 +113,10 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun msvc_deps.deps_info.command_output = compiler_output; ret_deps_info = std::move(msvc_deps.deps_info); } + } else { + /** + * We have no deps-mode set, so we can't really figure out what to do. + */ } // MSVC prints the filename of the source file. Remove it from the output. @@ -121,23 +132,25 @@ do_compile(const compile_file_full& cf, build_env_ref env, compile_counter& coun // Log a compiler failure if (!compiled_okay) { - log::error("Compilation failed: {}", source_path.string()); - log::error("Subcommand FAILED [Exitted {}]: {}\n{}", - compile_retc, - quote_command(cf.cmd_info.command), - compiler_output); + dds_log(error, "Compilation failed: {}", source_path.string()); + dds_log(error, + "Subcommand FAILED [Exitted {}]: {}\n{}", + compile_retc, + quote_command(cf.cmd_info.command), + compiler_output); if (compile_signal) { - log::error("Process exited via signal {}", compile_signal); + dds_log(error, "Process exited via signal {}", compile_signal); } throw_user_error("Compilation failed [{}]", source_path.string()); } // Print any compiler output, sans whitespace if (!dds::trim_view(compiler_output).empty()) { - log::warn("While compiling file {} [{}]:\n{}", - source_path.string(), - quote_command(cf.cmd_info.command), - compiler_output); + dds_log(warn, + "While compiling file {} [{}]:\n{}", + source_path.string(), + quote_command(cf.cmd_info.command), + compiler_output); } // We'll only get here if the compilation was successful, otherwise we throw @@ -157,24 +170,35 @@ compile_file_full realize_plan(const compile_file_plan& plan, build_env_ref env) */ bool should_compile(const compile_file_full& comp, const database& db) { if (!fs::exists(comp.object_file_path)) { + dds_log(trace, "Compile {}: Output does not exist", comp.plan.source_path().string()); // The output file simply doesn't exist. We have to recompile, of course. return true; } auto rb_info = get_rebuild_info(db, comp.object_file_path); if (rb_info.previous_command.empty()) { // We have no previous compile command for this file. Assume it is new. + dds_log(trace, "Recompile {}: No prior compilation info", comp.plan.source_path().string()); return true; } if (!rb_info.newer_inputs.empty()) { // Inputs to this file have changed from a prior execution. + dds_log(trace, + "Recompile {}: Inputs have changed (or no input information)", + comp.plan.source_path().string()); return true; } auto cur_cmd_str = quote_command(comp.cmd_info.command); if (cur_cmd_str != rb_info.previous_command) { + dds_log(trace, + "Recompile {}: Compile command has changed", + comp.plan.source_path().string()); // The command used to generate the output is new return true; } // Nope. This file is up-to-date. + dds_log(debug, + "Skip compilation of {} (Result is up-to-date)", + comp.plan.source_path().string()); return false; } @@ -213,6 +237,7 @@ bool dds::detail::compile_all(const ref_vector& compile // Update compile dependency information auto tr = env.db.transaction(); for (auto& info : all_new_deps) { + dds_log(trace, "Update dependency info on {}", info.output.string()); update_deps_info(neo::into(env.db), info); } diff --git a/src/dds/build/plan/exe.cpp b/src/dds/build/plan/exe.cpp index d272f182..afc21197 100644 --- a/src/dds/build/plan/exe.cpp +++ b/src/dds/build/plan/exe.cpp @@ -21,18 +21,24 @@ void link_executable_plan::link(build_env_ref env, const library_plan& lib) cons link_exe_spec spec; spec.output = calc_executable_path(env); spec.inputs = _input_libs; + dds_log(debug, "Performing link for {}", spec.output.string()); for (const lm::usage& links : _links) { + dds_log(trace, " - Link with: {}/{}", links.name, links.namespace_); extend(spec.inputs, env.ureqs.link_paths(links)); } if (lib.archive_plan()) { // The associated library has compiled components. Add the static library a as a linker // input + dds_log(trace, "Adding the library's archive as a linker input"); spec.inputs.push_back(env.output_root / lib.archive_plan()->calc_archive_file_path(env.toolchain)); + } else { + dds_log(trace, "Executable has no corresponding archive library input"); } // The main object should be a linker input, of course. auto main_obj = _main_compile.calc_object_file_path(env); + dds_log(trace, "Add entry point object file: {}", main_obj.string()); spec.inputs.push_back(std::move(main_obj)); // Linker inputs are order-dependent in some cases. The top-most input should appear first, and @@ -48,10 +54,10 @@ void link_executable_plan::link(build_env_ref env, const library_plan& lib) cons auto msg = fmt::format("[{}] Link: {:30}", lib.qualified_name(), fs::relative(spec.output, env.output_root).string()); - log::info(msg); + dds_log(info, msg); auto [dur_ms, proc_res] = timed([&] { return run_proc(link_command); }); - log::info("{} - {:>6L}ms", msg, dur_ms.count()); + dds_log(info, "{} - {:>6L}ms", msg, dur_ms.count()); // Check and throw if errant if (!proc_res.okay()) { @@ -76,19 +82,19 @@ bool link_executable_plan::is_test() const noexcept { std::optional link_executable_plan::run_test(build_env_ref env) const { auto exe_path = calc_executable_path(env); auto msg = fmt::format("Run test: {:30}", fs::relative(exe_path, env.output_root).string()); - log::info(msg); + dds_log(info, msg); using namespace std::chrono_literals; auto&& [dur, res] = timed( [&] { return run_proc({.command = {exe_path.string()}, .timeout = 10s}); }); if (res.okay()) { - log::info("{} - PASSED - {:>9L}μs", msg, dur.count()); + dds_log(info, "{} - PASSED - {:>9L}μs", msg, dur.count()); return std::nullopt; } else { auto exit_msg = fmt::format(res.signal ? "signalled {}" : "exited {}", res.signal ? res.signal : res.retc); auto fail_str = res.timed_out ? "TIMEOUT" : "FAILED "; - log::error("{} - {} - {:>9L}μs [{}]", msg, fail_str, dur.count(), exit_msg); + dds_log(error, "{} - {} - {:>9L}μs [{}]", msg, fail_str, dur.count(), exit_msg); test_failure f; f.executable_path = exe_path; f.output = res.output; diff --git a/src/dds/build/plan/library.cpp b/src/dds/build/plan/library.cpp index 3d7d5e5c..ad8a2b1e 100644 --- a/src/dds/build/plan/library.cpp +++ b/src/dds/build/plan/library.cpp @@ -1,6 +1,7 @@ #include "./library.hpp" #include +#include #include #include @@ -81,10 +82,15 @@ library_plan library_plan::create(const library_root& lib, // for this library std::optional archive_plan; if (!lib_compile_files.empty()) { + dds_log(debug, "Generating an archive library for {}", qual_name); archive_plan.emplace(lib.manifest().name, qual_name, params.out_subdir, std::move(lib_compile_files)); + } else { + dds_log(debug, + "Library {} has no compiled inputs, so no archive will be generated", + qual_name); } // Collect the paths to linker inputs that should be used when generating executables for this diff --git a/src/dds/catalog/catalog.cpp b/src/dds/catalog/catalog.cpp index fb553872..467ce602 100644 --- a/src/dds/catalog/catalog.cpp +++ b/src/dds/catalog/catalog.cpp @@ -140,6 +140,7 @@ void store_with_remote(neo::sqlite3::statement_cache& stmts, void do_store_pkg(neo::sqlite3::database& db, neo::sqlite3::statement_cache& st_cache, const package_info& pkg) { + dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string()); std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote); auto db_pkg_id = db.last_insert_rowid(); auto& new_dep_st = st_cache(R"( @@ -159,6 +160,7 @@ void do_store_pkg(neo::sqlite3::database& db, new_dep_st.reset(); assert(dep.versions.num_intervals() == 1); auto iv_1 = *dep.versions.iter_intervals().begin(); + dds_log(trace, " Depends on: {}", dep.to_string()); sqlite3::exec(new_dep_st, std::forward_as_tuple(db_pkg_id, dep.name, @@ -168,6 +170,7 @@ void do_store_pkg(neo::sqlite3::database& db, } void store_init_packages(sqlite3::database& db, sqlite3::statement_cache& st_cache) { + dds_log(debug, "Restoring initial package data"); for (auto& pkg : init_catalog_packages()) { do_store_pkg(db, st_cache, pkg); } @@ -186,11 +189,13 @@ void ensure_migrated(sqlite3::database& db) { auto meta = nlohmann::json::parse(meta_json); if (!meta.is_object()) { + dds_log(critical, "Root of catalog dds_cat_meta cell should be a JSON object"); throw_external_error(); } auto version_ = meta["version"]; if (!version_.is_number_integer()) { + dds_log(critical, "'version' key in dds_cat_meta is not an integer"); throw_external_error( "The catalog database metadata is invalid [bad dds_meta.version]"); } @@ -204,20 +209,27 @@ void ensure_migrated(sqlite3::database& db) { bool import_init_packages = version == 0; if (version > current_database_version) { + dds_log(critical, + "Catalog version is {}, but we only support up to {}", + version, + current_database_version); throw_external_error(); } if (version < 1) { + dds_log(debug, "Applying catalog migration 1"); migrate_repodb_1(db); } if (version < 2) { + dds_log(debug, "Applying catalog migration 2"); migrate_repodb_2(db); } meta["version"] = 2; exec(db, "UPDATE dds_cat_meta SET meta=?", std::forward_as_tuple(meta.dump())); if (import_init_packages) { - log::info( + dds_log( + info, "A new catalog database case been created, and has been populated with some initial " "contents."); neo::sqlite3::statement_cache stmts{db}; @@ -235,18 +247,22 @@ void check_json(bool b, std::string_view what) { catalog catalog::open(const std::string& db_path) { if (db_path != ":memory:") { - fs::create_directories(fs::weakly_canonical(db_path).parent_path()); + auto pardir = fs::weakly_canonical(db_path).parent_path(); + dds_log(trace, "Ensuring parent directory [{}]", pardir.string()); + fs::create_directories(pardir); } + dds_log(debug, "Opening package catalog [{}]", db_path); auto db = sqlite3::database::open(db_path); try { ensure_migrated(db); } catch (const sqlite3::sqlite3_error& e) { - log::critical( - "Failed to load the repository database. It appears to be invalid/corrupted. The " - "exception message is: {}", - e.what()); + dds_log(critical, + "Failed to load the repository database. It appears to be invalid/corrupted. The " + "exception message is: {}", + e.what()); throw_external_error(); } + dds_log(trace, "Successfully opened catalog"); return catalog(std::move(db)); } @@ -255,11 +271,12 @@ catalog::catalog(sqlite3::database db) void catalog::store(const package_info& pkg) { sqlite3::transaction_guard tr{_db}; - do_store_pkg(_db, _stmt_cache, pkg); } std::optional catalog::get(const package_id& pk_id) const noexcept { + auto ver_str = pk_id.version.to_string(); + dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); auto& st = _stmt_cache(R"( SELECT pkg_id, @@ -275,7 +292,7 @@ std::optional catalog::get(const package_id& pk_id) const noexcept WHERE name = ? AND version = ? )"_sql); st.reset(); - st.bindings = std::forward_as_tuple(pk_id.name, pk_id.version.to_string()); + st.bindings = std::forward_as_tuple(pk_id.name, ver_str); auto opt_tup = sqlite3::unpack_single_opt catalog::get(const package_id& pk_id) const noexcept }, }; - auto append_transform = [](auto transform) { - return [transform = std::move(transform)](auto& remote) { - if constexpr (neo::alike) { - // Do nothing - } else { - remote.transforms.push_back(std::move(transform)); - } - }; - }; - if (!repo_transform.empty()) { - auto tr_json = json5::parse_data(repo_transform); - check_json(tr_json.is_array(), + // Transforms are stored in the DB as JSON strings. Convert them back to real objects. + auto tr_data = json5::parse_data(repo_transform); + check_json(tr_data.is_array(), fmt::format("Database record for {} has an invalid 'repo_transform' field [1]", pkg_id)); - for (const auto& el : tr_json.as_array()) { + for (const auto& el : tr_data.as_array()) { check_json( el.is_object(), fmt::format("Database record for {} has an invalid 'repo_transform' field [2]", pkg_id)); auto tr = fs_transformation::from_json(el); - std::visit(append_transform(tr), info.remote); + std::visit( + [&](auto& remote) { + if constexpr (neo::alike) { + // Do nothing + } else { + remote.transforms.push_back(std::move(tr)); + } + }, + info.remote); } } return info; @@ -377,6 +393,7 @@ std::vector catalog::by_name(std::string_view sv) const noexcept { } std::vector catalog::dependencies_of(const package_id& pkg) const noexcept { + dds_log(trace, "Lookup dependencies of {}@{}", pkg.name, pkg.version.to_string()); return sqlite3::exec_iter( // @@ -395,12 +412,16 @@ std::vector catalog::dependencies_of(const package_id& pkg) const no std::forward_as_tuple(pkg.name, pkg.version.to_string())) // | ranges::views::transform([](auto&& pair) { auto& [name, low, high] = pair; - return dependency{name, {semver::version::parse(low), semver::version::parse(high)}}; + auto dep + = dependency{name, {semver::version::parse(low), semver::version::parse(high)}}; + dds_log(trace, " Depends: {}", dep.to_string()); + return dep; }) // | ranges::to_vector; } void catalog::import_json_str(std::string_view content) { + dds_log(trace, "Importing JSON string into catalog"); auto pkgs = parse_packages_json(content); sqlite3::transaction_guard tr{_db}; @@ -411,6 +432,6 @@ void catalog::import_json_str(std::string_view content) { void catalog::import_initial() { sqlite3::transaction_guard tr{_db}; - log::info("Restoring built-in initial catalog contents"); + dds_log(info, "Restoring built-in initial catalog contents"); store_init_packages(_db, _stmt_cache); } diff --git a/src/dds/catalog/get.cpp b/src/dds/catalog/get.cpp index ccf063d3..804017ed 100644 --- a/src/dds/catalog/get.cpp +++ b/src/dds/catalog/get.cpp @@ -7,11 +7,6 @@ #include #include -#include -#include -#include -#include -#include #include #include @@ -32,32 +27,9 @@ temporary_sdist do_pull_sdist(const package_info& listing, std::monostate) { temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) { auto tmpdir = dds::temporary_dir::create(); - log::info("Cloning Git repository: {} [{}] ...", git.url, git.ref); - git.clone(tmpdir.path()); - - for (const auto& tr : git.transforms) { - tr.apply_to(tmpdir.path()); - } - - log::info("Create sdist from clone ..."); - if (git.auto_lib.has_value()) { - log::info("Generating library data automatically"); - - auto pkg_strm - = dds::open(tmpdir.path() / "package.json5", std::ios::binary | std::ios::out); - auto man_json = nlohmann::json::object(); - man_json["name"] = listing.ident.name; - man_json["version"] = listing.ident.version.to_string(); - man_json["namespace"] = git.auto_lib->namespace_; - pkg_strm << nlohmann::to_string(man_json); - - auto lib_strm - = dds::open(tmpdir.path() / "library.json5", std::ios::binary | std::ios::out); - auto lib_json = nlohmann::json::object(); - lib_json["name"] = git.auto_lib->name; - lib_strm << nlohmann::to_string(lib_json); - } + git.pull_to(listing.ident, tmpdir.path()); + dds_log(info, "Create sdist from clone ..."); sdist_params params; params.project_dir = tmpdir.path(); auto sd_tmp_dir = dds::temporary_dir::create(); @@ -99,7 +71,7 @@ void dds::get_all(const std::vector& pkgs, repository& repo, const c }); auto okay = parallel_run(absent_pkg_infos, 8, [&](package_info inf) { - log::info("Download package: {}", inf.ident.to_string()); + dds_log(info, "Download package: {}", inf.ident.to_string()); auto tsd = get_package_sdist(inf); std::scoped_lock lk{repo_mut}; repo.add_sdist(tsd.sdist, if_exists::throw_exc); diff --git a/src/dds/catalog/git.hpp b/src/dds/catalog/git.hpp deleted file mode 100644 index 3aedf92c..00000000 --- a/src/dds/catalog/git.hpp +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include - -namespace dds { - -struct git_remote_listing { - std::string url; - std::string ref; - std::optional auto_lib; - - void clone(path_ref path) const; -}; - -} // namespace dds diff --git a/src/dds/catalog/import.cpp b/src/dds/catalog/import.cpp index 81b10509..f6dcda29 100644 --- a/src/dds/catalog/import.cpp +++ b/src/dds/catalog/import.cpp @@ -1,6 +1,7 @@ #include "./import.hpp" #include +#include #include #include @@ -31,7 +32,7 @@ struct any_key { }; template -any_key(KF&&, Args&&...)->any_key; +any_key(KF&&, Args&&...) -> any_key; namespace { @@ -174,6 +175,7 @@ std::vector parse_json_v1(const json5::data& data) { std::vector dds::parse_packages_json(std::string_view content) { json5::data data; try { + dds_log(trace, "Parsing packages JSON data: {}", content); data = json5::parse_data(content); } catch (const json5::parse_error& e) { throw_user_error("JSON5 syntax error: {}", e.what()); @@ -194,6 +196,7 @@ std::vector dds::parse_packages_json(std::string_view content) { try { if (version == 1.0) { + dds_log(trace, "Processing JSON data as v1 data"); return parse_json_v1(data); } else { throw_user_error("Unknown catalog JSON version '{}'", diff --git a/src/dds/catalog/remote/git.cpp b/src/dds/catalog/remote/git.cpp index e4a66e09..baeb6664 100644 --- a/src/dds/catalog/remote/git.cpp +++ b/src/dds/catalog/remote/git.cpp @@ -2,10 +2,14 @@ #include #include +#include -void dds::git_remote_listing::clone(dds::path_ref dest) const { +#include + +void dds::git_remote_listing::pull_to(const dds::package_id& pid, dds::path_ref dest) const { fs::remove_all(dest); using namespace std::literals; + dds_log(info, "Clone Git repository [{}] (at {}) to [{}]", url, ref, dest.string()); auto command = {"git"s, "clone"s, "--depth=1"s, "--branch"s, ref, url, dest.generic_string()}; auto git_res = run_proc(command); if (!git_res.okay()) { @@ -15,4 +19,24 @@ void dds::git_remote_listing::clone(dds::path_ref dest) const { git_res.retc, git_res.output); } + + for (const auto& tr : transforms) { + tr.apply_to(dest); + } + + if (auto_lib.has_value()) { + dds_log(info, "Generating library data automatically"); + + auto pkg_strm = dds::open(dest / "package.json5", std::ios::binary | std::ios::out); + auto man_json = nlohmann::json::object(); + man_json["name"] = pid.name; + man_json["version"] = pid.version.to_string(); + man_json["namespace"] = auto_lib->namespace_; + pkg_strm << nlohmann::to_string(man_json); + + auto lib_strm = dds::open(dest / "library.json5", std::ios::binary | std::ios::out); + auto lib_json = nlohmann::json::object(); + lib_json["name"] = auto_lib->name; + lib_strm << nlohmann::to_string(lib_json); + } } diff --git a/src/dds/catalog/remote/git.hpp b/src/dds/catalog/remote/git.hpp index 8e022f90..172c0ad4 100644 --- a/src/dds/catalog/remote/git.hpp +++ b/src/dds/catalog/remote/git.hpp @@ -18,7 +18,7 @@ struct git_remote_listing { std::vector transforms; - void clone(path_ref path) const; + void pull_to(const package_id& pid, path_ref path) const; }; } // namespace dds diff --git a/src/dds/db/database.cpp b/src/dds/db/database.cpp index 01b5aa1e..b49e3e2d 100644 --- a/src/dds/db/database.cpp +++ b/src/dds/db/database.cpp @@ -87,7 +87,8 @@ database database::open(const std::string& db_path) { try { ensure_migrated(db); } catch (const sqlite3::sqlite3_error& e) { - log::error( + dds_log( + error, "Failed to load the databsae. It appears to be invalid/corrupted. We'll delete it and " "create a new one. The exception message is: {}", e.what()); @@ -96,10 +97,10 @@ database database::open(const std::string& db_path) { try { ensure_migrated(db); } catch (const sqlite3::sqlite3_error& e) { - log::critical( - "Failed to apply database migrations to recovery database. This is a critical " - "error. The exception message is: {}", - e.what()); + dds_log(critical, + "Failed to apply database migrations to recovery database. This is a critical " + "error. The exception message is: {}", + e.what()); std::terminate(); } } diff --git a/src/dds/library/root.cpp b/src/dds/library/root.cpp index e1f70ac5..ea3f9d27 100644 --- a/src/dds/library/root.cpp +++ b/src/dds/library/root.cpp @@ -28,7 +28,9 @@ auto collect_pf_sources(path_ref path) { // Drop any source files we found within `include/` erase_if(sources, [&](auto& info) { if (info.kind != source_kind::header) { - log::warn("Source file in `include` will not be compiled: {}", info.path.string()); + dds_log(warn, + "Source file in `include` will not be compiled: {}", + info.path.string()); return true; } return false; diff --git a/src/dds/package/manifest.cpp b/src/dds/package/manifest.cpp index 1b144042..dfb3a89e 100644 --- a/src/dds/package/manifest.cpp +++ b/src/dds/package/manifest.cpp @@ -63,10 +63,10 @@ package_manifest parse_json(const json5::data& data, std::string_view fpath) { if_key{"depends", [&](auto&& dat) { if (dat.is_object()) { - log::warn( - "{}: Using a JSON object for 'depends' is deprecated. Use an " - "array of strings instead.", - fpath); + dds_log(warn, + "{}: Using a JSON object for 'depends' is deprecated. Use an " + "array of strings instead.", + fpath); return mapping{push_depends_obj_kv}(dat); } else if (dat.is_array()) { return for_each{put_into{std::back_inserter(ret.dependencies), diff --git a/src/dds/proc.nix.cpp b/src/dds/proc.nix.cpp index 144b21d8..c2861072 100644 --- a/src/dds/proc.nix.cpp +++ b/src/dds/proc.nix.cpp @@ -62,7 +62,7 @@ spawn_child(const std::vector& command, int stdout_pipe, int close_ } // namespace proc_result dds::run_proc(const proc_options& opts) { - log::debug("Spawning subprocess: {}", quote_command(opts.command)); + dds_log(debug, "Spawning subprocess: {}", quote_command(opts.command)); int stdio_pipe[2] = {}; auto rc = ::pipe(stdio_pipe); check_rc(rc == 0, "Create stdio pipe for subprocess"); @@ -100,7 +100,7 @@ proc_result dds::run_proc(const proc_options& opts) { ::kill(child, SIGINT); timeout = -1ms; res.timed_out = true; - log::debug("Subprocess [{}] timed out", quote_command(opts.command)); + dds_log(debug, "Subprocess [{}] timed out", quote_command(opts.command)); continue; } std::string buffer; diff --git a/src/dds/proc.win.cpp b/src/dds/proc.win.cpp index 066be53c..951d45ad 100644 --- a/src/dds/proc.win.cpp +++ b/src/dds/proc.win.cpp @@ -25,6 +25,7 @@ namespace { proc_result dds::run_proc(const proc_options& opts) { auto cmd_str = quote_command(opts.command); + dds_log(debug, "Spawning subprocess: {}", cmd_str); ::SECURITY_ATTRIBUTES security = {}; security.bInheritHandle = TRUE; diff --git a/src/dds/repo/repo.cpp b/src/dds/repo/repo.cpp index 177c2cda..128a6e25 100644 --- a/src/dds/repo/repo.cpp +++ b/src/dds/repo/repo.cpp @@ -32,9 +32,10 @@ auto load_sdists(path_ref root) { try { return sdist::from_directory(p); } catch (const std::runtime_error& e) { - log::error("Failed to load source distribution from directory '{}': {}", - p.string(), - e.what()); + dds_log(error, + "Failed to load source distribution from directory '{}': {}", + p.string(), + e.what()); return std::nullopt; } }; @@ -53,8 +54,8 @@ auto load_sdists(path_ref root) { } // namespace void repository::_log_blocking(path_ref dirpath) noexcept { - log::warn("Another process has the repository directory locked [{}]", dirpath.string()); - log::warn("Waiting for repository to be released..."); + dds_log(warn, "Another process has the repository directory locked [{}]", dirpath.string()); + dds_log(warn, "Waiting for repository to be released..."); } void repository::_init_repo_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); } @@ -68,7 +69,8 @@ repository repository::_open_for_directory(bool writeable, path_ref dirpath) { void repository::add_sdist(const sdist& sd, if_exists ife_action) { if (!_write_enabled) { - log::critical( + dds_log( + critical, "DDS attempted to write into a repository that wasn't opened with a write-lock. This " "is a hard bug and should be reported. For the safety and integrity of the local " "repository, we'll hard-exit immediately."); @@ -81,10 +83,10 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) { if (ife_action == if_exists::throw_exc) { throw_user_error(msg); } else if (ife_action == if_exists::ignore) { - log::warn(msg); + dds_log(warn, msg); return; } else { - log::info(msg + " - Replacing"); + dds_log(info, msg + " - Replacing"); } } auto tmp_copy = sd_dest; @@ -99,7 +101,7 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) { } fs::rename(tmp_copy, sd_dest); _sdists.insert(sdist::from_directory(sd_dest)); - log::info("Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); + dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); } const sdist* repository::find(const package_id& pkg) const noexcept { diff --git a/src/dds/solve/solve.cpp b/src/dds/solve/solve.cpp index ffc77df3..ec34f0ad 100644 --- a/src/dds/solve/solve.cpp +++ b/src/dds/solve/solve.cpp @@ -76,21 +76,31 @@ struct solver_provider { mutable std::map> pkgs_by_name = {}; std::optional best_candidate(const req_type& req) const { + dds_log(debug, "Find best candidate of {}", req.dep.to_string()); + // Look up in the cachce for the packages we have with the given name auto found = pkgs_by_name.find(req.dep.name); if (found == pkgs_by_name.end()) { + // If it isn't there, insert an entry in the cache found = pkgs_by_name.emplace(req.dep.name, pkgs_for_name(req.dep.name)).first; } - auto& vec = found->second; - auto cand = std::find_if(vec.cbegin(), vec.cend(), [&](const package_id& pk) { + // Find the first package with the version contained by the ranges in the requirement + auto& for_name = found->second; + auto cand = std::find_if(for_name.cbegin(), for_name.cend(), [&](const package_id& pk) { return req.dep.versions.contains(pk.version); }); - if (cand == vec.cend()) { + if (cand == for_name.cend()) { + dds_log(debug, "No candidate for requirement {}", req.dep.to_string()); return std::nullopt; } + dds_log(debug, "Select candidate {}@{}", cand->to_string()); return req_type{dependency{cand->name, {cand->version, cand->version.next_after()}}}; } std::vector requirements_of(const req_type& req) const { + dds_log(trace, + "Lookup requirements of {}@{}", + req.key(), + (*req.dep.versions.iter_intervals().begin()).low.to_string()); auto pk_id = as_pkg_id(req); auto deps = deps_for_pkg(pk_id); return deps // @@ -129,7 +139,7 @@ struct explainer { void operator()(pubgrub::explain::premise pr) { strm.str(""); put(pr.value); - log::error("{} {},", at_head ? "┌─ Given that" : "│ and", strm.str()); + dds_log(error, "{} {},", at_head ? "┌─ Given that" : "│ and", strm.str()); at_head = false; } @@ -138,10 +148,10 @@ struct explainer { at_head = true; strm.str(""); put(cncl.value); - log::error("╘═ then {}.", strm.str()); + dds_log(error, "╘═ then {}.", strm.str()); } - void operator()(pubgrub::explain::separator) { log::error(""); } + void operator()(pubgrub::explain::separator) { dds_log(error, ""); } }; } // namespace @@ -156,7 +166,7 @@ std::vector dds::solve(const std::vector& deps, auto solution = pubgrub::solve(wrap_req, solver_provider{pkgs_prov, deps_prov}); return solution | ranges::views::transform(as_pkg_id) | ranges::to_vector; } catch (const solve_fail_exc& failure) { - log::error("Dependency resolution has failed! Explanation:"); + dds_log(error, "Dependency resolution has failed! Explanation:"); pubgrub::generate_explaination(failure, explainer()); throw_user_error(); } diff --git a/src/dds/source/dist.cpp b/src/dds/source/dist.cpp index 7ebe41bd..fb2b8622 100644 --- a/src/dds/source/dist.cpp +++ b/src/dds/source/dist.cpp @@ -18,7 +18,7 @@ namespace { void sdist_export_file(path_ref out_root, path_ref in_root, path_ref filepath) { auto relpath = fs::relative(filepath, in_root); - log::debug("Export file {}", relpath.string()); + dds_log(debug, "Export file {}", relpath.string()); auto dest = out_root / relpath; fs::create_directories(dest.parent_path()); fs::copy(filepath, dest); @@ -52,7 +52,7 @@ void sdist_copy_library(path_ref out_root, const library_root& lib, const sdist_ } sdist_export_file(out_root, params.project_dir, *lib_man_path); - log::info("sdist: Export library from {}", lib.path().string()); + dds_log(info, "sdist: Export library from {}", lib.path().string()); fs::create_directories(out_root); for (const auto& source : sources_to_keep) { sdist_export_file(out_root, params.project_dir, source.path); @@ -77,7 +77,7 @@ sdist dds::create_sdist(const sdist_params& params) { } fs::create_directories(dest.parent_path()); safe_rename(tempdir.path(), dest); - log::info("Source distribution created in {}", dest.string()); + dds_log(info, "Source distribution created in {}", dest.string()); return sdist::from_directory(dest); } @@ -98,7 +98,7 @@ sdist dds::create_sdist_in_dir(path_ref out, const sdist_params& params) { auto pkg_man = package_manifest::load_from_file(*man_path); sdist_export_file(out, params.project_dir, *man_path); - log::info("Generated export as {}", pkg_man.pkg_id.to_string()); + dds_log(info, "Generated export as {}", pkg_man.pkg_id.to_string()); return sdist::from_directory(out); } diff --git a/src/dds/toolchain/toolchain.cpp b/src/dds/toolchain/toolchain.cpp index 1176672e..d4d205a8 100644 --- a/src/dds/toolchain/toolchain.cpp +++ b/src/dds/toolchain/toolchain.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -72,10 +73,15 @@ static auto shortest_path_args(path_ref base, R&& r) { } compile_command_info toolchain::create_compile_command(const compile_file_spec& spec, - path_ref, + path_ref cwd, toolchain_knobs knobs) const noexcept { using namespace std::literals; + dds_log(trace, + "Calculate compile command for source file [{}] to object file [{}]", + spec.source_path.string(), + spec.out_path.string()); + language lang = spec.lang; if (lang == language::automatic) { if (spec.source_path.extension() == ".c" || spec.source_path.extension() == ".C") { @@ -87,15 +93,20 @@ compile_command_info toolchain::create_compile_command(const compile_file_spec& vector flags; if (knobs.is_tty) { + dds_log(trace, "Enabling TTY flags."); extend(flags, _tty_flags); } + dds_log(trace, "#include-search dirs:"); for (auto&& inc_dir : spec.include_dirs) { - auto inc_args = include_args(inc_dir); + dds_log(trace, " - search: {}", inc_dir.string()); + auto shortest = shortest_path_from(inc_dir, cwd); + auto inc_args = include_args(shortest); extend(flags, inc_args); } for (auto&& ext_inc_dir : spec.external_include_dirs) { + dds_log(trace, " - search (external): {}", ext_inc_dir.string()); auto inc_args = external_include_args(ext_inc_dir); extend(flags, inc_args); } @@ -142,10 +153,14 @@ vector toolchain::create_archive_command(const archive_spec& spec, path_ref cwd, toolchain_knobs) const noexcept { vector cmd; - + dds_log(trace, "Creating archive command [output: {}]", spec.out_path.string()); auto out_arg = shortest_path_from(spec.out_path, cwd).string(); for (auto& arg : _link_archive) { if (arg == "[in]") { + dds_log(trace, "Expand [in] placeholder:"); + for (auto&& in : spec.input_files) { + dds_log(trace, " - input: [{}]", in.string()); + } extend(cmd, shortest_path_args(cwd, spec.input_files)); } else { cmd.push_back(replace(arg, "[out]", out_arg)); @@ -158,8 +173,13 @@ vector toolchain::create_link_executable_command(const link_exe_spec& sp path_ref cwd, toolchain_knobs) const noexcept { vector cmd; + dds_log(trace, "Creating link command [output: {}]", spec.output.string()); for (auto& arg : _link_exe) { if (arg == "[in]") { + dds_log(trace, "Expand [in] placeholder:"); + for (auto&& in : spec.inputs) { + dds_log(trace, " - input: [{}]", in.string()); + } extend(cmd, shortest_path_args(cwd, spec.inputs)); } else { cmd.push_back(replace(arg, "[out]", shortest_path_from(spec.output, cwd).string())); @@ -280,7 +300,9 @@ std::optional dds::toolchain::get_default() { user_home_dir() / "toolchain.json", }; for (auto&& cand : candidates) { + dds_log(trace, "Checking for default toolchain at [{}]", cand.string()); if (fs::exists(cand)) { + dds_log(debug, "Using default toolchain file: {}", cand.string()); return parse_toolchain_json5(slurp_file(cand)); } } diff --git a/src/dds/util/log.hpp b/src/dds/util/log.hpp index 71410d71..5d3a7a08 100644 --- a/src/dds/util/log.hpp +++ b/src/dds/util/log.hpp @@ -19,12 +19,13 @@ inline level current_log_level = level::info; void log_print(level l, std::string_view s) noexcept; -// clang-format off template -concept formattable = requires (const T item) { +concept formattable = requires(const T item) { fmt::format("{}", item); }; +inline bool level_enabled(level l) { return int(l) >= int(current_log_level); } + template void log(level l, std::string_view s, const Args&... args) noexcept { if (int(l) >= int(current_log_level)) { @@ -38,31 +39,11 @@ void trace(std::string_view s, const Args&... args) { log(level::trace, s, args...); } -template -void debug(std::string_view s, const Args&... args) { - log(level::debug, s, args...); -} - -template -void info(std::string_view s, const Args&... args) { - log(level::info, s, args...); -} - -template -void warn(std::string_view s, const Args&... args) { - log(level::warn, s, args...); -} - -template -void error(std::string_view s, const Args&... args) { - log(level::error, s, args...); -} - -template -void critical(std::string_view s, const Args&&... args) { - log(level::critical, s, args...); -} - -// clang-format on +#define dds_log(Level, str, ...) \ + do { \ + if (int(dds::log::level::Level) >= int(dds::log::current_log_level)) { \ + ::dds::log::log(::dds::log::level::Level, str __VA_OPT__(, ) __VA_ARGS__); \ + } \ + } while (0) } // namespace dds::log \ No newline at end of file diff --git a/src/dds/util/parallel.cpp b/src/dds/util/parallel.cpp index db8986f2..cd2ede5e 100644 --- a/src/dds/util/parallel.cpp +++ b/src/dds/util/parallel.cpp @@ -8,6 +8,6 @@ void dds::log_exception(std::exception_ptr eptr) noexcept { try { std::rethrow_exception(eptr); } catch (const std::exception& e) { - log::error(e.what()); + dds_log(error, e.what()); } } diff --git a/src/dds/util/paths.linux_fbsd.cpp b/src/dds/util/paths.linux_fbsd.cpp index 7ebfe41d..64856e7a 100644 --- a/src/dds/util/paths.linux_fbsd.cpp +++ b/src/dds/util/paths.linux_fbsd.cpp @@ -12,7 +12,7 @@ fs::path dds::user_home_dir() { static auto ret = []() -> fs::path { auto home_env = std::getenv("HOME"); if (!home_env) { - log::error("No HOME environment variable set!"); + dds_log(error, "No HOME environment variable set!"); return "/"; } return fs::absolute(fs::path(home_env)); diff --git a/src/dds/util/paths.macos.cpp b/src/dds/util/paths.macos.cpp index e81613f2..9c942c3d 100644 --- a/src/dds/util/paths.macos.cpp +++ b/src/dds/util/paths.macos.cpp @@ -12,7 +12,7 @@ fs::path dds::user_home_dir() { static auto ret = []() -> fs::path { auto home_env = std::getenv("HOME"); if (!home_env) { - log::warn("No HOME environment variable set!"); + dds_log(warn, "No HOME environment variable set!"); return "/"; } return fs::absolute(fs::path(home_env)); diff --git a/src/dds/util/paths.win.cpp b/src/dds/util/paths.win.cpp index 0c91b6c6..769cfe11 100644 --- a/src/dds/util/paths.win.cpp +++ b/src/dds/util/paths.win.cpp @@ -13,7 +13,7 @@ fs::path dds::user_home_dir() { static auto ret = []() -> fs::path { auto userprofile_env = std::getenv("USERPROFILE"); if (!userprofile_env) { - log::warn("No USERPROFILE environment variable set!"); + dds_log(warn, "No USERPROFILE environment variable set!"); return "/"; } return fs::absolute(fs::path(userprofile_env)); From f989fdec0c40b712f1bf4f210c19c80ea212ee4d Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Sun, 19 Jul 2020 16:51:21 -0600 Subject: [PATCH 13/36] Convert docs to emit HTML templates for Hugo to import --- docs/conf.py | 4 ++-- docs/index.rst | 7 ------- docs/layout.html | 30 ++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 docs/layout.html diff --git a/docs/conf.py b/docs/conf.py index 9f79683f..099291a0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ # -- General configuration --------------------------------------------------- extensions = [] -templates_path = ['_templates'] +templates_path = ['.'] source_suffix = '.rst' master_doc = 'index' language = None @@ -21,7 +21,7 @@ pygments_style = None # -- Options for HTML output ------------------------------------------------- -html_theme = 'nature' +html_theme = 'basic' html_theme_options = {} html_static_path = [] html_sidebars = {} diff --git a/docs/index.rst b/docs/index.rst index a67ccfd0..840d3c42 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,10 +19,3 @@ the :doc:`tut/index` page. design dev/index err/index - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/layout.html b/docs/layout.html new file mode 100644 index 00000000..5e9d4979 --- /dev/null +++ b/docs/layout.html @@ -0,0 +1,30 @@ +--- +title: "{{ title | striptags }}" +layout: single +{% if not pagename.endswith('index') -%} +url: docs/{{pagename}}{{file_suffix}} +{%- endif %} +--- + +
+ +
+ +
+ +
+ +
+ {{ body }} + {% if next %} +
+ Next: {{ next.title }} +
+ {% endif %} +
\ No newline at end of file From db5900a3a6d628916b6b116355e16186f3404281 Mon Sep 17 00:00:00 2001 From: vector-of-bool Date: Sun, 19 Jul 2020 21:23:52 -0600 Subject: [PATCH 14/36] Hugo docs are generated differently --- Makefile | 3 +++ docs/conf.py | 10 ++++++++-- docs/layout.html | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d92e437b..9b05e367 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,9 @@ docs: -Wqanj8 echo "Docs generated to _build/docs" +hugo-docs: + env GEN_FOR_HUGO=1 $(MAKE) docs + docs-server: docs echo "Docs are visible on http://localhost:9794/" cd _build/docs && \ diff --git a/docs/conf.py b/docs/conf.py index 099291a0..d519d6f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # Refer: http://www.sphinx-doc.org/en/master/config +import os + # -- Project information ----------------------------------------------------- project = 'dds' copyright = '2020, vector-of-bool' @@ -13,7 +15,7 @@ # -- General configuration --------------------------------------------------- extensions = [] -templates_path = ['.'] +templates_path = [] source_suffix = '.rst' master_doc = 'index' language = None @@ -21,7 +23,11 @@ pygments_style = None # -- Options for HTML output ------------------------------------------------- -html_theme = 'basic' +html_theme = 'nature' html_theme_options = {} html_static_path = [] html_sidebars = {} + +if os.environ.get('GEN_FOR_HUGO'): + templates_path.append('.') + html_theme = 'basic' diff --git a/docs/layout.html b/docs/layout.html index 5e9d4979..861c5315 100644 --- a/docs/layout.html +++ b/docs/layout.html @@ -6,14 +6,14 @@ {%- endif %} --- -
+
-
+