From 57aa085fc8f8bd9cc30c3614752ff267a3c1e25b Mon Sep 17 00:00:00 2001 From: Chuck Walbourn Date: Sat, 5 Oct 2024 16:20:15 -0700 Subject: [PATCH] Always run SDL even if no changes --- .onefuzz | 0 CMakeLists.txt | 17 +++ CMakePresets.json | 9 ++ build/DirectXMesh-OneFuzz-Coverage.yml | 92 ++++++++++++++++ build/DirectXMesh-OneFuzz.yml | 146 +++++++++++++++++++++++++ build/DirectXMesh-SDL.yml | 1 + build/OneFuzzConfig.json | 49 +++++++++ build/onefuzz-setup.ps1 | 92 ++++++++++++++++ 8 files changed, 406 insertions(+) create mode 100644 .onefuzz create mode 100644 build/DirectXMesh-OneFuzz-Coverage.yml create mode 100644 build/DirectXMesh-OneFuzz.yml create mode 100644 build/OneFuzzConfig.json create mode 100644 build/onefuzz-setup.ps1 diff --git a/.onefuzz b/.onefuzz new file mode 100644 index 0000000..e69de29 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5134555..838fc82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,8 @@ option(DISABLE_MSVC_ITERATOR_DEBUGGING "Disable iterator debugging in Debug conf option(ENABLE_CODE_ANALYSIS "Use Static Code Analysis on build" OFF) +option(BUILD_FUZZING "Build for fuzz testing" OFF) + set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -275,6 +277,18 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") foreach(t IN LISTS TOOL_EXES) target_compile_options(${t} PRIVATE ${WarningsEXE}) endforeach() + + if(BUILD_FUZZING AND (NOT WINDOWS_STORE)) + string(REPLACE "/DNDEBUG" "" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + string(REPLACE "/DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) + + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 19.32) + foreach(t IN LISTS TOOL_EXES ITEMS ${PROJECT_NAME}) + target_compile_options(${t} PRIVATE ${ASAN_SWITCHES}) + target_link_libraries(${t} PRIVATE ${ASAN_LIBS}) + endforeach() + endif() + endif() endif() if(WIN32) @@ -315,5 +329,8 @@ if(WIN32 AND (NOT WINDOWS_STORE) AND (NOT (DEFINED XBOX_CONSOLE_TARGET))) if(BUILD_TESTING AND (EXISTS "${CMAKE_CURRENT_LIST_DIR}/Tests/CMakeLists.txt")) enable_testing() add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/Tests) + elseif(BUILD_FUZZING AND (EXISTS "${CMAKE_CURRENT_LIST_DIR}/Tests/fuzzloaders/CMakeLists.txt")) + message(STATUS "Building for fuzzing") + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/Tests/fuzzloaders) endif() endif() diff --git a/CMakePresets.json b/CMakePresets.json index e3c0456..e7b9681 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -182,6 +182,14 @@ "strategy": "external" } }, + { + "name": "Fuzzing", + "cacheVariables": { + "BUILD_FUZZING": true, + "BUILD_TESTING": false + }, + "hidden": true + }, { "name": "x64-Debug" , "description": "MSVC for x64 (Debug) with DX12", "inherits": [ "base", "x64", "Debug", "MSVC" ] }, { "name": "x64-Release" , "description": "MSVC for x64 (Release) with DX12", "inherits": [ "base", "x64", "Release", "MSVC" ] }, @@ -267,6 +275,7 @@ { "name": "arm64-Debug-Linux", "description": "WSL Linux ARM64 (Debug)", "inherits": [ "base", "ARM64", "Debug", "VCPKG" ] }, { "name": "arm64-Release-Linux", "description": "WSL Linux ARM64 (Release)", "inherits": [ "base", "ARM64", "Release", "VCPKG" ] }, + { "name": "x64-Fuzzing" , "description": "MSVC for x64 (Release) with ASan", "inherits": [ "base", "x64", "Release", "MSVC", "Fuzzing" ] }, { "name": "x64-Analyze" , "description": "MSVC for x64 (Debug) with DX12 using /analyze", "inherits": [ "base", "x64", "Debug", "MSVC" ], "cacheVariables": { "ENABLE_CODE_ANALYSIS": true } } ], "testPresets": [ diff --git a/build/DirectXMesh-OneFuzz-Coverage.yml b/build/DirectXMesh-OneFuzz-Coverage.yml new file mode 100644 index 0000000..8e25ea7 --- /dev/null +++ b/build/DirectXMesh-OneFuzz-Coverage.yml @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# https://go.microsoft.com/fwlink/?LinkID=324981 + +# OneFuzz code coverage pipeline + +pr: none +trigger: none + +resources: + repositories: + - repository: self + type: git + ref: refs/heads/main + - repository: testRepo + name: walbourn/directxmeshtest + type: github + endpoint: microsoft + ref: refs/heads/main + +pool: + vmImage: windows-latest + +parameters: + - name: sasUrl + type: string + displayName: SAS URL + - name: branch + type: string + displayName: Branch + - name: jobID + type: string + displayName: OneFuzz Job ID + - name: buildDate + type: string + displayName: Build Date + - name: commitID + type: string + displayName: Commit ID + +variables: + - name: coverage-file + value: cobertura-coverage.xml + - name: job-ID + value: ${{ parameters.jobID }} + - name: build-date + value: ${{ parameters.buildDate }} + - name: branch + value: ${{ parameters.branch }} + - name: sas-url + value: ${{ parameters.sasUrl }} + - name: commit-ID + value: ${{ parameters.commitID }} + +jobs: +- job: prod + displayName: Prod Task + workspace: + clean: all + steps: + - checkout: self + clean: true + fetchTags: false + fetchDepth: 1 + path: 's' + - checkout: testRepo + displayName: Fetch Tests + clean: true + fetchTags: false + fetchDepth: 1 + path: 's/Tests' + - powershell: | + Write-Host "Job ID: $(job-ID), Build Date: $(build-date), Branch: $(branch)" + $SASUrl = [System.Uri]::new("$(sas-url)") + azcopy cp $SASUrl.AbsoluteUri ./ --recursive + $ContainerName = $SASURL.LocalPath.Split("/")[1] + Write-Host "##vso[task.setvariable variable=container-name;]$ContainerName" + cd $ContainerName + $size = ((Get-Item .\$(coverage-file)).length) + if ($size -eq 0) { + Write-Host "Cobertura coverage XML is empty." + exit 1 + } + displayName: 'Get code coverage from OneFuzz' + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: 'Cobertura' + summaryFileLocation: ./$(container-name)\$(coverage-file) + pathToSources: $(Build.SourcesDirectory) + displayName: 'Generate coverage report' diff --git a/build/DirectXMesh-OneFuzz.yml b/build/DirectXMesh-OneFuzz.yml new file mode 100644 index 0000000..cf47515 --- /dev/null +++ b/build/DirectXMesh-OneFuzz.yml @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# https://go.microsoft.com/fwlink/?LinkID=324981 + +# Builds the library using CMake and submit for file fuzzing + +schedules: +- cron: "0 12 1 * *" + displayName: 'Submit for File Fuzzing (Monthly)' + branches: + include: + - main + always: true + +trigger: none +pr: none + +resources: + repositories: + - repository: self + type: git + ref: refs/heads/main + - repository: testRepo + name: walbourn/directxmeshtest + type: github + endpoint: microsoft + ref: refs/heads/main + +name: $(Year:yyyy).$(Month).$(DayOfMonth)$(Rev:.r) + +variables: + Codeql.Enabled: false + VS_GENERATOR: 'Visual Studio 17 2022' + WIN11_SDK: '10.0.22000.0' + +pool: + vmImage: windows-2022 + +jobs: +- job: FUZZ_BUILD + displayName: 'Build for file fuzzing' + steps: + - checkout: self + clean: true + fetchTags: false + fetchDepth: 1 + path: 's' + - checkout: testRepo + displayName: Fetch Tests + clean: true + fetchTags: false + fetchDepth: 1 + path: 's/Tests' + - task: CMake@1 + displayName: 'CMake (MSVC): Config with ASan' + inputs: + cwd: '$(Build.SourcesDirectory)' + cmakeArgs: '-G "$(VS_GENERATOR)" -A x64 -B out -DCMAKE_SYSTEM_VERSION=$(WIN11_SDK) -DBUILD_TOOLS=OFF -DBUILD_FUZZING=ON -DBUILD_TESTING=OFF' + - task: CMake@1 + displayName: 'CMake (MSVC): Build with ASan' + inputs: + cwd: '$(Build.SourcesDirectory)' + cmakeArgs: --build out -v --config RelWithDebInfo + - task: CopyFiles@2 + displayName: Copy fuzzer + inputs: + Contents: | + build\OneFuzzConfig.json + out\bin\RelWithDebInfo\fuzzloaders.exe + TargetFolder: .drop + OverWrite: true + flattenFolders: true + - task: CopyFiles@2 + displayName: Copy symbols + inputs: + Contents: | + out\bin\RelWithDebInfo\fuzzloaders.pdb + TargetFolder: .drop\symbols + OverWrite: true + flattenFolders: true + - task: PowerShell@2 + displayName: Download seed files + inputs: + targetType: inline + script: | + $seedfiles = "AlphaEdge.dds", + "cubea8r8g8b8.dds", + "default_texture_nm.dds", + "dx5_logo.dds", + "hdrtest.dds", + "normalmap.dds", + "grad4d.hdr", + "grad4dunc.hdr", + "BigTree.hdr", + "CBW8.TGA", + "ccm8.tga", + "CTC16.TGA", + "CTC24.TGA", + "CTC32.TGA", + "UTC16.TGA", + "UTC24.TGA", + "UTC32.TGA", + "UBW8.TGA", + "ucm8.tga", + "testimg.ppm", + "grad4d.pfm"; + + New-Item -ItemType Directory -Force -Path .drop\seeds\ + + foreach($filename in $seedfiles) + { + Write-Host "Fetching: $filename" + $url = "https://raw.githubusercontent.com/walbourn/directxmeshmedia/main/" + $filename + $target = [System.IO.Path]::Combine(".drop\seeds\", $filename) + Invoke-WebRequest -Uri $url -OutFile $target + } + + - task: PowerShell@2 + displayName: Copy OneFuzz setup script + inputs: + targetType: 'inline' + script: | + Copy-Item -Path .\build\onefuzz-setup.ps1 -Destination .drop/setup.ps1 + + - task: MSBuild@1 + displayName: 'Copy ASan binaries' + inputs: + solution: build/CopyASAN.targets + msbuildArguments: /p:TargetFolder=$(Build.SourcesDirectory)\.drop + msbuildVersion: 17.0 + msbuildArchitecture: x64 + - task: PowerShell@2 + displayName: List drop files + inputs: + targetType: inline + script: | + Get-ChildItem ".drop" -Recurse | select FullName + + - task: onefuzz-task@0 + displayName: 'Submit to OneFuzz' + inputs: + onefuzzOSes: 'Windows' + env: + onefuzzDropDirectory: $(Build.SourcesDirectory)\.drop + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/build/DirectXMesh-SDL.yml b/build/DirectXMesh-SDL.yml index c637d77..691e364 100644 --- a/build/DirectXMesh-SDL.yml +++ b/build/DirectXMesh-SDL.yml @@ -11,6 +11,7 @@ schedules: branches: include: - main + always: true # GitHub Actions handles CodeQL and PREFAST for CI/PR trigger: none diff --git a/build/OneFuzzConfig.json b/build/OneFuzzConfig.json new file mode 100644 index 0000000..fa2e930 --- /dev/null +++ b/build/OneFuzzConfig.json @@ -0,0 +1,49 @@ +{ + "ConfigVersion": 3, + "Entries": [ + { + "MinAvailableMemoryMB": 50, + "JobNotificationEmail": "directxtkdev@microsoft.com", + "Skip": false, + "Fuzzer": { + "$type": "libfuzzer", + "FuzzingHarnessExecutableName": "fuzzloaders.exe", + "FuzzingTargetBinaries": [ + "fuzzloaders.exe" + ] + }, + "RebootAfterSetup": false, + "OneFuzzJobs": [ + { + "ProjectName": "Direct3D", + "TargetName": "DirectXMesh", + "TargetOptions": [ + " -rss_limit_mb=4096" + ], + "TargetEnv": { + "ASAN_OPTIONS": "allocator_may_return_null=1" + } + } + ], + "JobDependencies": [ + "fuzzloaders.exe", + "fuzzloaders.pdb", + "clang_rt.asan_dynamic-x86_64.dll", + "msdia140.dll", + "setup.ps1" + ], + "AdoTemplate": { + "Org": "microsoft", + "Project": "OS", + "AssignedTo": "chuckw@microsoft.com", + "AreaPath": "OS\\Core\\SiGMa\\GRFX-Graphics", + "IterationPath": "OS\\Future" + }, + "CodeCoverage": { + "Org": "mscodehub", + "Project": "DirectXMesh", + "PipelineId": "1" + } + } + ] +} \ No newline at end of file diff --git a/build/onefuzz-setup.ps1 b/build/onefuzz-setup.ps1 new file mode 100644 index 0000000..6e68c53 --- /dev/null +++ b/build/onefuzz-setup.ps1 @@ -0,0 +1,92 @@ +<# +Copyright (c) Microsoft Corporation. +Licensed under the MIT License. +#> + +function Execute-Setup { + # Temporary work-around while OneFuzz does not run script from setup dir + Set-Location -Path $PSScriptRoot + + Write-Log "Executing custom setup script in $(pwd)" + + # Exclude any uploaded DLL from known DLLs + gci -filter '*.dll' | Exclude-Library + + # Done. Useful to know that the script did not prematurely error out + Write-Log 'Setup script finished successfully' +} + +# Write log statements into the event log. +function Write-Log { + Param( + [Parameter(Position=0, + Mandatory, + ValueFromPipeline, + ValueFromRemainingArguments)] + [String[]] $Messages, + [Parameter()] [Int] $EventId=0) + Begin { + $EventSource = 'onefuzz-setup.ps1' + $EventLog = 'Application' + if (![System.Diagnostics.EventLog]::SourceExists($EventSource)) { + New-EventLog -LogName $EventLog -Source $EventSource + } + } + Process { + $Messages.ForEach({ + Write-EventLog -LogName $EventLog -Source $EventSource -EventId $EventId -EntryType Information -Message $_ + }) + } + End {} +} + +# This function is used to exclude DLL's that the fuzzer is dependent on. The dependent DLL's +# have been built with ASan and copied into the setup directory along with the fuzzer. +function Exclude-Library { + Param( + [Parameter(Position=0, + Mandatory, + ValueFromPipeline, + ValueFromRemainingArguments)] + [String[]] $Libraries) + Begin { + $Path = 'HKLM:System\CurrentControlSet\Control\Session Manager' + $Name = 'ExcludeFromKnownDLLs' + } + Process { + # Get existing excluded libraries + $ExistingProperty = Get-ItemProperty -Path $Path -Name $Name -ErrorAction SilentlyContinue + $ExistingExclusions = @() + + # Normalize DLL name to lowercase for comparison + if ($ExistingProperty -ne $null) { + $ExistingExclusions = $ExistingProperty.$Name.ForEach("ToLower") + } + + # Normalize DLL name to lowercase for comparison, remove duplicates + $Libs = $Libraries.ForEach("ToLower") | Select-Object -Unique + Write-Log "Excluding libraries $Libs" + + # Discard empty strings and libraries already excluded + $Libs = $Libs.Where({$_.Length -gt 0 -and !($ExistingExclusions.Contains($_))}) + + # If anything remains either add or update registry key + if ($Libs.Length -gt 0) { + if ($ExistingProperty -eq $null) { + # Create registry key to exclude DLLs + New-ItemProperty -Path $Path -Name $Name -PropertyType MultiString -Value $Libs + Write-Log "Created known DLLs exclusions with $Libs" + } else { + # Update registry key to exclude DLLs + Set-ItemProperty -Path $Path -Name $Name -Value ($ExistingProperty.$Name + $Libs) + Write-Log "Updated known DLLs exclusions with $Libs" + } + } else { + # DLLs already excluded + Write-Log "Known DLL exclusions already exist for $Libraries" + } + } + End {} +} + +Execute-Setup