chore(ci): collect coverage data (#8951)

This PR makes sure we run the tests with coverage collection when the
"itr:noskip" keyword is added to the commit message, or if the branch is
not a PR (main, 2.x, etc).

Also, it removes the usage of the coverage parameter in the yaml config,
simply uploading the coverage files if they exist.

## Checklist

- [x] Change(s) are motivated and described in the PR description
- [x] Testing strategy is described if automated tests are not included
in the PR
- [x] Risks are described (performance impact, potential for breakage,
maintainability)
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed or label `changelog/no-changelog` is set
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/))
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
- [x] If this PR changes the public interface, I've notified
`@DataDog/apm-tees`.

## Reviewer Checklist

- [x] Title is accurate
- [x] All changes are related to the pull request's stated goal
- [x] Description motivates each change
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- [x] Testing strategy adequately addresses listed risks
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] Release note makes sense to a user of the library
- [x] Author has acknowledged and discussed the performance implications
of this PR as reported in the benchmarks PR comment
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

.circleci/config.templ.yml

@@ -63,6 +63,7 @@ commands:
       - run: |
           set -ex
           mkdir coverage
+          touch ./coverage/empty.$CIRCLE_BUILD_NUM-$CIRCLE_JOB-$CIRCLE_NODE_INDEX.coverage
           if [ -f .coverage ];
           then
             cp .coverage ./coverage/$CIRCLE_BUILD_NUM-$CIRCLE_JOB-$CIRCLE_NODE_INDEX.coverage
@@ -151,7 +152,7 @@ commands:
             - run:
                 command: |
                   mkdir -p /tmp/core_dumps
-                  cp core.* /tmp/core_dumps
+                  cp core.* /tmp/core_dumps || true
                   ./scripts/bt
                 when: on_fail
             - store_artifacts:
@@ -186,17 +187,15 @@ commands:
             - run:
                 command: |
                   mkdir -p /tmp/core_dumps
-                  cp core.* /tmp/core_dumps
+                  cp core.* /tmp/core_dumps || true
                   ./scripts/bt
                 when: on_fail
             - store_artifacts:
                 name: "Store core dumps"
                 path: /tmp/core_dumps
       - when:
           condition:
-            and:
-              - << pipeline.parameters.coverage >>
-              - << parameters.store_coverage >>
+            << parameters.store_coverage >>
           steps:
             - save_coverage
       - store_test_results:
@@ -283,7 +282,7 @@ commands:
             - run:
                 command: |
                   mkdir -p /tmp/core_dumps
-                  cp core.* /tmp/core_dumps
+                  cp core.* /tmp/core_dumps || true
                   ./scripts/bt
                 when: on_fail
             - store_artifacts:
@@ -299,9 +298,7 @@ commands:
                   hatch env show --json | jq -r 'keys[] | select(. | contains("<< parameters.env >>"))' | sort | circleci tests split | xargs -n 1 -I {} hatch run {}:test
       - when:
           condition:
-            and:
-              - << pipeline.parameters.coverage >>
-              - << parameters.store_coverage >>
+            << parameters.store_coverage >>
           steps:
             - save_coverage
       - store_test_results:
@@ -414,43 +411,35 @@ jobs:
   coverage_report:
     executor: python310
     steps:
-      - when:
-          condition:
-            << pipeline.parameters.coverage >>
-          steps:
-            - checkout
-            - attach_workspace:
-                at: .
-            - run: pip install -r ci/coverage/requirements.txt
-            - run: ls -hal *.coverage 2>/dev/null || true
-            # Combine all job coverage reports into one
-            - run: coverage combine *.coverage
-            # Upload coverage report to Codecov
-            # DEV: Do not use the bash uploader, it cannot be trusted
-            - run: codecov
-            # Generate and save xml report
-            # DEV: "--ignore-errors" to skip over files that are missing
-            - run: coverage xml --ignore-errors
-            - store_artifacts:
-                path: coverage.xml
-            # Generate and save JSON report
-            # DEV: "--ignore-errors" to skip over files that are missing
-            - run: coverage json --ignore-errors
-            - store_artifacts:
-                path: coverage.json
-            # Print ddtrace/ report to stdout
-            # DEV: "--ignore-errors" to skip over files that are missing
-            - run: coverage report --ignore-errors --omit=tests/
-            # Print tests/ report to stdout
-            # DEV: "--ignore-errors" to skip over files that are missing
-            - run: coverage report --ignore-errors --omit=ddtrace/
-            # Print diff-cover report to stdout (compares against origin/1.x)
-            - run: diff-cover --compare-branch $(git rev-parse --abbrev-ref origin/HEAD) coverage.xml
-      - unless:
-          condition:
-            << pipeline.parameters.coverage >>
-          steps:
-            - run: echo "No coverage data collected. Set run workflow manually with pipeline parameter 'coverage=true'."
+      - checkout
+      - attach_workspace:
+          at: .
+      - run: pip install -r ci/coverage/requirements.txt
+      - run: ls -hal *.coverage 2>/dev/null || true
+      - run: rm -f empty.*.coverage
+      # Combine all job coverage reports into one
+      - run: coverage combine *.coverage
+      # Upload coverage report to Codecov
+      # DEV: Do not use the bash uploader, it cannot be trusted
+      - run: codecov
+      # Generate and save xml report
+      # DEV: "--ignore-errors" to skip over files that are missing
+      - run: coverage xml --ignore-errors
+      - store_artifacts:
+          path: coverage.xml
+      # Generate and save JSON report
+      # DEV: "--ignore-errors" to skip over files that are missing
+      - run: coverage json --ignore-errors
+      - store_artifacts:
+          path: coverage.json
+      # Print ddtrace/ report to stdout
+      # DEV: "--ignore-errors" to skip over files that are missing
+      - run: coverage report --ignore-errors --omit=tests/
+      # Print tests/ report to stdout
+      # DEV: "--ignore-errors" to skip over files that are missing
+      - run: coverage report --ignore-errors --omit=ddtrace/
+      # Print diff-cover report to stdout (compares against origin/1.x)
+      - run: diff-cover --compare-branch $(git rev-parse --abbrev-ref origin/HEAD) coverage.xml
 
 
   build_base_venvs:
@@ -562,7 +551,6 @@ jobs:
     <<: *contrib_job_small
     steps:
       - run_test:
-          store_coverage: false
           pattern: "datastreams"
 
   tracer:
@@ -605,7 +593,6 @@ jobs:
       - run_test:
           pattern: "telemetry"
           snapshot: true
-          store_coverage: false
 
   debugger:
     <<: *contrib_job
@@ -642,7 +629,6 @@ jobs:
     parallelism: 16
     steps:
       - run_test:
-          store_coverage: false
           # We don't want to run the profile-diff venvs
           pattern: 'profile$'
           trace_agent_url: ""
@@ -666,7 +652,7 @@ jobs:
       - run:
           command: |
             mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
+            cp core.* /tmp/core_dumps || true
             ./scripts/bt
           when: on_fail
       - store_artifacts:
@@ -678,7 +664,6 @@ jobs:
     steps:
       - run_test:
           snapshot: true
-          store_coverage: false
           pattern: 'integration-snapshot*'
           trace_agent_url: ""
 
@@ -707,7 +692,6 @@ jobs:
       - image: *redis_image
     steps:
       - run_test:
-          store_coverage: false
           pattern: 'ddtracerun'
           trace_agent_url: ""
 
@@ -806,7 +790,6 @@ jobs:
         - ENABLED_CHECKS=trace_stall,meta_tracer_version_header,trace_content_length,trace_peer_service,trace_dd_service # disable flaky content length check
     steps:
       - run_test:
-          store_coverage: false
           pattern: 'celery'
 
   cherrypy:
@@ -903,7 +886,6 @@ jobs:
       - run_test:
           # Run both flask and flask_cache test suites
           # TODO: Re-enable coverage for Flask tests
-          store_coverage: false
           snapshot: true
           pattern: "flask"
           docker_services: "memcached redis"

riotfile.py

@@ -1665,7 +1665,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
         ),
         Venv(
             name="pytest-benchmark",
-            command="pytest {cmdargs} tests/contrib/pytest_benchmark/",
+            command="pytest {cmdargs} --no-cov tests/contrib/pytest_benchmark/",
             pkgs={
                 "msgpack": latest,
                 "pytest-randomly": latest,

scripts/run-test-suite

@@ -5,7 +5,18 @@ set -o pipefail
 CHECKPOINT_FILENAME="latest-success-commit"
 RIOT_PATTERN=${1}
 DDTRACE_FLAG=$([ -v _CI_DD_API_KEY ] && echo '--ddtrace')
-COVERAGE_FLAG=$([[ "${2:-false}" == false ]] && echo '--no-cov')
+GIT_MESSAGE_LOWERCASE=$(git log -1 --pretty=%B | tr "[:upper:]" "[:lower:]")
+if [[ $GIT_MESSAGE_LOWERCASE == *"itr:noskip"* ]]; then
+    COLLECT_COVERAGE=true
+else
+    if [[ -v CIRCLE_PULL_REQUEST ]]; then
+        COLLECT_COVERAGE=false
+    else
+        echo "Getting coverage for non-PR branch"
+        COLLECT_COVERAGE=true
+    fi
+fi
+COVERAGE_FLAG=$([[ "${2:-false}" == false && $COLLECT_COVERAGE == false ]] && echo '--no-cov')
 DDTEST_CMD=$([[ ${3} == "1" ]] && echo "./scripts/ddtest")
 
 RIOT_HASHES=( $(riot list --hash-only $RIOT_PATTERN | sort) )