Functional tests for WP-CLI commands

.github/workflows/behat-test.yml

@@ -0,0 +1,165 @@
+name: Behat Testing
+
+on:
+  push:
+    branches:
+      - trunk
+      - 'release/**'
+    # Only run if PHP-related files changed.
+    paths:
+      - '.github/workflows/behat-test.yml'
+      - '**.php'
+      - '**.feature'
+      - 'behat.yml'
+      - 'composer.json'
+      - 'composer.lock'
+  pull_request:
+    branches:
+      - trunk
+      - 'release/**'
+      - 'feature/**'
+    # Only run if PHP-related files changed.
+    paths:
+      - '.github/workflows/behat-test.yml'
+      - '**.php'
+      - '**.feature'
+      - 'behat.yml'
+      - 'composer.json'
+      - 'composer.lock'
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  behat-test:
+    name: PHP ${{ matrix.php }} - WP ${{ matrix.wordpress }}${{ matrix.experimental && ' (experimental)' || '' }}
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental == true }}
+    timeout-minutes: 20
+    services:
+      mysql:
+        image: mysql:8.0
+        ports:
+        - 3306/tcp
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 3
+          -e MYSQL_ROOT_PASSWORD=root
+          -e MYSQL_DATABASE=wp_cli_test
+          --entrypoint sh mysql:8.0
+          -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
+
+    strategy:
+      fail-fast: true
+      matrix:
+        php:
+        - '7.0'
+        - '7.1'
+        - '7.2'
+        - '7.3'
+        - '7.4'
+        - '8.1'
+        - '8.2'
+        wordpress: [ 'latest' ]
+        include:
+          - php: '8.0'
+            wordpress: 'latest'
+            coverage: true
+          - php: '8.0'
+            wordpress: '6.0'
+          - php: '8.0'
+            wordpress: '6.1'
+          - php: '8.2'
+            wordpress: 'trunk'
+            experimental: true
+    env:
+      WP_ENV_PHP_VERSION: ${{ matrix.php }}
+      WP_ENV_CORE: ${{ matrix.wordpress == 'trunk' && 'WordPress/WordPress' || format( 'https://wordpress.org/wordpress-{0}.zip', matrix.wordpress ) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          extensions: mysql
+          tools: composer
+          php-version: ${{ matrix.php }}
+          coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
+          ini-values: pcov.directory=.,pcov.exclude=~(vendor|tests)~
+
+      - name: Install PHP dependencies
+        uses: ramsey/composer-install@83af392bf5f031813d25e6fe4cd626cdba9a2df6
+        with:
+          composer-options: '--prefer-dist'
+
+      - name: Make Composer packages available globally
+        run: |
+          echo "${PWD}/vendor/bin" >> $GITHUB_PATH
+
+      - name: Install WP-CLI
+        run: |
+          curl -O https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar
+          chmod +x wp-cli.phar
+          mkdir -p bin
+          mv wp-cli.phar bin/wp
+          echo "WP_CLI_BIN_DIR=${PWD}/bin" >> $GITHUB_ENV
+
+      - name: Update PHPUnit to get latest php-code-coverage library
+        if: ${{ matrix.coverage == true }}
+        # phpunit/phpunit has to be updated as the one in use provides an older version of phpunit/php-code-coverage,
+        # but we need the v9.x branch.
+        # It cannot be removed, as it is a requirement of wp-cli/wp-cli-tests as well.
+        run: |
+          composer require --dev --ignore-platform-reqs --update-with-all-dependencies phpunit/phpunit
+
+      - name: Start MySQL server
+        run: sudo systemctl start mysql
+
+      - name: Configure DB environment
+        run: |
+          echo "MYSQL_HOST=127.0.0.1" >> $GITHUB_ENV
+          echo "MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }}" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBROOTUSER=root" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBROOTPASS=root" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBNAME=wp_cli_test" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBUSER=wp_cli_test" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBPASS=password1" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBHOST=127.0.0.1:${{ job.services.mysql.ports['3306'] }}" >> $GITHUB_ENV
+
+      - name: Prepare test database
+        run: composer prepare-behat-tests
+
+      - name: Check Behat environment
+        run: composer behat
+        env:
+          WP_CLI_TEST_DEBUG_BEHAT_ENV: 1
+
+      - name: Run tests
+        env:
+          BEHAT_CODE_COVERAGE: ${{ matrix.coverage }}
+        run: composer behat || composer behat-rerun
+
+      - name: Retrieve list of coverage files
+        id: coverage_files
+        if: ${{ matrix.coverage == true }}
+        run: |
+          FILES=$(ls -d -1 "$GITHUB_WORKSPACE/build/logs/clover-behat/"*.* | paste --serial --delimiters=",")
+          test -n "$FILES"
+          echo "Coverage files: $FILES"
+          echo "files=$FILES" >> $GITHUB_OUTPUT
+
+      - name: Upload code coverage report
+        if: ${{ matrix.coverage }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: ${{ steps.coverage_files.outputs.files }}
+          flags: feature
+          fail_ci_if_error: true

behat.yml

@@ -0,0 +1,7 @@
+default:
+  suites:
+    default:
+      contexts:
+        - WordPress\Plugin_Check\Behat_Utils\FeatureContext
+      paths:
+        - tests/behat/features

composer.json

@@ -12,7 +12,10 @@
   },
   "require-dev": {
     "wp-phpunit/wp-phpunit": "^6.1",
-    "yoast/phpunit-polyfills": "^1.0"
+    "yoast/phpunit-polyfills": "^1.0",
+    "wp-cli/extension-command": "^2.1",
+    "wp-cli/wp-cli": "^2.8",
+    "wp-cli/wp-cli-tests": "^v4.2.2"
   },
   "scripts": {
     "lint": [
@@ -31,7 +34,20 @@
     "phpstan": [
       "composer --working-dir=build-cs install",
       "build-cs/vendor/bin/phpstan analyse --memory-limit=2048M"
-    ]
+    ],
+    "behat": "BEHAT_FEATURES_FOLDER=tests/behat/features run-behat-tests",
+    "behat-rerun": "BEHAT_FEATURES_FOLDER=tests/behat/features rerun-behat-tests",
+    "prepare-behat-tests": "install-package-tests"
+  },
+  "scripts-descriptions": {
+    "lint": "Detect coding standards issues",
+    "format": "Detect and automatically fix most coding standards issues",
+    "test": "Run unit tests",
+    "phpmd": "Run PHP mess detector",
+    "phpstan": "Run static analysis",
+    "behat": "Run functional tests",
+    "behat-rerun": "Re-run failed functional tests",
+    "prepare-behat-tests": "Prepare functional tests"
   },
   "config": {
     "allow-plugins": {
@@ -52,7 +68,8 @@
   "autoload-dev": {
     "psr-4": {
       "WordPress\\Plugin_Check\\Test_Data\\": "tests/phpunit/testdata",
-      "WordPress\\Plugin_Check\\Test_Utils\\": "tests/phpunit/utils"
+      "WordPress\\Plugin_Check\\Test_Utils\\": "tests/phpunit/utils",
+      "WordPress\\Plugin_Check\\Behat_Utils\\": "tests/behat/includes"
     }
   }
 }