Functional tests for WP-CLI commands (#124)

* First pass at new behat config

* Use https url for dist-archive-command repo

Needed as long as the command isn't available on Packagist

* Build zip on travis

* Use binaries inside vendor/bin/

* Only create a single build before running all tests

* Add quotes

* Add some debugging lines for travis

* Update wp-cli/dist-archive-command dependency

* Try moving build step to before_script

* Further tests

* Add debug flag  to cli command

* Try cd'ing

* prepare behat tests on install

* Use build-ci script again

* Use hardcoded directory name

* Add git folder to distignore

* Use "./" as a workaround for dist-archive

* Use wp-cli-tests v3

* Lint fixes

* Load wp-cli-stubs with scanFiles: (#209)

* Normalize Composer configuration (#211)

* Update composer script for code coverage

* Don't run behat tests in the PHPUnit workflow

* Apply suggestions from code review

* Remove unneeded dependency

* Add GitHub action workflow

* Install WordPress test setup

* Update workflow

* Hardcode mysql version

* Add `maybe-generate-wp-cli-coverage.php`

* Stop on missing coverage files in CI (#213)

* Use `set-output`

* Update codecov flags

* Update PHPUnit to get latest php-code-coverage library

* Undo some changes

* Normalize composer.json

* Run file through PHPCS

* Update workflow

* Add FeatureContext to install GlotPress

* Add some experiment new tests

* Fix some whitespace

* Use `vendor/bin/behat` for now

* Fix code coverage

* Ensure pretty permalinks for GlotPress

* Fix GlotPress installation in CLI context

* Update tests

* GlotPress 3.x requires PHP 7.2+

* Add another test to check for coverage update

* Update `install-wp-tests.sh`

* Update gitignore

* Set `pcov.directory` appropriately

* Set `pcov.exclude` as well

Co-authored-by: Viktor Szépe <[email protected]>
Co-authored-by: Dominik Schilling <[email protected]>

.github/workflows/behat.yml

@@ -0,0 +1,141 @@
+name: Behat
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  behat:
+    name: Behat (PHP ${{ matrix.php }}, WordPress ${{ matrix.wordpress }}, GlotPress ${{ matrix.glotpress }})
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    services:
+      mysql:
+        image: mysql:5.7
+        ports:
+          - 3306/tcp
+        env:
+          MYSQL_ROOT_PASSWORD: password
+        # Set health checks to wait until mysql has started
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 3
+          -e MYSQL_ROOT_PASSWORD=root
+          -e MYSQL_DATABASE=wp_cli_test
+          --entrypoint sh mysql:5.7
+          -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
+
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        php: [ '7.4', '7.3', '7.2' ]
+        wordpress: [ 'latest', 'nightly' ]
+        glotpress: [ 'develop' ]
+        experimental: [ false ]
+        include:
+        # - php: '8.0'
+        #   os: ubuntu-latest
+        #   experimental: true
+          - os: ubuntu-latest
+            php: '7.4'
+            wordpress: 'latest'
+            glotpress: 'develop'
+            experimental: false
+            coverage: true
+
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          coverage: ${{ matrix.coverage && 'pcov' || 'none' }}
+          tools: composer
+          ini-values: pcov.directory=.,pcov.exclude=~(vendor|tests)~
+
+      - name: Shutdown default MySQL service
+        run: sudo service mysql stop
+
+      - name: Verify MariaDB connection
+        run: |
+          while ! mysqladmin ping -h"127.0.0.1" -P"${{ job.services.mysql.ports[3306] }}" --silent; do
+            sleep 1
+          done
+
+      - name: Get Composer cache directory
+        id: composer-cache
+        run: |
+          echo "::set-output name=dir::$(composer config cache-files-dir)"
+
+      - name: Cache PHP dependencies
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.json') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+
+      - name: Install WP-CLI
+        run: |
+          curl -O https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar
+          chmod +x wp-cli.phar
+          mkdir -p bin
+          mv wp-cli.phar bin/wp
+          echo "WP_CLI_BIN_DIR=${PWD}/bin" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          composer install --no-progress --no-suggest --prefer-dist --optimize-autoloader
+          echo "${PWD}/vendor/bin" >> $GITHUB_PATH
+
+      - name: Update PHPUnit to get latest php-code-coverage library
+        if: ${{ matrix.coverage == true }}
+        # phpunit/phpunit has to be updated as the one in use provides an older version of phpunit/php-code-coverage,
+        # but we need the v9.x branch.
+        # It cannot be removed, as it is a requirement of wp-cli/wp-cli-tests as well.
+        run: |
+          composer require --dev --ignore-platform-reqs --update-with-all-dependencies phpunit/phpunit
+
+      - name: Configure DB environment
+        run: |
+          export MYSQL_HOST=127.0.0.1
+          export MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }}
+          echo "WP_CLI_TEST_DBROOTUSER=root" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBROOTPASS=root" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBUSER=wp_cli_test" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBPASS=password1" >> $GITHUB_ENV
+          echo "WP_CLI_TEST_DBHOST=$MYSQL_HOST:$MYSQL_TCP_PORT" >> $GITHUB_ENV
+
+      - name: Prepare test database
+        run: composer prepare-tests
+
+      - name: Check Behat environment
+        run: WP_CLI_TEST_DEBUG_BEHAT_ENV=1 composer behat
+
+      - name: Run tests
+        env:
+          BEHAT_CODE_COVERAGE: ${{ matrix.coverage }}
+        run: vendor/bin/behat
+
+      - name: Retrieve list of coverage files
+        id: coverage_files
+        if: ${{ matrix.coverage == true }}
+        run: |
+          FILES=$(ls -d -1 "$GITHUB_WORKSPACE/build/logs/clover-behat/"*.* | paste --serial --delimiters=",")
+          test -n "$FILES"
+          echo "Coverage files: $FILES"
+          echo "::set-output name=COVERAGE_FILES::$FILES"
+
+      - name: Upload code coverage report
+        if: ${{ matrix.coverage }}
+        uses: codecov/[email protected]
+        with:
+          files: ${{ steps.coverage_files.outputs.COVERAGE_FILES }}
+          flags: feature
+          fail_ci_if_error: true

.github/workflows/phpunit.yml

@@ -110,14 +110,15 @@ jobs:
 
       - name: Run tests
         if: ${{ ! matrix.coverage }}
-        run: composer run-script test
+        run: composer run-script phpunit
 
       - name: Run tests with code coverage
         if: ${{ matrix.coverage }}
-        run: composer run-script test -- --coverage-clover coverage-clover-${{ github.sha }}.xml
+        run: composer run-script phpunit -- --coverage-clover coverage-clover-${{ github.sha }}.xml
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.coverage }}
         uses: codecov/codecov-action@v1
         with:
           file: coverage-clover-${{ github.sha }}.xml
+          flags: php

.gitignore

@@ -6,4 +6,10 @@
 /node_modules/
 
 # PHPUnit
-/coverage.xml
+/coverage*.xml
+
+# Behat
+*.log
+
+# Build process
+/build

CONTRIBUTING.md

@@ -25,4 +25,4 @@ When contributing code...
     * Ideally write unit tests for any code changes and verify them using `composer test`
 * Commit and push changes to your fork and [submit a pull request](https://github.com/wearerequired/traduttore/compare) to the `master` branch
 
-We try to review incoming pull requests as soon as possible and either merge them or make suggestions for some further improvements. 
+We try to review incoming pull requests as soon as possible and either merge them or make suggestions for some further improvements.

behat.yml

@@ -0,0 +1,7 @@
+default:
+  suites:
+    default:
+      contexts:
+      - Required\Traduttore\Tests\Behat\FeatureContext
+      paths:
+      - tests/features

bin/install-wp-tests.sh

@@ -21,7 +21,9 @@ GP_TESTS_DIR="$WP_CORE_DIR/build/wp-content/plugins/glotpress/tests/phpunit"
 
 download() {
     if [ `which curl` ]; then
-        curl -s "$1" > "$2";
+        #curl -s "$1" > "$2";
+        # WordPress.org seems to block requests without proper user agent.
+        curl -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (K HTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" -s "$1" > "$2";
     elif [ `which wget` ]; then
         wget -nv -O "$2" "$1"
     fi

composer.json

@@ -40,14 +40,21 @@
   "require-dev": {
     "dealerdirect/phpcodesniffer-composer-installer": "^0.7.0",
     "php-stubs/wp-cli-stubs": "^2.4",
-    "phpunit/phpunit": "^6.5 || ^7.5",
+    "phpunit/phpunit": "^7.5.20",
     "szepeviktor/phpstan-wordpress": "^0.7.0",
     "wearerequired/coding-standards": "^1.5",
+    "wp-cli/extension-command": "^2.0",
+    "wp-cli/rewrite-command": "^2.0",
+    "wp-cli/wp-cli-tests": "^3.0.11",
     "wpackagist-plugin/glotpress": "^2.3.1"
   },
   "suggest": {
     "wpackagist-plugin/slack": "Send Slack notifications for various events"
   },
+  "config": {
+    "process-timeout": 7200,
+    "sort-packages": true
+  },
   "extra": {
     "branch-alias": {
       "dev-master": "3.x-dev"
@@ -78,8 +85,28 @@
   ],
   "scripts": {
     "analyze": "vendor/bin/phpstan analyze --no-progress --memory-limit=1024M",
+    "behat": "run-behat-tests",
+    "behat-rerun": "rerun-behat-tests",
     "format": "vendor/bin/phpcbf --report-summary --report-source .",
     "lint": "vendor/bin/phpcs --report-summary --report-source .",
-    "test": "vendor/bin/phpunit"
+    "phpunit": "vendor/bin/phpunit",
+    "prepare-tests": "install-package-tests",
+    "test": [
+      "@behat",
+      "@phpunit"
+    ]
+  },
+  "scripts-descriptions": {
+    "analyze": "Run static analysis",
+    "behat": "Run functional tests",
+    "behat-rerun": "Re-run failed functional tests",
+    "format": "Detect and automatically fix most coding standards issues",
+    "lint": "Detect coding standards issues",
+    "phpunit": "Run unit tests",
+    "prepare-tests": "Prepare functional tests",
+    "test": "Run all tests at once"
+  },
+  "support": {
+    "issues": "https://github.com/wearerequired/traduttore/issues"
   }
 }

inc/CLI/CacheCommand.php

@@ -47,9 +47,9 @@ class CacheCommand extends WP_CLI_Command {
 	 *
 	 * @since 2.0.0
 	 *
-	 * @param array $args Command args.
+	 * @param array<mixed> $args Command args.
 	 */
-	public function clear( $args ): void {
+	public function clear( array $args ): void {
 		$locator = new ProjectLocator( $args[0] );
 		$project = $locator->get_project();
 

inc/CLI/InfoCommand.php

@@ -48,13 +48,13 @@ class InfoCommand extends WP_CLI_Command {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @param array $args Command args.
-	 * @param array $assoc_args Associative args.
+	 * @param array<mixed> $args Command args.
+	 * @param array<mixed> $assoc_args Associative args.
 	 */
-	public function __invoke( $args, $assoc_args ): void {
+	public function __invoke( array $args, array $assoc_args ): void {
 		$plugin_version = \Required\Traduttore\VERSION;
 		$wp_version     = get_bloginfo( 'version' );
-		$gp_version     = GP_VERSION;
+		$gp_version     = \defined( 'GP_VERSION' ) ? GP_VERSION : null;
 
 		$wp_cli_version = WP_CLI_VERSION;
 		$git_binary     = $this->get_git_binary_path();

inc/CLI/LanguagePackCommand.php

@@ -74,10 +74,10 @@ class LanguagePackCommand extends WP_CLI_Command {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @param array $args Command args.
-	 * @param array $assoc_args Associative args.
+	 * @param array<mixed> $args Command args.
+	 * @param array<mixed> $assoc_args Associative args.
 	 */
-	public function list( $args, $assoc_args ): void {
+	public function list( array $args, array $assoc_args ): void {
 		$locator = new ProjectLocator( $args[0] );
 		$project = $locator->get_project();
 
@@ -156,10 +156,10 @@ public function list( $args, $assoc_args ): void {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @param array $args Command args.
-	 * @param array $assoc_args Associative args.
+	 * @param array<mixed> $args Command args.
+	 * @param array<mixed> $assoc_args Associative args.
 	 */
-	public function build( $args, $assoc_args ): void {
+	public function build( array $args, array $assoc_args ): void {
 		$all      = get_flag_value( $assoc_args, 'all', false );
 		$force    = get_flag_value( $assoc_args, 'force', false );
 		$projects = $this->check_optional_args_and_all( $args, $all );
@@ -216,11 +216,11 @@ public function build( $args, $assoc_args ): void {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @param array $args Passed arguments.
-	 * @param bool  $all  All flag.
-	 * @return array Same as $args if not all, otherwise all slugs.
+	 * @param array<mixed> $args Passed arguments.
+	 * @param bool         $all  All flag.
+	 * @return array<mixed> Same as $args if not all, otherwise all slugs.
 	 */
-	protected function check_optional_args_and_all( $args, $all ): array {
+	protected function check_optional_args_and_all( array $args, bool $all ): array {
 		if ( $all ) {
 			$args = $this->get_all_projects();
 		}

inc/CLI/ProjectCommand.php

@@ -61,10 +61,10 @@ class ProjectCommand extends WP_CLI_Command {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @param array $args Command args.
-	 * @param array $assoc_args Associative args.
+	 * @param array<mixed> $args Command args.
+	 * @param array<mixed> $assoc_args Associative args.
 	 */
-	public function info( $args, $assoc_args ): void {
+	public function info( array $args, array $assoc_args ): void {
 		$locator = new ProjectLocator( $args[0] );
 		$project = $locator->get_project();
 
@@ -162,10 +162,10 @@ public function info( $args, $assoc_args ): void {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @param array $args Command args.
-	 * @param array $assoc_args Associative args.
+	 * @param array<mixed> $args Command args.
+	 * @param array<mixed> $assoc_args Associative args.
 	 */
-	public function update( $args, $assoc_args ): void {
+	public function update( array $args, array $assoc_args ): void {
 		$delete  = get_flag_value( $assoc_args, 'delete', false );
 		$cached  = get_flag_value( $assoc_args, 'cached', false );
 		$locator = new ProjectLocator( $args[0] );

inc/Configuration.php

@@ -60,7 +60,7 @@ public function get_path(): string {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @return array Repository configuration.
+	 * @return array<string, mixed> Repository configuration.
 	 */
 	public function get_config(): array {
 		return $this->config;
@@ -87,7 +87,7 @@ public function get_config_value( string $key ) {
 	 *
 	 * @since 3.0.0
 	 *
-	 * @return array Configuration data if found.
+	 * @return array<string, array<string, string>> Configuration data if found.
 	 */
 	protected function load_config(): array {
 		$config_file   = trailingslashit( $this->path ) . 'traduttore.json';