From c5de6bf3c2a99d485145d78cf6c83b782d796733 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 11:07:04 +0530 Subject: [PATCH 01/30] test(easyengine): add behat context with shared steps Base the core context on site-command's, fix its unanchored step regexes and inverted container count check, and add generic steps for exit codes, output, json paths, files, EasyEngine options and a per-scenario scratch dir. Commands run through EE\Process so the harness no longer writes them to ee.log, and cleanup only touches bh-easyengine-* sites. --- features/bootstrap/FeatureContext.php | 911 +++++++++++++++++++++++++- 1 file changed, 890 insertions(+), 21 deletions(-) diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 11dfd75db..f0b47a644 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -14,7 +14,18 @@ define( 'EE', true ); define( 'EE_VERSION', trim( file_get_contents( EE_ROOT . '/VERSION' ) ) ); -define( 'EE_ROOT_DIR', '/opt/easyengine' ); + +$root_alias = '/opt'; +if ( 'Darwin' === php_uname( 's' ) ) { + $sys_name = posix_getpwuid(posix_geteuid())['name']; + if ( ! empty( $sys_name ) ) { + $root_alias = '/Users/' . $sys_name; + } +} +// site-command's autoloaded site-command.php may already define it. +if ( ! defined( 'EE_ROOT_DIR' ) ) { + define( 'EE_ROOT_DIR', $root_alias . '/easyengine' ); +} require_once EE_ROOT . '/php/bootstrap.php'; @@ -46,34 +57,82 @@ use Behat\Behat\Context\Context; use Behat\Behat\Hook\Scope\AfterFeatureScope; +use Behat\Behat\Hook\Scope\AfterScenarioScope; +use Behat\Gherkin\Node\PyStringNode, + Behat\Gherkin\Node\TableNode; -use Behat\Gherkin\Node\PyStringNode; +define( 'EE_SITE_ROOT', EE_ROOT_DIR . '/sites' ); class FeatureContext implements Context { - public $command; + public $commands; public $webroot_path; + public $ee_path; + + /** + * Prefix of every site this suite may create; cleanup never touches other sites. + */ + const SITE_PREFIX = 'bh-easyengine-'; + + /** + * Base directory of the per-scenario scratch directories ({TMP}). + */ + const TMP_BASE = '/var/tmp/ee-behat-core'; + + /** + * Per-scenario scratch directory, created on first use of {TMP}. + * + * @var string|null + */ + private $tmp_dir; + + /** + * Per-scenario random token ({RAND}). + * + * @var string|null + */ + private $rand; + + /** + * Original values of EasyEngine options changed by a scenario, restored afterwards. + * + * @var array + */ + private $saved_options = []; + + /** + * Container counts recorded by "I record the number of containers with label". + * + * @var array + */ + private $container_counts = []; + + /** + * Initializes context. + */ + public function __construct() + { + $this->commands = []; + $this->ee_path = getcwd(); + } /** * @Given ee phar is generated */ public function eePharIsPresent() { - // Checks if phar already exists, replaces it + // Building the phar is slow, so reuse the copy cached by an earlier scenario. if(file_exists('ee-old.phar')) { - // Below exec call is required as currenly `ee cli update` is ran with root - // which updates ee.phar with root privileges. - exec("sudo rm ee.phar"); + @unlink('ee.phar'); copy('ee-old.phar','ee.phar'); return 0; } - exec("php -dphar.readonly=0 utils/make-phar.php ee.phar", $output, $return_status); + exec("php -dphar.readonly=0 utils/make-phar.php ee.phar --quiet", $output, $return_status); if (0 !== $return_status) { throw new Exception("Unable to generate phar" . $return_status); } - // Cache generaed phar as it is expensive to generate one copy('ee.phar','ee-old.phar'); } @@ -89,50 +148,283 @@ public function isInstalled($command) } /** - * @When I run :command + * @When /^I run (?|'(.*)'|"(.*)")$/ */ public function iRun($command) { - $this->command = EE::launch($command, false, true); + // Not EE::launch(): it would log the command line to ee.log, polluting log assertions. + $this->commands[] = \EE\Process::create( $this->replace_variables( $command ), null, null )->run(); } + + /** + * @When I create subsite :subsite in :site + */ + public function iCreateSubsiteInOfType($subsite, $site ) + { + $php_container = implode( explode( '.', $subsite ) ) . '_php_1'; + + EE::launch( "docker exec --user='www-data' $php_container sh -c 'wp site create --slug=$site'", false, true ); + } + /** * @Then return value should be 0 */ - public function returnCodeShouldBe0() + public function returnValueShouldBe0() + { + if ( 0 !== $this->commands[0]->return_code ) { + throw new Exception("Actual return code is not zero: \n" . $this->commands[0]); + } + } + + /** + * @Then return value of command :index should be 0 + */ + public function returnValueOfCommandShouldBe0($index) { - if ( 0 !== $this->command->return_code ) { - throw new Exception("Actual return code is not zero: \n" . $this->command); + if ( 0 !== $this->commands[ $index - 1 ]->return_code ) { + throw new Exception("Actual return code is not zero: \n" . $this->commands[ $index - 1 ]); } } /** - * @Then /(STDOUT|STDERR) should return exactly/ + * @Then After delay of :time seconds + */ + public function afterDelayOfSeconds( $time ) + { + sleep( $time ); + } + + /** + * @Then /^(STDOUT|STDERR) should return exactly$/ */ public function stdoutShouldReturnExactly($output_stream, PyStringNode $expected_output) { - $command_output = $output_stream === "STDOUT" ? $this->command->stdout : $this->command->stderr; + $command_output = $output_stream === "STDOUT" ? $this->commands[0]->stdout : $this->commands[0]->stderr; $command_output = str_replace(["\033[1;31m","\033[0m"],'',$command_output); - $expected_out = isset($expected_output->getStrings()[0]) ? $expected_output->getStrings()[0] : ''; - if ( $expected_out !== trim($command_output)) { + if ( $this->replace_variables( $expected_output->getRaw() ) !== trim($command_output)) { throw new Exception("Actual output is:\n" . $command_output); } } /** - * @Then /(STDOUT|STDERR) should return something like/ + * @Then /^(STDOUT|STDERR) of command (\d+) should return exactly$/ + */ + public function stdoutOfCommandShouldReturnExactly($output_stream, $index, PyStringNode $expected_output) + { + $command_output = $output_stream === "STDOUT" ? $this->commands[ $index - 1 ]->stdout : $this->commands[ $index - 1 ]->stderr; + + $command_output = str_replace(["\033[1;31m","\033[0m"],'',$command_output); + + if ( $this->replace_variables( $expected_output->getRaw() ) !== trim($command_output)) { + throw new Exception("Actual output is:\n" . $command_output); + } + } + + /** + * @Then /^(STDOUT|STDERR) should return something like$/ */ public function stdoutShouldReturnSomethingLike($output_stream, PyStringNode $expected_output) { - $command_output = $output_stream === "STDOUT" ? $this->command->stdout : $this->command->stderr; + $command_output = $output_stream === "STDOUT" ? $this->commands[0]->stdout : $this->commands[0]->stderr; - $expected_out = isset($expected_output->getStrings()[0]) ? $expected_output->getStrings()[0] : ''; + $expected_out = $this->replace_variables( $expected_output->getRaw() ); if (strpos($command_output, $expected_out) === false) { throw new Exception("Actual output is:\n" . $command_output); } } + + /** + * @Then /^(STDOUT|STDERR) of command (\d+) should return something like$/ + */ + public function stdoutOfCommandShouldReturnSomethingLike($output_stream, $index, PyStringNode $expected_output) + { + $command_output = $output_stream === "STDOUT" ? $this->commands[ $index - 1 ]->stdout : $this->commands[ $index - 1 ]->stderr; + + $expected_out = $this->replace_variables( $expected_output->getRaw() ); + if (strpos($command_output, $expected_out) === false) { + throw new Exception("Actual output is:\n" . $command_output); + } + } + + /** + * @Then The site :site should be :type multisite + */ + public function theSiteShouldBeMultisite( $site, $type ) + { + $result = EE::launch( "cd " . EE_SITE_ROOT . "/$site && docker-compose exec -T --user='www-data' php sh -c 'wp config get SUBDOMAIN_INSTALL'", false, true ); + + if( $result->stderr ) { + throw new Exception("Found error while executing command: $result->stderr"); + } + + if($type === 'subdomain' && trim($result->stdout) !== '1') { + throw new Exception("Expecting SUBDOMAIN_INSTALL to be 1. Got: $result->stdout"); + } + else if($type === 'subdir' && trim($result->stdout) !== '') { + throw new Exception("Expecting SUBDOMAIN_INSTALL to be empty. Got: $result->stdout"); + } + else if($type !== 'subdomain' && $type !== 'subdir') { + throw new Exception("Found unknown site type: $type"); + } + } + + /** + * @Then The :site db entry should be removed + */ + public function theDbEntryShouldBeRemoved($site) + { + $out = shell_exec("sudo bin/ee site list"); + if (strpos($out, $site) !== false) { + throw new Exception("$site db entry not been removed!"); + } + + } + + /** + * @Then The :site webroot should be removed + */ + public function theWebrootShouldBeRemoved($site) + { + if ( file_exists( EE_SITE_ROOT . '/' . $site ) ) { + throw new Exception("Webroot has not been removed!"); + } + } + + /** + * @Then Following containers of site :site should be removed: + */ + public function followingContainersOfSiteShouldBeRemoved($site, TableNode $table) + { + $containers = $table->getHash(); + $site_name = implode(explode('.', $site)); + + foreach ($containers as $container) { + + $sevice = $container['container']; + $container_name = $site_name . '_' . $sevice . '_1'; + + exec("docker inspect -f '{{.State.Running}}' $container_name > /dev/null 2>&1", $exec_out, $return); + if (!$return) { + throw new Exception("$container_name has not been removed!"); + } + } + } + + /** + * @Then The site :site should have webroot + */ + public function theSiteShouldHaveWebroot($site) + { + if ( ! file_exists( EE_SITE_ROOT . '/' . $site ) ) { + throw new Exception("Webroot has not been created!"); + } + } + + /** + * @Then The site :site should have WordPress + */ + public function theSiteShouldHaveWordpress($site) + { + if ( ! file_exists( EE_SITE_ROOT . '/' . $site . '/app/wp-config.php' ) ) { + throw new Exception("WordPress data not found!"); + } + } + + /** + * @Then Request on :site should contain following headers: + */ + public function requestOnShouldContainFollowingHeaders($site, TableNode $table) + { + $url = 'http://' . $site; + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $headers = curl_exec($ch); + + curl_close($ch); + + $rows = $table->getHash(); + + foreach ($rows as $row) { + if (strpos($headers, $row['header']) === false) { + throw new Exception("Unable to find " . $row['header'] . "\nActual output is : " . $headers); + } + } + } + + /** + * @Then Request on :host with header :header should contain following headers: + */ + public function requestOnWithHeaderShouldContainFollowingHeaders($host, $header, TableNode $table) + { + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $host); + curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, [ $header ]); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $headers = curl_exec($ch); + + curl_close($ch); + + $rows = $table->getHash(); + + foreach ($rows as $row) { + if (strpos($headers, $row['header']) === false) { + throw new Exception("Unable to find " . $row['header'] . "\nActual output is : " . $headers); + } + } + } + + /** + * @Then Request on :host with resolve option :resolve should contain following headers: + */ + public function requestOnWithResolveOptionShouldContainFollowingHeaders($host, $resolve, TableNode $table) + { + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $host); + curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($ch, CURLOPT_RESOLVE, [ $resolve ]); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $headers = curl_exec($ch); + curl_close($ch); + + $rows = $table->getHash(); + + foreach ($rows as $row) { + if (strpos($headers, $row['header']) === false) { + throw new Exception("Unable to find " . $row['header'] . "\nActual output is : " . $headers); + } + } + } + + /** + * @Then There should be :expected_running_containers containers with labels + */ + public function thereShouldBeContainersWithLabel($expected_running_containers, PyStringNode $pyStringNode) + { + $labels = $pyStringNode->getStrings(); + $label_string = implode(' -f label=',$labels); + + $result = EE::launch( "docker ps -qf label=$label_string | wc -l", false, true ); + $running_containers = (int) trim($result->stdout); + + if((int) $expected_running_containers !== $running_containers) { + throw new Exception("Expected $expected_running_containers running containers. Found: $running_containers"); + } + } + /** * @Then ee should be deleted */ @@ -154,11 +446,397 @@ public function eeShouldBeDeleted() } } + /* + * --------------------------------------------------------------------- + * Shared generic steps (easyengine core suite). + * + * Output steps act on the LAST command run in the scenario. Step strings + * support the placeholders {TMP} (per-scenario scratch dir under + * /var/tmp), {EE} (absolute bin/ee), {EE_ROOT}, {EE_ROOT_DIR}, + * {EE_VERSION}, {PHP} (PHP binary running Behat) and {RAND}. + * --------------------------------------------------------------------- + */ + + /** + * @Then the command should exit with code :code + */ + public function theCommandShouldExitWithCode( $code ) + { + $last = $this->last_command(); + if ( (int) $code !== $last->return_code ) { + throw new Exception( "Expected exit code $code, got {$last->return_code}.\n" . $this->describe( $last ) ); + } + } + + /** + * @Then the command should succeed + */ + public function theCommandShouldSucceed() + { + $this->theCommandShouldExitWithCode( 0 ); + } + + /** + * @Then the command should fail + */ + public function theCommandShouldFail() + { + $last = $this->last_command(); + if ( 0 === $last->return_code ) { + throw new Exception( "Expected a non-zero exit code, got 0.\n" . $this->describe( $last ) ); + } + } + + /** + * @Then /^(STDOUT|STDERR) should (not )?contain "(.*)"$/ + */ + public function streamShouldContainText( $stream, $negate, $text ) + { + $this->assert_contains( $stream, $this->replace_variables( $text ), '' !== $negate ); + } + + /** + * @Then /^(STDOUT|STDERR) should (not )?contain:$/ + */ + public function streamShouldContainBlock( $stream, $negate, PyStringNode $text ) + { + $this->assert_contains( $stream, $this->replace_variables( $text->getRaw() ), '' !== $negate ); + } + + /** + * @Then /^(STDOUT|STDERR) should be:$/ + */ + public function streamShouldBe( $stream, PyStringNode $expected ) + { + $actual = trim( $this->stream_output( $stream ) ); + $expected = trim( $this->replace_variables( $expected->getRaw() ) ); + if ( $expected !== $actual ) { + throw new Exception( "$stream is not exactly:\n$expected\n\n" . $this->describe( $this->last_command() ) ); + } + } + + /** + * @Then /^(STDOUT|STDERR) should be empty$/ + */ + public function streamShouldBeEmpty( $stream ) + { + if ( '' !== trim( $this->stream_output( $stream ) ) ) { + throw new Exception( "$stream is not empty.\n" . $this->describe( $this->last_command() ) ); + } + } + + /** + * @Then /^(STDOUT|STDERR) should (not )?match "(.*)"$/ + */ + public function streamShouldMatch( $stream, $negate, $regex ) + { + $regex = $this->replace_variables( $regex ); + $matched = preg_match( $regex, $this->stream_output( $stream ) ); + if ( false === $matched ) { + throw new Exception( "Invalid regex: $regex" ); + } + if ( ( '' === $negate ) !== ( 1 === $matched ) ) { + throw new Exception( "$stream should " . $negate . "match $regex\n" . $this->describe( $this->last_command() ) ); + } + } + + /** + * @Then /^(STDOUT|STDERR) should (not )?contain ANSI color codes$/ + */ + public function streamShouldContainAnsiCodes( $stream, $negate ) + { + $has_codes = 1 === preg_match( '/\x1b\[[0-9;]*m/', $this->stream_output( $stream ) ); + if ( ( '' === $negate ) !== $has_codes ) { + throw new Exception( "$stream should " . $negate . "contain ANSI color codes.\n" . $this->describe( $this->last_command(), true ) ); + } + } + + /** + * @Then STDOUT of command :first should be identical to STDOUT of command :second + */ + public function stdoutOfCommandsShouldBeIdentical( $first, $second ) + { + $a = $this->command_at( $first ); + $b = $this->command_at( $second ); + if ( $a->stdout !== $b->stdout ) { + throw new Exception( "STDOUT differs.\nCommand $first:\n" . $this->describe( $a ) . "\nCommand $second:\n" . $this->describe( $b ) ); + } + } + + /** + * @Then STDOUT should be valid json + */ + public function stdoutShouldBeValidJson() + { + $this->decoded_stdout(); + } + + /** + * @Then STDOUT should be empty or valid json + */ + public function stdoutShouldBeEmptyOrValidJson() + { + if ( '' !== trim( $this->last_command()->stdout ) ) { + $this->decoded_stdout(); + } + } + + /** + * @Then the json in STDOUT should have the keys :keys + */ + public function theJsonShouldHaveTheKeys( $keys ) + { + $this->theJsonAtShouldHaveTheKeys( '', $keys ); + } + + /** + * @Then the json in STDOUT at :path should have the keys :keys + */ + public function theJsonAtShouldHaveTheKeys( $path, $keys ) + { + $node = $this->single_json_node( $path ); + $missing = []; + foreach ( array_map( 'trim', explode( ',', $this->replace_variables( $keys ) ) ) as $key ) { + if ( ! is_array( $node ) || ! array_key_exists( $key, $node ) ) { + $missing[] = $key; + } + } + if ( $missing ) { + throw new Exception( 'Missing keys at "' . $path . '": ' . implode( ', ', $missing ) . "\n" . $this->describe( $this->last_command() ) ); + } + } + + /** + * @Then the json in STDOUT at :path should be :value + */ + public function theJsonAtShouldBe( $path, $value ) + { + $actual = self::json_scalar( $this->single_json_node( $path ) ); + $expected = $this->replace_variables( $value ); + if ( $expected !== $actual ) { + throw new Exception( "Expected \"$path\" to be \"$expected\", got \"$actual\".\n" . $this->describe( $this->last_command() ) ); + } + } + + /** + * @Then the json in STDOUT at :path should match :regex + */ + public function theJsonAtShouldMatch( $path, $regex ) + { + $actual = self::json_scalar( $this->single_json_node( $path ) ); + if ( 1 !== preg_match( $this->replace_variables( $regex ), $actual ) ) { + throw new Exception( "Expected \"$path\" to match $regex, got \"$actual\".\n" . $this->describe( $this->last_command() ) ); + } + } + + /** + * @Then /^the json in STDOUT at "([^"]*)" should (not )?contain "([^"]*)"$/ + */ + public function theJsonAtShouldContain( $path, $negate, $value ) + { + $nodes = $this->json_nodes( $path ); + // A single array node means "this list contains"; several nodes come from a "*" path. + if ( 1 === count( $nodes ) && is_array( $nodes[0] ) ) { + $nodes = array_values( $nodes[0] ); + } + $value = $this->replace_variables( $value ); + $values = array_map( array( __CLASS__, 'json_scalar' ), $nodes ); + if ( ( '' === $negate ) !== in_array( $value, $values, true ) ) { + throw new Exception( "Expected \"$path\" to " . $negate . "contain \"$value\". Values: " . implode( ', ', $values ) ); + } + } + + /** + * @Given a file :path with: + */ + public function aFileWith( $path, PyStringNode $content ) + { + $path = $this->replace_variables( $path ); + if ( ! is_dir( dirname( $path ) ) ) { + mkdir( dirname( $path ), 0755, true ); + } + if ( false === file_put_contents( $path, $this->replace_variables( $content->getRaw() ) . "\n" ) ) { + throw new Exception( "Could not write $path" ); + } + } + + /** + * @Given the directory :path exists + */ + public function theDirectoryExists( $path ) + { + $path = $this->replace_variables( $path ); + if ( ! is_dir( $path ) && ! mkdir( $path, 0755, true ) ) { + throw new Exception( "Could not create $path" ); + } + } + + /** + * @Given the file :path does not exist + */ + public function theFileDoesNotExist( $path ) + { + $path = $this->replace_variables( $path ); + if ( file_exists( $path ) && ! unlink( $path ) ) { + throw new Exception( "Could not delete $path" ); + } + } + + /** + * @Given the executable :name is linked into :dir + */ + public function theExecutableIsLinkedInto( $name, $dir ) + { + $dir = $this->replace_variables( $dir ); + $this->theDirectoryExists( $dir ); + $binary = trim( (string) shell_exec( 'command -v ' . escapeshellarg( $name ) ) ); + if ( '' === $binary ) { + throw new Exception( "$name is not installed." ); + } + if ( ! file_exists( "$dir/$name" ) && ! symlink( $binary, "$dir/$name" ) ) { + throw new Exception( "Could not link $binary into $dir" ); + } + } + + /** + * @Then the file :path should exist + */ + public function theFileShouldExist( $path ) + { + $path = $this->replace_variables( $path ); + if ( ! file_exists( $path ) ) { + throw new Exception( "$path does not exist." ); + } + } + + /** + * @Then the file :path should not exist + */ + public function theFileShouldNotExist( $path ) + { + $path = $this->replace_variables( $path ); + if ( file_exists( $path ) ) { + throw new Exception( "$path exists." ); + } + } + + /** + * @Then the file :path should contain :text + */ + public function theFileShouldContain( $path, $text ) + { + $this->assert_file_contains( $path, $text, false ); + } + + /** + * @Then the file :path should not contain :text + */ + public function theFileShouldNotContain( $path, $text ) + { + $this->assert_file_contains( $path, $text, true ); + } + + /** + * @Given no EasyEngine migration is pending + */ + public function noEasyEngineMigrationIsPending() + { + // Commands that check requirements (e.g. `cli info`) would otherwise run a real migration. + if ( false !== strpos( EE_VERSION, 'nightly' ) ) { + throw new Exception( 'VERSION ' . EE_VERSION . ' is a nightly: every checked command would run the migration. Refusing to continue.' ); + } + if ( ! file_exists( self::db_path() ) ) { + if ( getenv( 'EE_BEHAT_ALLOW_INIT' ) ) { + return; + } + throw new Exception( self::db_path() . ' does not exist: the next checked command would initialise EasyEngine. Set EE_BEHAT_ALLOW_INIT=1 to allow that.' ); + } + $db_version = self::get_option( 'version' ); + if ( EE_VERSION !== $db_version ) { + throw new Exception( "DB version '$db_version' differs from VERSION '" . EE_VERSION . "': a checked command would run the migration. Refusing to continue." ); + } + } + + /** + * @Given the EasyEngine option :key is set to :value + */ + public function theEasyEngineOptionIsSetTo( $key, $value ) + { + if ( ! array_key_exists( $key, $this->saved_options ) ) { + $this->saved_options[ $key ] = self::get_option( $key ); + } + self::set_option( $key, $this->replace_variables( $value ) ); + } + + /** + * @Then the EasyEngine option :key should be :value + */ + public function theEasyEngineOptionShouldBe( $key, $value ) + { + $actual = self::get_option( $key ); + $expected = $this->replace_variables( $value ); + if ( $expected !== $actual ) { + throw new Exception( "Expected option '$key' to be '$expected', got " . var_export( $actual, true ) ); + } + } + + /** + * @Given I record the number of containers with label :label + */ + public function iRecordTheNumberOfContainersWithLabel( $label ) + { + $this->container_counts[ $label ] = self::count_containers( $label ); + } + + /** + * @Then the number of containers with label :label should be unchanged + */ + public function theNumberOfContainersWithLabelShouldBeUnchanged( $label ) + { + if ( ! isset( $this->container_counts[ $label ] ) ) { + throw new Exception( "No recorded container count for $label." ); + } + $now = self::count_containers( $label ); + if ( $now !== $this->container_counts[ $label ] ) { + throw new Exception( "Containers with label $label: {$this->container_counts[ $label ]} before, $now now." ); + } + } + + /** + * @AfterScenario + */ + public function cleanupScenario(AfterScenarioScope $scope) + { + foreach ( $this->saved_options as $key => $value ) { + if ( false === $value ) { + self::delete_option( $key ); + } else { + self::set_option( $key, $value ); + } + } + $this->saved_options = []; + $this->container_counts = []; + $this->commands = []; + $this->rand = null; + + if ( $this->tmp_dir && 0 === strpos( $this->tmp_dir, self::TMP_BASE . '/' ) ) { + exec( 'rm -rf ' . escapeshellarg( $this->tmp_dir ) ); + } + $this->tmp_dir = null; + + chdir($this->ee_path); + } + /** * @AfterFeature */ public static function cleanup(AfterFeatureScope $scope) { + // Only this suite's sites, never every site on the host. + foreach ( glob( EE_SITE_ROOT . '/' . self::SITE_PREFIX . '*', GLOB_ONLYDIR ) ?: [] as $site_dir ) { + exec( 'bin/ee site delete ' . escapeshellarg( basename( $site_dir ) ) . ' --yes' ); + } + if(file_exists('ee.phar')) { unlink('ee.phar'); } @@ -166,4 +844,195 @@ public static function cleanup(AfterFeatureScope $scope) unlink('ee-old.phar'); } } + + private function replace_variables( $string ) + { + $map = [ + '{EE}' => realpath( EE_ROOT ) . '/bin/ee', + '{EE_ROOT_DIR}' => EE_ROOT_DIR, + '{EE_ROOT}' => realpath( EE_ROOT ), + '{EE_VERSION}' => EE_VERSION, + '{PHP}' => PHP_BINARY, + ]; + if ( false !== strpos( $string, '{TMP}' ) ) { + $map['{TMP}'] = $this->tmp_dir(); + } + if ( false !== strpos( $string, '{RAND}' ) ) { + if ( null === $this->rand ) { + $this->rand = bin2hex( random_bytes( 6 ) ); + } + $map['{RAND}'] = $this->rand; + } + + return strtr( $string, $map ); + } + + private function tmp_dir() + { + if ( null === $this->tmp_dir ) { + $this->tmp_dir = self::TMP_BASE . '/' . uniqid( 'scenario-', true ); + if ( ! mkdir( $this->tmp_dir, 0755, true ) ) { + throw new Exception( "Could not create {$this->tmp_dir}" ); + } + } + + return $this->tmp_dir; + } + + private function last_command() + { + if ( empty( $this->commands ) ) { + throw new Exception( 'No command has been run in this scenario.' ); + } + + return end( $this->commands ); + } + + private function command_at( $index ) + { + if ( ! isset( $this->commands[ $index - 1 ] ) ) { + throw new Exception( "Command $index has not been run in this scenario." ); + } + + return $this->commands[ $index - 1 ]; + } + + private function stream_output( $stream ) + { + $last = $this->last_command(); + + return 'STDOUT' === $stream ? $last->stdout : $last->stderr; + } + + private function describe( $run, $escape = false ) + { + $show = function ( $text ) use ( $escape ) { + return $escape ? addcslashes( $text, "\033" ) : $text; + }; + + return "Command: {$run->command}\nExit code: {$run->return_code}\nSTDOUT:\n" . $show( $run->stdout ) . "\nSTDERR:\n" . $show( $run->stderr ); + } + + private function assert_contains( $stream, $text, $negate ) + { + $found = false !== strpos( $this->stream_output( $stream ), $text ); + if ( $found === $negate ) { + throw new Exception( "$stream should " . ( $negate ? 'not ' : '' ) . "contain:\n$text\n\n" . $this->describe( $this->last_command() ) ); + } + } + + private function assert_file_contains( $path, $text, $negate ) + { + $path = $this->replace_variables( $path ); + $text = $this->replace_variables( $text ); + if ( ! is_readable( $path ) ) { + throw new Exception( "$path is not readable." ); + } + $found = false !== strpos( file_get_contents( $path ), $text ); + if ( $found === $negate ) { + throw new Exception( "$path should " . ( $negate ? 'not ' : '' ) . "contain \"$text\". Content:\n" . file_get_contents( $path ) ); + } + } + + private function decoded_stdout() + { + $stdout = $this->last_command()->stdout; + $decoded = json_decode( $stdout, true ); + if ( null === $decoded && 'null' !== trim( $stdout ) ) { + throw new Exception( 'STDOUT is not valid json: ' . json_last_error_msg() . "\n" . $this->describe( $this->last_command() ) ); + } + + return $decoded; + } + + /** + * Resolves a dot path: "key", "*" (every element) or "field=value" (elements whose field matches). + */ + private function json_nodes( $path ) + { + $nodes = [ $this->decoded_stdout() ]; + $path = $this->replace_variables( $path ); + foreach ( '' === $path ? [] : explode( '.', $path ) as $part ) { + $next = []; + foreach ( $nodes as $node ) { + if ( ! is_array( $node ) ) { + continue; + } + if ( '*' === $part ) { + $next = array_merge( $next, array_values( $node ) ); + } elseif ( false !== strpos( $part, '=' ) ) { + list( $field, $value ) = explode( '=', $part, 2 ); + foreach ( $node as $element ) { + if ( is_array( $element ) && isset( $element[ $field ] ) && (string) $element[ $field ] === $value ) { + $next[] = $element; + } + } + } elseif ( array_key_exists( $part, $node ) ) { + $next[] = $node[ $part ]; + } + } + $nodes = $next; + } + + return $nodes; + } + + private function single_json_node( $path ) + { + $nodes = $this->json_nodes( $path ); + if ( 1 !== count( $nodes ) ) { + throw new Exception( 'Expected one value at "' . $path . '", found ' . count( $nodes ) . ".\n" . $this->describe( $this->last_command() ) ); + } + + return $nodes[0]; + } + + private static function json_scalar( $value ) + { + return is_string( $value ) ? $value : json_encode( $value, JSON_UNESCAPED_SLASHES ); + } + + private static function db_path() + { + return EE_ROOT_DIR . '/db/ee.sqlite'; + } + + private static function db() + { + if ( ! file_exists( self::db_path() ) ) { + throw new Exception( self::db_path() . ' does not exist.' ); + } + $pdo = new PDO( 'sqlite:' . self::db_path() ); + $pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + + return $pdo; + } + + private static function get_option( $key ) + { + $query = self::db()->prepare( 'SELECT value FROM options WHERE "key" = ?' ); + $query->execute( [ $key ] ); + + return $query->fetchColumn(); + } + + private static function set_option( $key, $value ) + { + $db = self::db(); + if ( false === self::get_option( $key ) ) { + $db->prepare( 'INSERT INTO options ("key", value) VALUES (?, ?)' )->execute( [ $key, $value ] ); + } else { + $db->prepare( 'UPDATE options SET value = ? WHERE "key" = ?' )->execute( [ $value, $key ] ); + } + } + + private static function delete_option( $key ) + { + self::db()->prepare( 'DELETE FROM options WHERE "key" = ?' )->execute( [ $key ] ); + } + + private static function count_containers( $label ) + { + return (int) trim( (string) shell_exec( 'docker ps -aq --filter ' . escapeshellarg( 'label=' . $label ) . ' | wc -l' ) ); + } } From a92140b6db44a969be4f665c8e2ae5ccdd78b3b1 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 11:19:32 +0530 Subject: [PATCH 02/30] test(easyengine): skip destructive scenarios by default Tag the self-uninstall feature @destructive, exclude that tag in behat.yml (profiles destructive and all run it) and cover declining the uninstall prompt. --- behat.yml | 16 ++++++++++++++++ features/cli.feature | 11 +++++++++++ 2 files changed, 27 insertions(+) create mode 100644 behat.yml diff --git a/behat.yml b/behat.yml new file mode 100644 index 000000000..6dfb0b0be --- /dev/null +++ b/behat.yml @@ -0,0 +1,16 @@ +# Destructive scenarios (self-uninstall, DB version changes, real migrations) are skipped by default. +# Note: a CLI --tags option replaces these filters, so add `&&~@destructive` to it yourself. +default: + gherkin: + filters: + tags: "~@destructive" + +destructive: + gherkin: + filters: + tags: "@destructive" + +all: + gherkin: + filters: + tags: "@destructive,~@destructive" diff --git a/features/cli.feature b/features/cli.feature index 5223ec5d2..144ca08f4 100644 --- a/features/cli.feature +++ b/features/cli.feature @@ -1,5 +1,16 @@ +# Wipes every EasyEngine container, volume, network, image, /opt/easyengine and all sites (TEST-3). +# Excluded by default (behat.yml); run with `vendor/bin/behat -p destructive`. +@destructive Feature: CLI Command + Scenario: Declining the self-uninstall prompt removes nothing + Given I record the number of containers with label "org.label-schema.vendor=EasyEngine" + When I run 'echo n | bin/ee cli self-uninstall' + Then the command should exit with code 0 + And STDOUT should contain "Are you sure you want to remove EasyEngine and all its sites(along with their data)?" + And the number of containers with label "org.label-schema.vendor=EasyEngine" should be unchanged + And the file "{EE_ROOT_DIR}/db/ee.sqlite" should exist + Scenario: ee uninstall works properly When I run 'sudo bin/ee cli info' And I run 'sudo bin/ee cli self-uninstall --yes' From 13866c4086b7fc9b6de252dab89c3f4244377ceb Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 11:36:04 +0530 Subject: [PATCH 03/30] test(easyengine): cover cli version, info, check-update and update --- features/cli-info.feature | 63 ++++++++++++++++++++++++++++++++++ features/cli-update.feature | 65 ++++++++++++++++++++++++++++++++++++ features/cli-version.feature | 42 +++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 features/cli-info.feature create mode 100644 features/cli-update.feature create mode 100644 features/cli-version.feature diff --git a/features/cli-info.feature b/features/cli-info.feature new file mode 100644 index 000000000..4ce923b91 --- /dev/null +++ b/features/cli-info.feature @@ -0,0 +1,63 @@ +# `cli info` runs the migration check when docker is available, so every scenario is guarded. +Feature: ee cli info + + Background: + Given no EasyEngine migration is pending + + Scenario: cli info lists every diagnostic row + When I run 'bin/ee cli info' + Then the command should exit with code 0 + And STDOUT should not contain "Executing migrations" + And STDOUT should match "#^\| OS\s+\| \S+#m" + And STDOUT should match "#^\| Shell\s+\|#m" + And STDOUT should match "#^\| PHP binary\s+\| \S+#m" + And STDOUT should match "#^\| PHP version\s+\| \d+\.\d+#m" + And STDOUT should match "#^\| php\.ini used\s+\|#m" + And STDOUT should match "#^\| EE root dir\s+\| {EE_ROOT}\s+\|#m" + And STDOUT should match "#^\| EE vendor dir\s+\| {EE_ROOT}/vendor\s+\|#m" + And STDOUT should match "#^\| EE phar path\s+\|\s+\|#m" + And STDOUT should match "#^\| EE packages dir\s+\|#m" + And STDOUT should match "#^\| EE global config\s+\|#m" + And STDOUT should match "#^\| EE project config\s+\|#m" + And STDOUT should match "#^\| EE version\s+\| {EE_VERSION}\s+\|#m" + + Scenario: cli info --format=json returns the documented keys + When I run 'bin/ee cli info --format=json' + Then the command should exit with code 0 + And STDOUT should be valid json + And the json in STDOUT should have the keys "php_binary_path, global_config_path, project_config_path, ee_dir_path, ee_packages_dir_path, ee_version, system_os, shell" + And the json in STDOUT at "ee_version" should be "{EE_VERSION}" + And the json in STDOUT at "ee_dir_path" should be "{EE_ROOT}" + And the json in STDOUT at "project_config_path" should be "null" + + Scenario: cli info reports the global config chosen with EE_CONFIG_PATH + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli info --format=json' + Then the command should exit with code 0 + And the json in STDOUT at "global_config_path" should be "{TMP}/config.yml" + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli info' + Then STDOUT should match "#^\| EE global config\s+\| {TMP}/config\.yml\s+\|#m" + + Scenario: cli info rejects an unsupported format + When I run 'bin/ee cli info --format=yaml' + Then the command should exit with code 1 + And STDERR should contain: + """ + Error: Parameter errors: + Invalid value specified for 'format' (Render output in a particular format.) + """ + + Scenario: cli info suggests the closest flag for a typo + When I run 'bin/ee cli info --fromat=json' + Then the command should exit with code 1 + And STDERR should contain "unknown --fromat parameter" + And STDERR should contain "Did you mean '--format'?" + + Scenario: cli info rejects positional arguments + When I run 'bin/ee cli info extra' + Then the command should exit with code 1 + And STDERR should contain "Error: Too many positional arguments: extra" diff --git a/features/cli-update.feature b/features/cli-update.feature new file mode 100644 index 000000000..92a55221c --- /dev/null +++ b/features/cli-update.feature @@ -0,0 +1,65 @@ +Feature: ee cli check-update and ee cli update + + @network + Scenario: check-update reports either the available updates or that EE is current + When I run 'bin/ee cli check-update' + Then the command should exit with code 0 + And STDOUT should match "#(^Success: EasyEngine is at the latest version\.$|^\| version\s+\| update_type\s+\| package_url\s+\|$)#m" + + @network + Scenario: check-update --major reports that no major release exists + When I run 'bin/ee cli check-update --major' + Then the command should exit with code 0 + And STDOUT should be: + """ + Success: EasyEngine is at the latest major version. + """ + + @network + Scenario Outline: check-update filters by update type + When I run 'bin/ee cli check-update --' + Then the command should exit with code 0 + And STDOUT should match "#(^Success: EasyEngine is at the latest version\.$|\| \s+\|)#m" + + Examples: + | type | + | minor | + | patch | + + @network + Scenario: check-update --format=json prints nothing or a json list + When I run 'bin/ee cli check-update --format=json' + Then the command should exit with code 0 + And STDOUT should be empty or valid json + And STDOUT should not contain "Success:" + + @network + Scenario: check-update --format=count prints nothing or a number + When I run 'bin/ee cli check-update --format=count' + Then the command should exit with code 0 + And STDOUT should match "#^\s*\d*\s*$#" + + Scenario: check-update rejects an unsupported format before any request + When I run 'https_proxy=http://127.0.0.1:9 bin/ee cli check-update --format=xml' + Then the command should exit with code 1 + And STDERR should contain "Invalid value specified for 'format'" + + Scenario: check-update fails cleanly when GitHub is unreachable + When I run 'https_proxy=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 bin/ee cli check-update' + Then the command should exit with code 1 + And STDERR should contain "Error: Failed to get url 'https://api.github.com/repos/EasyEngine/easyengine/releases?per_page=100'" + + Scenario Outline: cli update refuses to run from a source checkout + When I run 'bin/ee cli update ' + Then the command should exit with code 1 + And STDERR should be: + """ + Error: You can only self-update Phar files. + """ + + Examples: + | flags | + | | + | --yes | + | --stable --yes | + | --nightly --yes | diff --git a/features/cli-version.feature b/features/cli-version.feature new file mode 100644 index 000000000..602b53759 --- /dev/null +++ b/features/cli-version.feature @@ -0,0 +1,42 @@ +Feature: ee cli version + + Scenario: cli version prints the source VERSION + When I run 'bin/ee cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ + And STDERR should be empty + + Scenario: The global --version flag is an alias of cli version + When I run 'bin/ee --version' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ + + Scenario: cli version takes no arguments, so extra ones are ignored + When I run 'bin/ee cli version extra --unknown-flag' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ + + Scenario: cli version --help shows the command help instead of the version + When I run 'bin/ee cli version --help' + Then the command should exit with code 0 + And STDOUT should contain "ee cli version" + And STDOUT should contain "Print EE version." + And STDOUT should not contain "EE {EE_VERSION}" + + Scenario: cli version does not need docker or docker-compose + Given the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ From a264a8fa1ba4b9235347370605f24cde16e5c510 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 11:52:37 +0530 Subject: [PATCH 04/30] test(easyengine): cover cli dumps, completions, aliases and has-command --- features/cli-alias.feature | 206 +++++++++++++++++++++++++++++++ features/cli-completions.feature | 113 +++++++++++++++++ features/cli-dump.feature | 99 +++++++++++++++ features/cli-has-command.feature | 60 +++++++++ 4 files changed, 478 insertions(+) create mode 100644 features/cli-alias.feature create mode 100644 features/cli-completions.feature create mode 100644 features/cli-dump.feature create mode 100644 features/cli-has-command.feature diff --git a/features/cli-alias.feature b/features/cli-alias.feature new file mode 100644 index 000000000..ec2108ccf --- /dev/null +++ b/features/cli-alias.feature @@ -0,0 +1,206 @@ +Feature: ee cli alias and @alias execution + + Scenario: No aliases are listed for a config without aliases + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli alias' + Then the command should exit with code 0 + And STDOUT should be: + """ + --- + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli alias --format=json' + Then STDOUT should be: + """ + [] + """ + + Scenario: Aliases from the config file are listed with @all first + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli alias' + Then the command should exit with code 0 + And STDOUT should be: + """ + --- + @all: Run command against every registered alias. + @bh-one: + path: /var/tmp + """ + + Scenario: Alias json output keeps only alias keys and absolutizes relative paths + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + bogus: ignored + @bh-rel: + path: relative/dir + @bh-group: + - @bh-one + - @bh-rel + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli alias --format=json' + Then the command should exit with code 0 + And the json in STDOUT should have the keys "@all, @bh-one, @bh-rel, @bh-group" + And the json in STDOUT at "@all" should be "Run command against every registered alias." + And the json in STDOUT at "@bh-one" should be '{"path":"/var/tmp"}' + And the json in STDOUT at "@bh-rel.path" should be "{TMP}/relative/dir" + And the json in STDOUT at "@bh-group" should be '["@bh-one","@bh-rel"]' + + Scenario: cli aliases is an alias of cli alias + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli alias --format=json' + And I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli aliases --format=json' + Then the command should exit with code 0 + And STDOUT of command 1 should be identical to STDOUT of command 2 + + Scenario: cli alias rejects an unsupported format + When I run 'bin/ee cli alias --format=table' + Then the command should exit with code 1 + And STDERR should contain "Invalid value specified for 'format'" + + Scenario: Aliases can be provided at runtime with EE_RUNTIME_ALIAS + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-file: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml EE_RUNTIME_ALIAS='{"@bh-rt":{"path":"/srv","bogus":"x"},"not-an-alias":{"path":"/x"}}' bin/ee cli alias --format=json' + Then the command should exit with code 0 + And the json in STDOUT should have the keys "@all, @bh-rt" + And the json in STDOUT at "@bh-rt" should be '{"path":"/srv"}' + And STDOUT should not contain "@bh-file" + And STDOUT should not contain "not-an-alias" + + Scenario: A command runs locally through an alias without ssh + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-one cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ + + Scenario: @all runs the command once per alias + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-two: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + @bh-one + EE {EE_VERSION} + @bh-two + EE {EE_VERSION} + """ + + Scenario: An alias group runs the command for each member + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-two: + path: /var/tmp + @bh-group: + - @bh-two + - @bh-one + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-group cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + @bh-two + EE {EE_VERSION} + @bh-one + EE {EE_VERSION} + """ + + Scenario: A group with an unknown member is rejected + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-broken: + - @bh-one + - @bh-missing + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-broken cli version' + Then the command should exit with code 1 + And STDERR should contain "Error: Group '@bh-broken' contains one or more invalid aliases: @bh-missing" + + Scenario: An unknown alias fails with a suggestion + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-onee cli version' + Then the command should exit with code 1 + And STDERR should be: + """ + Error: Alias '@bh-onee' not found. + Did you mean '@bh-one'? + """ + + Scenario: @all fails when no alias is registered + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli version' + Then the command should exit with code 1 + And STDERR should contain "Error: Cannot use '@all' when no aliases are registered." + + # New (suspected): Runner::run_ssh_command() is never called, so an alias with `ssh:` silently runs the + # command on the local machine instead of the remote one. + @known-bug + Scenario: A command for an ssh alias is not run locally + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-remote: + ssh: bh-nobody@127.0.0.1:1 + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-remote cli version' + Then the command should fail + And STDOUT should not contain "EE {EE_VERSION}" diff --git a/features/cli-completions.feature b/features/cli-completions.feature new file mode 100644 index 000000000..011597138 --- /dev/null +++ b/features/cli-completions.feature @@ -0,0 +1,113 @@ +Feature: ee cli completions + + Scenario: Completing an empty line lists every top-level command + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli completions --line="ee " --point=3' + Then the command should exit with code 0 + And STDOUT should match "#^cli$#m" + And STDOUT should match "#^help$#m" + And STDOUT should match "#^site$#m" + And STDOUT should match "#^service$#m" + And STDERR should be empty + + Scenario: A partial subcommand is completed + When I run 'bin/ee cli completions --line="ee cli ve" --point=100' + Then the command should exit with code 0 + And STDOUT should be: + """ + version + """ + + Scenario: Every cli subcommand is offered after "ee cli " + When I run 'bin/ee cli completions --line="ee cli " --point=7' + Then STDOUT should be: + """ + alias + check-update + cmd-dump + completions + has-command + info + param-dump + self-uninstall + update + version + """ + + Scenario: The line is cut at the cursor position + When I run 'bin/ee cli completions --line="ee cli version" --point=9' + Then STDOUT should be: + """ + version + """ + + Scenario: zsh completions include the short description + When I run 'bin/ee cli completions --line="ee cli ve" --point=100 --shell=zsh' + Then STDOUT should be: + """ + version:Print EE version. + """ + + Scenario: Options of a leaf command are completed + When I run 'bin/ee cli completions --line="ee cli info --" --point=100' + Then STDOUT should be: + """ + --format= + """ + When I run 'bin/ee cli completions --line="ee cli check-update --ma" --point=100' + Then STDOUT should match "#^--major $#m" + + Scenario: Options already on the line are not offered again + When I run 'bin/ee cli completions --line="ee cli info --format=json --" --point=100' + Then the command should exit with code 0 + And STDOUT should be empty + + Scenario: help arguments are completed like commands + When I run 'bin/ee cli completions --line="ee help cli ve" --point=100' + Then STDOUT should be: + """ + version + """ + + Scenario: Site subcommands are completed for the default html type + When I run 'bin/ee cli completions --line="ee site cr" --point=100' + Then STDOUT should be: + """ + create + """ + + Scenario: Site options follow the --type on the line + When I run 'bin/ee cli completions --line="ee site create --type=wp --admin-u" --point=100' + Then STDOUT should match "#^--admin-user=$#m" + When I run 'bin/ee cli completions --line="ee site create --type=html --admin-u" --point=100' + Then STDOUT should be empty + + Scenario: Aliases from the config file are completed + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli completions --line="ee @" --point=100' + Then STDOUT should match "#^@bh-one $#m" + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli completions --line="ee @bh-one cli ve" --point=100' + Then STDOUT should be: + """ + version + """ + + Scenario: Nothing is completed for an unknown command + When I run 'bin/ee cli completions --line="ee nosuchcommand " --point=100' + Then the command should exit with code 0 + And STDOUT should be empty + + Scenario: completions rejects an unsupported shell + When I run 'bin/ee cli completions --line="ee " --point=3 --shell=fish' + Then the command should exit with code 1 + And STDERR should contain "Invalid value specified for 'shell' (Shell for completions output.)" + + Scenario: completions does not write to ee.log + When I run 'bin/ee cli completions --line="ee cli bh{RAND}" --point=100' + Then the command should exit with code 0 + And the file "{EE_ROOT_DIR}/logs/ee.log" should not contain "bh{RAND}" diff --git a/features/cli-dump.feature b/features/cli-dump.feature new file mode 100644 index 000000000..55f5f3ce4 --- /dev/null +++ b/features/cli-dump.feature @@ -0,0 +1,99 @@ +Feature: ee cli param-dump and cmd-dump + + Scenario: param-dump prints the global parameter spec as json by default + When I run 'bin/ee cli param-dump' + Then the command should exit with code 0 + And STDOUT should be valid json + And the json in STDOUT should have the keys "disabled_commands, sites_path, locale, le-mail, env, wp-mail, sysctl, skip-tty, custom-compose, ee_installer_version, color, debug, quiet" + And the json in STDOUT at "debug" should have the keys "runtime, file, synopsis, default, multiple, desc" + And the json in STDOUT at "color.default" should be "auto" + And the json in STDOUT at "debug.runtime" should be "[=]" + And the json in STDOUT at "disabled_commands.runtime" should be "false" + And the json in STDOUT at "quiet.desc" should be "Suppress informational messages." + + Scenario: param-dump without --with-values has no current values + When I run 'bin/ee cli param-dump' + Then STDOUT should not contain "current" + + Scenario: param-dump --format=var_export prints a PHP array + When I run 'bin/ee cli param-dump --format=var_export' + Then the command should exit with code 0 + And STDOUT should match "#^array \(#" + And STDOUT should contain "'disabled_commands' =>" + And STDOUT should contain "'desc' => 'Whether to colorize the output.'," + + Scenario: param-dump rejects an unsupported format + When I run 'bin/ee cli param-dump --format=yaml' + Then the command should exit with code 1 + And STDERR should contain "Invalid value specified for 'format'" + + Scenario: param-dump suggests the closest flag for a typo + When I run 'bin/ee cli param-dump --with-value' + Then the command should exit with code 1 + And STDERR should contain "unknown --with-value parameter" + And STDERR should contain "Did you mean '--with-values'?" + + Scenario: param-dump --with-values shows values from the global config file + Given a file "{TMP}/config.yml" with: + """ + locale: fr_FR + ee_installer_version: stable + le-mail: global@example.test + disabled_commands: + - cli alias + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli param-dump --with-values' + Then the command should exit with code 0 + And the json in STDOUT at "le-mail.current" should be "global@example.test" + And the json in STDOUT at "locale.current" should be "fr_FR" + And the json in STDOUT at "disabled_commands.current" should be '["cli alias"]' + And the json in STDOUT at "sites_path.current" should be "null" + And the json in STDOUT at "quiet.current" should be "false" + + Scenario: A runtime flag overrides the global config value + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + le-mail: global@example.test + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli param-dump --with-values --le-mail=runtime@example.test' + Then the command should exit with code 0 + And the json in STDOUT at "le-mail.current" should be "runtime@example.test" + + Scenario: cmd-dump prints the whole command tree as json + When I run 'bin/ee cli cmd-dump' + Then the command should exit with code 0 + And STDOUT should be valid json + And the json in STDOUT at "name" should be "ee" + And the json in STDOUT at "description" should be "Manage EasyEngine through the command-line." + And the json in STDOUT at "subcommands.*.name" should contain "admin-tools" + And the json in STDOUT at "subcommands.*.name" should contain "auth" + And the json in STDOUT at "subcommands.*.name" should contain "cli" + And the json in STDOUT at "subcommands.*.name" should contain "config" + And the json in STDOUT at "subcommands.*.name" should contain "cron" + And the json in STDOUT at "subcommands.*.name" should contain "dash" + And the json in STDOUT at "subcommands.*.name" should contain "help" + And the json in STDOUT at "subcommands.*.name" should contain "log" + And the json in STDOUT at "subcommands.*.name" should contain "mailhog" + And the json in STDOUT at "subcommands.*.name" should contain "service" + And the json in STDOUT at "subcommands.*.name" should contain "shell" + And the json in STDOUT at "subcommands.*.name" should contain "site" + + Scenario: cmd-dump includes subcommands with their synopsis + When I run 'bin/ee cli cmd-dump' + Then the json in STDOUT at "subcommands.name=cli.subcommands.*.name" should contain "has-command" + And the json in STDOUT at "subcommands.name=cli.subcommands.*.name" should contain "self-uninstall" + And the json in STDOUT at "subcommands.name=cli.subcommands.name=info.synopsis" should be "[--format=]" + And the json in STDOUT at "subcommands.name=cli.subcommands.name=has-command.synopsis" should be "..." + And the json in STDOUT at "subcommands.name=cli.subcommands.name=version.description" should be "Print EE version." + And the json in STDOUT at "subcommands.name=config.subcommands.*.name" should contain "set" + + Scenario: site cmd-dump bypasses the requirement checks and dumps every site type + When I run 'bin/ee site cmd-dump' + Then the command should exit with code 0 + And STDOUT should be valid json + And STDOUT should not contain "Executing migrations" + And the json in STDOUT at "subcommands.name=site.subcommands.*.name" should contain "create --type=wp" + And the json in STDOUT at "subcommands.name=site.subcommands.*.name" should contain "create --type=php" + And the json in STDOUT at "subcommands.*.name" should not contain "site_type_wp" diff --git a/features/cli-has-command.feature b/features/cli-has-command.feature new file mode 100644 index 000000000..66a14583a --- /dev/null +++ b/features/cli-has-command.feature @@ -0,0 +1,60 @@ +Feature: ee cli has-command + + Scenario Outline: Registered commands exit with 0 and print nothing + When I run 'bin/ee cli has-command ' + Then the command should exit with code 0 + And STDOUT should be empty + And STDERR should be empty + + Examples: + | command | + | cli | + | "cli version" | + | cli version | + | "site create" | + | "site delete" | + | "config get" | + | "service restart" | + | help | + + Scenario Outline: Unknown commands exit with 1 and print nothing + When I run 'bin/ee cli has-command ' + Then the command should exit with code 1 + And STDOUT should be empty + And STDERR should be empty + + Examples: + | command | + | foo | + | "foo bar" | + | "" | + + # New (suspected): find_command_to_run() stops at the first leaf and ignores the remaining words. + # `site` is a leaf too (Site_Command::__invoke routes site types), so any "site " is "registered". + @known-bug + Scenario Outline: Extra words after a leaf command are not a registered command + When I run 'bin/ee cli has-command ' + Then the command should exit with code 1 + + Examples: + | command | + | "cli version extra" | + | "site nosuchcommand" | + + Scenario: has-command without a command prints its usage + When I run 'bin/ee cli has-command' + Then the command should exit with code 1 + And STDOUT should contain "usage: ee cli has-command ..." + + Scenario: A command disabled in the config file is reported as missing + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli version + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli has-command "cli version"' + Then the command should exit with code 1 + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli has-command "cli info"' + Then the command should exit with code 0 From c4f4563351c5275046cf58810fc0ba98c10e0f5c Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 12:09:07 +0530 Subject: [PATCH 05/30] test(easyengine): cover help for every command and unknown commands --- features/help.feature | 227 +++++++++++++++++++++++++++++++ features/unknown-command.feature | 71 ++++++++++ 2 files changed, 298 insertions(+) create mode 100644 features/help.feature create mode 100644 features/unknown-command.feature diff --git a/features/help.feature b/features/help.feature new file mode 100644 index 000000000..2c2e999c2 --- /dev/null +++ b/features/help.feature @@ -0,0 +1,227 @@ +Feature: ee help + + Scenario: Running ee without arguments shows the top-level help + When I run 'bin/ee' + Then the command should exit with code 0 + And STDOUT should contain: + """ + NAME + + ee + + DESCRIPTION + + Manage EasyEngine through the command-line. + + SYNOPSIS + + ee + """ + And STDOUT should contain "GLOBAL PARAMETERS" + And STDOUT should contain "--[no-]color" + And STDOUT should contain "--debug[=]" + And STDOUT should contain "--quiet" + And STDOUT should contain "Run 'ee help ' to get more information on a specific command." + And STDERR should be empty + + Scenario: ee, ee help and ee --help print the same page + When I run 'bin/ee' + And I run 'bin/ee help' + And I run 'bin/ee --help' + Then STDOUT of command 1 should be identical to STDOUT of command 2 + And STDOUT of command 1 should be identical to STDOUT of command 3 + + Scenario Outline: The top-level help lists every installed command + When I run 'bin/ee help' + Then STDOUT should match "#^ \s+\S#m" + + Examples: + | command | + | admin-tools | + | auth | + | cli | + | config | + | cron | + | dash | + | help | + | log | + | mailhog | + | service | + | shell | + | site | + + Scenario Outline: Every top-level command has a help page, also reachable with --help + When I run 'bin/ee help ' + And I run 'bin/ee --help' + Then the command should exit with code 0 + And STDOUT should contain: + """ + NAME + + ee + + """ + And STDOUT should contain: + """ + SYNOPSIS + + ee + """ + And STDOUT of command 1 should be identical to STDOUT of command 2 + + Examples: + | command | + | admin-tools | + | auth | + | cli | + | config | + | cron | + | dash | + | help | + | log | + | mailhog | + | service | + | shell | + | site | + + Scenario Outline: Every subcommand has a help page, also reachable with --help + When I run 'bin/ee help ' + And I run 'bin/ee --help' + Then the command should exit with code 0 + And STDOUT should contain: + """ + NAME + + ee + + """ + And STDOUT should contain: + """ + SYNOPSIS + + ee + """ + And STDOUT of command 1 should be identical to STDOUT of command 2 + + Examples: + | command | + | admin-tools disable | + | admin-tools enable | + | auth create | + | auth delete | + | auth list | + | auth update | + | cli alias | + | cli check-update | + | cli cmd-dump | + | cli completions | + | cli has-command | + | cli info | + | cli param-dump | + | cli self-uninstall | + | cli update | + | cli version | + | config get | + | config list | + | config set | + | config unset | + | cron create | + | cron delete | + | cron list | + | cron run-now | + | cron update | + | dash init | + | log show | + | mailhog disable | + | mailhog enable | + | mailhog status | + | service disable | + | service enable | + | service refresh | + | service reload | + | service restart | + | site backup | + | site clean | + | site clone | + | site create | + | site delete | + | site disable | + | site enable | + | site info | + | site list | + | site refresh | + | site reload | + | site restart | + | site restore | + | site share | + | site ssl-info | + | site ssl-renew | + | site ssl-verify | + | site sync | + | site update | + + Scenario Outline: site help follows the requested site type + When I run 'bin/ee help site create --type=' + Then the command should exit with code 0 + And STDOUT should contain "" + + Examples: + | type | description | + | html | Runs the standard HTML site installation. | + | wp | Runs the standard WordPress Site installation. | + | php | Runs the standard PHP Site installation. | + + Scenario: The default site type for help is html + When I run 'bin/ee help site create' + Then STDOUT should contain "Runs the standard HTML site installation." + + Scenario: WordPress-only options appear only in the wp site create help + When I run 'bin/ee help site create --type=wp' + Then STDOUT should contain "--admin-user=" + When I run 'bin/ee help site create --type=html' + Then STDOUT should not contain "--admin-user=" + + Scenario: help for an unknown site type fails + When I run 'bin/ee help site create --type=bogus' + Then the command should exit with code 1 + And STDERR should contain "Error: 'bogus' is not a registered site type" + + Scenario: help for an unknown command fails with a hint + When I run 'bin/ee help nosuchcommand' + Then the command should exit with code 1 + And STDERR should be: + """ + Error: 'nosuchcommand' is not a registered ee command. See 'ee help' for available commands. + """ + + Scenario: help for an unknown subcommand fails with a hint + When I run 'bin/ee help cli nosuchsubcommand' + Then the command should exit with code 1 + And STDERR should be: + """ + Error: 'nosuchsubcommand' is not a registered subcommand of 'cli'. See 'ee help cli' for available subcommands. + """ + + Scenario: help suggests the closest command for a typo + When I run 'bin/ee help sitee' + Then the command should exit with code 1 + And STDERR should be: + """ + Error: 'sitee' is not a registered ee command. See 'ee help' for available commands. + Did you mean 'site'? + """ + When I run 'bin/ee help cli verison' + Then STDERR should contain "Did you mean 'version'?" + + Scenario: help honours the PAGER environment variable + When I run 'PAGER=cat bin/ee help cli version' + Then the command should exit with code 0 + And STDOUT should contain "ee cli version" + + Scenario: help pages include the command examples + When I run 'bin/ee help cli has-command' + Then STDOUT should contain "EXAMPLES" + And STDOUT should contain: + """ + $ ee cli has-command "site delete" + """ diff --git a/features/unknown-command.feature b/features/unknown-command.feature new file mode 100644 index 000000000..a2ea459a3 --- /dev/null +++ b/features/unknown-command.feature @@ -0,0 +1,71 @@ +Feature: Unknown commands and composite commands + + Scenario: An unknown cli subcommand suggests the closest one + When I run 'bin/ee cli verison' + Then the command should exit with code 1 + And STDOUT should be empty + And STDERR should be: + """ + Error: 'verison' is not a registered subcommand of 'cli'. See 'ee help cli' for available subcommands. + Did you mean 'version'? + """ + + Scenario Outline: Common shorthands map to the real subcommand + When I run 'bin/ee cli ' + Then the command should exit with code 1 + And STDERR should contain "Did you mean ''?" + + Examples: + | typed | suggestion | + | v | version | + | check | check-update | + | cmd-dmp | cmd-dump | + | param-dum | param-dump | + | alais | alias | + + Scenario: No suggestion is offered when nothing is close + When I run 'bin/ee cli zzzzzzzzzz' + Then the command should exit with code 1 + And STDERR should contain "Error: 'zzzzzzzzzz' is not a registered subcommand of 'cli'." + And STDERR should not contain "Did you mean" + + # Unknown top-level commands pass the requirement and migration checks before the lookup. + Scenario: An unknown top-level command suggests the closest one + Given no EasyEngine migration is pending + When I run 'bin/ee sitee' + Then the command should exit with code 1 + And STDERR should contain: + """ + Error: 'sitee' is not a registered ee command. See 'ee help' for available commands. + Did you mean 'site'? + """ + + Scenario: A composite command without a subcommand prints its usage + When I run 'bin/ee cli' + Then the command should exit with code 0 + And STDOUT should contain "usage: ee cli alias [--format=]" + And STDOUT should contain " or: ee cli version" + And STDOUT should contain "See 'ee help cli ' for more information on a specific command." + + Scenario Outline: Every cli subcommand appears in the cli usage + When I run 'bin/ee cli' + Then STDOUT should match "#^(usage| or): ee cli ( |$)#m" + + Examples: + | subcommand | + | alias | + | check-update | + | cmd-dump | + | completions | + | has-command | + | info | + | param-dump | + | self-uninstall | + | update | + | version | + + Scenario: A subcommand called without its required parameters fails + When I run 'bin/ee cli completions' + Then the command should exit with code 1 + And STDERR should contain "missing --line parameter" + And STDERR should contain "missing --point parameter" From 88c1b3979c62590888fafe2d16b2e52aaa3aaf7a Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 12:25:07 +0530 Subject: [PATCH 06/30] test(easyengine): cover global flags, config files and requirement checks --- features/config-file.feature | 177 ++++++++++++++++++++++++++++++++++ features/global-flags.feature | 125 ++++++++++++++++++++++++ features/requirements.feature | 46 +++++++++ 3 files changed, 348 insertions(+) create mode 100644 features/config-file.feature create mode 100644 features/global-flags.feature create mode 100644 features/requirements.feature diff --git a/features/config-file.feature b/features/config-file.feature new file mode 100644 index 000000000..516cce478 --- /dev/null +++ b/features/config-file.feature @@ -0,0 +1,177 @@ +Feature: Global and project config files + + Scenario: A missing EE_CONFIG_PATH file is created with the default keys + Given the file "{TMP}/new/config.yml" does not exist + When I run 'EE_CONFIG_PATH={TMP}/new/config.yml bin/ee cli version' + Then the command should exit with code 0 + And the file "{TMP}/new/config.yml" should contain "locale: en_US" + And the file "{TMP}/new/config.yml" should contain "ee_installer_version: stable" + + Scenario: Missing default keys are added without touching existing values + Given a file "{TMP}/config.yml" with: + """ + locale: fr_FR + le-mail: behat@example.test + rclone-path: bh-remote:bucket/path + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli version' + Then the command should exit with code 0 + And the file "{TMP}/config.yml" should contain "locale: fr_FR" + And the file "{TMP}/config.yml" should contain "le-mail: behat@example.test" + And the file "{TMP}/config.yml" should contain "rclone-path: bh-remote:bucket/path" + And the file "{TMP}/config.yml" should contain "ee_installer_version: stable" + And the file "{TMP}/config.yml" should not contain "locale: en_US" + + Scenario: The config path is resolved before the defaults file is created + Given the file "{TMP}/new/config.yml" does not exist + When I run 'EE_CONFIG_PATH={TMP}/new/config.yml bin/ee cli version --debug' + Then STDERR should contain "Debug (bootstrap): No readable global config found" + When I run 'EE_CONFIG_PATH={TMP}/new/config.yml bin/ee cli version --debug' + Then STDERR should contain "Debug (bootstrap): Using global config from EE_CONFIG_PATH env var: {TMP}/new/config.yml" + + Scenario: Without a project config file the debug output says so + When I run 'cd {TMP} && EE_CONFIG_PATH={TMP}/config.yml {EE} cli version --debug' + Then STDERR should contain "Debug (bootstrap): No project config found" + + Scenario: ee.yml in the working directory is used as project config + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + le-mail: global@example.test + """ + And a file "{TMP}/project/ee.yml" with: + """ + le-mail: project@example.test + """ + When I run 'cd {TMP}/project && EE_CONFIG_PATH={TMP}/config.yml {EE} cli param-dump --with-values' + Then the command should exit with code 0 + And the json in STDOUT at "le-mail.current" should be "project@example.test" + When I run 'cd {TMP}/project && EE_CONFIG_PATH={TMP}/config.yml {EE} cli version --debug' + Then STDERR should contain "Debug (bootstrap): Using project config: {TMP}/project/ee.yml" + + Scenario: The project config is found from a subdirectory + Given a file "{TMP}/project/ee.yml" with: + """ + le-mail: project@example.test + """ + And the directory "{TMP}/project/sub/dir" exists + When I run 'cd {TMP}/project/sub/dir && EE_CONFIG_PATH={TMP}/config.yml {EE} cli param-dump --with-values' + Then the json in STDOUT at "le-mail.current" should be "project@example.test" + + Scenario: ee.local.yml takes priority over ee.yml + Given a file "{TMP}/project/ee.yml" with: + """ + le-mail: project@example.test + """ + And a file "{TMP}/project/ee.local.yml" with: + """ + le-mail: local@example.test + """ + When I run 'cd {TMP}/project && EE_CONFIG_PATH={TMP}/config.yml {EE} cli param-dump --with-values' + Then the json in STDOUT at "le-mail.current" should be "local@example.test" + + Scenario: A runtime flag overrides the project config + Given a file "{TMP}/project/ee.yml" with: + """ + le-mail: project@example.test + """ + When I run 'cd {TMP}/project && EE_CONFIG_PATH={TMP}/config.yml {EE} cli param-dump --with-values --le-mail=runtime@example.test' + Then the json in STDOUT at "le-mail.current" should be "runtime@example.test" + + Scenario: cli info reports the project config file + Given no EasyEngine migration is pending + And a file "{TMP}/project/ee.yml" with: + """ + le-mail: project@example.test + """ + When I run 'cd {TMP}/project && EE_CONFIG_PATH={TMP}/config.yml {EE} cli info --format=json' + Then the command should exit with code 0 + And the json in STDOUT at "project_config_path" should be "{TMP}/project/ee.yml" + + Scenario: A config file can inherit another one + Given a file "{TMP}/base.yml" with: + """ + le-mail: inherited@example.test + wp-mail: base@example.test + """ + And a file "{TMP}/config.yml" with: + """ + _: + inherit: {TMP}/base.yml + locale: en_US + ee_installer_version: stable + wp-mail: child@example.test + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli param-dump --with-values' + Then the json in STDOUT at "le-mail.current" should be "inherited@example.test" + And the json in STDOUT at "wp-mail.current" should be "child@example.test" + + Scenario: A disabled command cannot be run + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli version + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli version' + Then the command should exit with code 1 + And STDOUT should be empty + And STDERR should be: + """ + Error: The 'cli version' command has been disabled from the config file. + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli param-dump' + Then the command should exit with code 0 + + Scenario: help for a disabled command reports that it is disabled + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli version + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee help cli version' + Then the command should exit with code 1 + And STDERR should contain "Error: The 'cli version' command has been disabled from the config file." + + Scenario: A disabled subcommand is left out of the composite usage + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli version + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli' + Then the command should exit with code 0 + And STDOUT should contain "ee cli update" + And STDOUT should not contain "ee cli version" + + # New (suspected): CompositeCommand::show_usage() counts the skipped disabled subcommand, so the + # first printed line loses its "usage:" prefix when the first subcommand is disabled. + @known-bug + Scenario: The composite usage still starts with "usage:" when its first subcommand is disabled + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli alias + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli' + Then STDOUT should match "#\Ausage: ee cli check-update#" + + Scenario: Disabling a command also disables its alias + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli alias + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli aliases' + Then the command should exit with code 1 + And STDERR should contain "has been disabled from the config file." diff --git a/features/global-flags.feature b/features/global-flags.feature new file mode 100644 index 000000000..8cc5393f8 --- /dev/null +++ b/features/global-flags.feature @@ -0,0 +1,125 @@ +Feature: Global parameters (--debug, --quiet, --color) + + Scenario: Without --debug nothing is written to STDERR + When I run 'bin/ee cli version' + Then STDERR should be empty + + Scenario: --debug prints bootstrap messages to STDERR only + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli version --debug' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ + And STDERR should contain "Debug (bootstrap): Using global config from EE_CONFIG_PATH env var: {TMP}/config.yml" + And STDERR should contain "Debug (bootstrap): Running command: cli version" + And STDERR should match "#^Debug \(bootstrap\): argv: .*cli version --debug \(\d+\.\d+s\)$#m" + + Scenario: --debug is accepted before the command too + When I run 'bin/ee --debug cli version' + Then the command should exit with code 0 + And STDERR should contain "Debug (bootstrap): Running command: cli version" + + Scenario: --debug= shows only that group, without the group label + When I run 'bin/ee cli version --debug=bootstrap' + Then STDERR should contain "Debug: Running command: cli version" + And STDERR should not contain "Debug (bootstrap)" + + Scenario: --debug with an unknown group shows nothing + When I run 'bin/ee cli version --debug=bh-no-such-group' + Then the command should exit with code 0 + And STDERR should be empty + + Scenario: debug can be enabled from the config file + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + debug: true + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli version' + Then STDERR should contain "Debug (bootstrap): Running command: cli version" + + Scenario: In strict args mode a global flag after the command is not global + When I run 'EE_STRICT_ARGS_MODE=1 bin/ee cli version --debug' + Then the command should exit with code 0 + And STDERR should be empty + When I run 'EE_STRICT_ARGS_MODE=1 bin/ee --debug cli version' + Then STDERR should contain "Debug (bootstrap): Running command: cli version" + + Scenario: Output is not colorized when it is piped + When I run 'bin/ee cli verison' + Then STDERR should contain "Error: 'verison' is not a registered subcommand of 'cli'." + And STDERR should not contain ANSI color codes + + Scenario: SHELL_PIPE=0 makes the automatic color detection colorize + When I run 'SHELL_PIPE=0 bin/ee cli verison' + Then STDERR should contain ANSI color codes + + # New (suspected): InitializeColorization/InitializeLogger run before ConfigureRunner in + # php/bootstrap.php, so --color, --no-color and `color:` in the config file are ignored. + @known-bug + Scenario: --color forces colors even when piped + When I run 'bin/ee cli verison --color' + Then STDERR should contain ANSI color codes + + @known-bug + Scenario: --no-color disables colors + When I run 'SHELL_PIPE=0 bin/ee cli verison --no-color' + Then STDERR should not contain ANSI color codes + + @known-bug + Scenario: color can be disabled from the config file + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + color: false + """ + When I run 'SHELL_PIPE=0 EE_CONFIG_PATH={TMP}/config.yml bin/ee cli verison' + Then STDERR should not contain ANSI color codes + + # New (suspected): same bootstrap order bug, the Quiet logger is never selected, so --quiet + # and `quiet: true` do not hide warnings. The missing docker-compose path warns without a migration. + @known-bug + Scenario: --quiet hides warnings + Given the executable "docker" is linked into "{TMP}/bin" + When I run 'PATH={TMP}/bin {PHP} {EE_ROOT}/php/boot-fs.php site list --quiet' + Then STDERR should contain "Error: EasyEngine requires docker-compose." + And STDERR should not contain "Warning:" + + @known-bug + Scenario: quiet can be enabled from the config file + Given the executable "docker" is linked into "{TMP}/bin" + And a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + quiet: true + """ + When I run 'PATH={TMP}/bin EE_CONFIG_PATH={TMP}/config.yml {PHP} {EE_ROOT}/php/boot-fs.php site list' + Then STDERR should contain "Error: EasyEngine requires docker-compose." + And STDERR should not contain "Warning:" + + @known-bug @network + Scenario: --quiet hides success messages + When I run 'bin/ee cli check-update --major --quiet' + Then the command should exit with code 0 + And STDOUT should not contain "Success:" + + Scenario: Errors are still printed with --quiet + When I run 'bin/ee cli verison --quiet' + Then the command should exit with code 1 + And STDERR should contain "Error: 'verison' is not a registered subcommand of 'cli'." + + # LOG-1 (won't fix for now): the full argv is logged at debug level to ee.log, including secrets. + @known-bug + Scenario: Secrets passed on the command line are not written to ee.log + When I run 'bin/ee cli version --pass=bh-secret-{RAND}' + Then the command should exit with code 0 + And the file "{EE_ROOT_DIR}/logs/ee.log" should not contain "bh-secret-{RAND}" diff --git a/features/requirements.feature b/features/requirements.feature new file mode 100644 index 000000000..8d7309098 --- /dev/null +++ b/features/requirements.feature @@ -0,0 +1,46 @@ +# The runner only checks requirements (docker, docker-compose) for commands other than cli/config/help. +# These scenarios hide the binaries with a private PATH, so they never reach a migration. +Feature: Requirement checks + + Scenario: help works without docker or docker-compose + # The default pager (`less -r`) is looked up in PATH too, so point PAGER at an absolute cat. + Given the directory "{TMP}/emptybin" exists + When I run 'PAGER=/bin/cat PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php help cli' + Then the command should exit with code 0 + And STDOUT should contain "ee cli" + + # New: help pipes through `less -r` unless PAGER is set; without less it prints nothing and exits 0. + @known-bug + Scenario: help still prints when the default pager is not installed + Given the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php help cli' + Then STDOUT should contain "ee cli" + + Scenario: cli info still works when docker is not available + # Requirements are only probed (4 attempts, ~7s of backoff), so no migration can run. + Given the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli info --format=json' + Then the command should exit with code 0 + And the json in STDOUT at "ee_version" should be "{EE_VERSION}" + + Scenario: A checked command reports a missing docker-compose + Given the executable "docker" is linked into "{TMP}/bin" + When I run 'PATH={TMP}/bin {PHP} {EE_ROOT}/php/boot-fs.php site list' + Then STDERR should contain "Warning: EasyEngine requires docker-compose version 2.26.1 or greater." + And STDERR should contain "Error: EasyEngine requires docker-compose." + And STDOUT should not contain "Executing migrations" + + # CORE-1: a failed requirement exits with status 0 (`EE::error( ..., false ); die;`). + @known-bug + Scenario: A failed requirement exits with a non-zero status + Given the executable "docker" is linked into "{TMP}/bin" + When I run 'PATH={TMP}/bin {PHP} {EE_ROOT}/php/boot-fs.php site list' + Then STDERR should contain "Error: EasyEngine requires docker-compose." + And the command should fail + + Scenario: A checked command retries the docker probe before giving up + Given the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php site list --debug' + Then STDERR should contain "Docker check failed (attempt 1/4), retrying in 1s..." + And STDERR should contain "Docker check failed (attempt 3/4), retrying in 4s..." + And STDERR should contain "Error: Docker not installed or not running (checked 4 times)." From 1d5203b62c1c52423bab52aed62dd0b2b12ec46e Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 12:37:55 +0530 Subject: [PATCH 07/30] test(easyengine): cover migration triggering and the phar build --- features/migration.feature | 76 ++++++++++++++++++++++++++++++++++++++ features/phar.feature | 45 ++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 features/migration.feature create mode 100644 features/phar.feature diff --git a/features/migration.feature b/features/migration.feature new file mode 100644 index 000000000..fe0ab643f --- /dev/null +++ b/features/migration.feature @@ -0,0 +1,76 @@ +# The runner compares options.version in /opt/easyengine/db/ee.sqlite with VERSION: +# - cli, config and help commands (and `site cmd-dump`) never check; `cli info` checks only when docker is up; +# - a DB version newer than VERSION: warning for `cli`, error for everything else, nothing migrated; +# - an older DB version: the full upgrade pipeline runs (db, custom container and image migrations, +# docker-compose update, cron config rewrite, logrotate) and the DB version is bumped. +# Every scenario here changes options.version, which the context restores after the scenario. +@destructive +Feature: Migration triggering + + Scenario Outline: Commands that skip the requirement checks never run the migration + Given no EasyEngine migration is pending + And the EasyEngine option "version" is set to "4.0.0" + When I run '' + Then STDOUT should not contain "Executing migrations" + And STDERR should not contain "It seems you're not running latest version" + And the EasyEngine option "version" should be "4.0.0" + + Examples: + | command | + | bin/ee | + | bin/ee --version | + | bin/ee cli version | + | bin/ee help | + | bin/ee help site create | + | bin/ee site create --help | + | bin/ee cli has-command site | + | bin/ee cli completions --line="ee " --point=3 | + | bin/ee cli param-dump | + | bin/ee cli cmd-dump | + | bin/ee cli alias | + | bin/ee cli update | + | bin/ee site cmd-dump | + | bin/ee config list | + + Scenario: A newer DB version makes cli info warn and other commands refuse to run + Given no EasyEngine migration is pending + And the EasyEngine option "version" is set to "99.0.0" + When I run 'bin/ee cli info' + Then the command should exit with code 0 + And STDERR should contain: + """ + Warning: It seems you're not running latest version. Update EasyEngine using `ee cli update --stable --yes`. + """ + And STDOUT should contain "EE version" + When I run 'bin/ee site list' + Then the command should exit with code 1 + And STDERR should contain: + """ + Error: It seems you're not running latest version. Update EasyEngine using `ee cli update --stable --yes`. + """ + And the EasyEngine option "version" should be "99.0.0" + + Scenario: The version check runs before an unknown command is looked up + Given no EasyEngine migration is pending + And the EasyEngine option "version" is set to "99.0.0" + When I run 'bin/ee sitee' + Then the command should exit with code 1 + And STDERR should contain "Error: It seems you're not running latest version." + And STDERR should not contain "is not a registered ee command" + + Scenario: The nightly suffix is ignored when comparing versions + Given no EasyEngine migration is pending + And the EasyEngine option "version" is set to "99.0.0-nightly-bh" + When I run 'bin/ee site list' + Then the command should exit with code 1 + And STDERR should contain "Error: It seems you're not running latest version." + + # Runs a real migration: image pulls (MIG-4), cron container restart and log rotation (MIG-5). + @migration + Scenario: An older DB version triggers the upgrade on the next checked command + Given no EasyEngine migration is pending + And the EasyEngine option "version" is set to "4.0.0" + When I run 'bin/ee cli info' + Then the command should exit with code 0 + And STDOUT should contain "Executing migrations. This might take some time." + And the EasyEngine option "version" should be "{EE_VERSION}" diff --git a/features/phar.feature b/features/phar.feature new file mode 100644 index 000000000..08a1582c8 --- /dev/null +++ b/features/phar.feature @@ -0,0 +1,45 @@ +# Builds ee.phar from the checkout (~1 min, cached for the feature) and checks the phar bootstrap. +# `cli update` is not exercised: from a phar it would really download and swap the phar. +@phar +Feature: Phar build + + Background: + Given ee phar is generated + + Scenario: The phar reports the source VERSION + When I run 'php ee.phar cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + EE {EE_VERSION} + """ + + Scenario: The phar serves help pages + When I run 'php ee.phar help cli' + Then the command should exit with code 0 + And STDOUT should contain "ee cli" + And STDOUT should contain "self-uninstall" + + Scenario: The phar registers the bundled command packages + When I run 'php ee.phar cli has-command "site create"' + Then the command should exit with code 0 + When I run 'php ee.phar cli has-command "service restart"' + Then the command should exit with code 0 + When I run 'php ee.phar cli has-command "admin-tools enable"' + Then the command should exit with code 0 + + Scenario: The phar cli info reports the phar as EE root + Given no EasyEngine migration is pending + When I run 'php ee.phar cli info --format=json' + Then the command should exit with code 0 + And the json in STDOUT at "ee_dir_path" should be "phar://ee.phar" + And the json in STDOUT at "ee_version" should be "{EE_VERSION}" + + # New (suspected): php/boot-phar.php defines EE_PHAR_PATH as getcwd(), so "EE phar path" shows the + # working directory; phar_safe_path() relies on the same constant. + @known-bug + Scenario: The phar cli info shows the path of the phar file + Given no EasyEngine migration is pending + When I run 'cd {TMP} && php {EE_ROOT}/ee.phar cli info' + Then the command should exit with code 0 + And STDOUT should match "#^\| EE phar path\s+\| {EE_ROOT}/ee\.phar\s+\|#m" From b38c72202609f0c75ade2fc579031818ede0d869 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 12:52:28 +0530 Subject: [PATCH 08/30] docs(easyengine): add behat coverage inventory --- features/COVERAGE.md | 221 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 features/COVERAGE.md diff --git a/features/COVERAGE.md b/features/COVERAGE.md new file mode 100644 index 000000000..25ada00ef --- /dev/null +++ b/features/COVERAGE.md @@ -0,0 +1,221 @@ +# Behat coverage: easyengine (core) + +Core owns the runner (bootstrap, config, aliases, requirement checks, migration trigger, logging), the `cli` command group and `help`. Every other command belongs to a package suite; core only checks that each one has a help page and is registered. + +## Inventory and mapping + +### `ee cli version` + +| Item | Covered by | +|---|---| +| prints `EE ` | cli-version.feature: "cli version prints the source VERSION" | +| global `--version` rewrite to `cli version` | cli-version.feature: "The global --version flag ..." | +| no synopsis, so extra args/flags are ignored | cli-version.feature: "cli version takes no arguments ..." | +| `--help` rewrite to `help cli version` | cli-version.feature, help.feature subcommand outline | +| no requirement check (works without docker) | cli-version.feature: "cli version does not need docker ..." | +| from the phar | phar.feature: "The phar reports the source VERSION" | + +### `ee cli info [--format=list|json]` + +| Item | Covered by | +|---|---| +| list output, every row (OS, Shell, PHP binary/version, php.ini, EE root/vendor dir, phar path, packages dir, global/project config, version) | cli-info.feature: "cli info lists every diagnostic row" | +| `--format=json` keys and values | cli-info.feature: "cli info --format=json ..." | +| global config path from `EE_CONFIG_PATH` (json and list) | cli-info.feature: "cli info reports the global config ..." | +| project config path | config-file.feature: "cli info reports the project config file" | +| invalid `--format`, typo suggestion, extra positional | cli-info.feature (3 scenarios) | +| runs the migration check only when docker is up | requirements.feature: "cli info still works when docker is not available"; migration.feature: newer/older version scenarios | +| no migration when versions match | cli-info.feature: "cli info lists every diagnostic row" (no "Executing migrations") | +| phar values (`ee_dir_path` = `phar://ee.phar`, phar path) | phar.feature (1 + 1 `@known-bug`) | + +### `ee cli check-update [--patch] [--minor] [--major] [--field] [--fields] [--format]` + +| Item | Covered by | +|---|---| +| default table or "latest version" success | cli-update.feature `@network` | +| `--major`, `--minor`, `--patch` | cli-update.feature `@network` (outline + `--major`) | +| `--format=json`, `--format=count` | cli-update.feature `@network` | +| invalid `--format` (validated before any request) | cli-update.feature | +| GitHub unreachable: clean error, exit 1 | cli-update.feature (proxy to a closed port, no network needed) | +| `--field`, `--fields`, `--format=csv|yaml` | not covered: output depends on live release data; formatter covered by package suites | +| nightly branch (`-alpha-` versions, `NIGHTLY_VERSION`) | not covered: needs a nightly VERSION | +| `GITHUB_TOKEN` header | not covered: no observable output | + +### `ee cli update [--stable] [--nightly] [--yes]` + +| Item | Covered by | +|---|---| +| refuses to run from source (`You can only self-update Phar files.`) with every flag combination | cli-update.feature outline (4 rows) | +| real update (download, md5 check, child `cli info`, phar swap), macOS brew message, not-writable errors | not covered: would replace the phar and run the new version's migrations (MIG-6); root can write everything; not on macOS | +| `auto_check_update()` on help (phar + tty only) | not covered: needs a tty and a writable phar | + +### `ee cli param-dump [--with-values] [--format=json|var_export]` + +| Item | Covered by | +|---|---| +| json spec with every global parameter and its fields | cli-dump.feature | +| `--format=var_export` | cli-dump.feature | +| `--with-values` from the global config, defaults, null values | cli-dump.feature | +| runtime flag overrides config (`--le-mail`) | cli-dump.feature, config-file.feature | +| invalid format, typo suggestion (`--with-value`) | cli-dump.feature | + +### `ee cli cmd-dump` and `ee site cmd-dump` + +| Item | Covered by | +|---|---| +| json tree, root name/description, all 12 top-level commands | cli-dump.feature | +| nested subcommands, synopsis, description | cli-dump.feature | +| `site cmd-dump` bypasses requirement checks, merges site types (`create --type=wp`) | cli-dump.feature | + +### `ee cli completions --line --point [--shell=bash|zsh]` + +| Item | Covered by | +|---|---| +| top-level list, partial subcommand, all cli subcommands | cli-completions.feature | +| `--point` truncation | cli-completions.feature | +| `--shell=zsh` descriptions | cli-completions.feature | +| options of a leaf, flags with trailing space, already used options skipped | cli-completions.feature | +| `help` prefix | cli-completions.feature | +| site subcommands and `--type`-aware site options | cli-completions.feature | +| aliases (`ee @`, `ee @alias cli ...`) | cli-completions.feature | +| unknown command, invalid shell, missing `--line/--point` | cli-completions.feature, unknown-command.feature | +| does not log to `ee.log` | cli-completions.feature | +| `` positional completion, site type from an existing site (`Site::find`) | not covered: no core command has a `` positional; needs a site | + +### `ee cli alias` / `ee cli aliases [--format=yaml|json]` and `@alias` execution + +| Item | Covered by | +|---|---| +| empty list (yaml, json) | cli-alias.feature | +| yaml listing with `@all` first | cli-alias.feature | +| json: only alias keys kept, relative `path` absolutized, groups | cli-alias.feature | +| `aliases` alias of `alias` | cli-alias.feature | +| invalid format | cli-alias.feature | +| `EE_RUNTIME_ALIAS` replaces file aliases, filters keys and names | cli-alias.feature | +| run through a local alias, `@all`, a group, a group with an unknown member, unknown alias suggestion, `@all` without aliases | cli-alias.feature | +| aliases with `ssh:` run remotely | cli-alias.feature `@known-bug` (they run locally) | +| ssh schemes (docker, docker-compose, vagrant, ssh), `EE_SSH_PRE_CMD` | not covered: `run_ssh_command()` is dead code in EE; would need a remote host | + +### `ee cli has-command ...` + +| Item | Covered by | +|---|---| +| registered commands (quoted and split forms, packages' commands) exit 0 silently | cli-has-command.feature outline (8 rows) | +| unknown commands and empty name exit 1 silently | cli-has-command.feature outline (3 rows) | +| trailing words after a leaf, `site ` | cli-has-command.feature `@known-bug` outline (2 rows) | +| missing argument prints usage | cli-has-command.feature | +| disabled command reported as missing | cli-has-command.feature | + +### `ee cli self-uninstall [--yes]` + +| Item | Covered by | +|---|---| +| declining the prompt removes nothing | cli.feature `@destructive` | +| full uninstall (containers, networks, volumes, images, site dirs, `/opt/easyengine`) | cli.feature `@destructive` (existing scenario, TEST-3) | +| refuses when docker is down, phar self-delete | not covered: needs docker stopped / a phar install | + +### `ee help [...]` + +| Item | Covered by | +|---|---| +| `ee`, `ee help`, `ee --help` print the same top-level page, with global parameters | help.feature | +| every top-level command (12) and subcommand (54) has a help page equal to ` --help` | help.feature outlines | +| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature | +| unknown command/subcommand errors and suggestions | help.feature | +| `PAGER`, examples section | help.feature | +| works without docker | requirements.feature | +| no output without `less` | requirements.feature `@known-bug` | +| disabled command | config-file.feature | + +### Global parameters and runner behaviour + +| Item | Covered by | +|---|---| +| `--debug`, `--debug` before the command, `--debug=`, unknown group, `debug:` in config | global-flags.feature | +| `EE_STRICT_ARGS_MODE` | global-flags.feature | +| color auto-detection (piped, `SHELL_PIPE=0`) | global-flags.feature | +| `--color`, `--no-color`, `color:` in config | global-flags.feature `@known-bug` (3) | +| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (1 + 3 `@known-bug`) | +| unknown parameter suggestions | cli-info.feature, cli-dump.feature | +| `--sites_path`, `--locale`, `--le-mail`, `--env`, `--wp-mail`, `--sysctl`, `--skip-tty`, `--custom-compose` | runtime parsing covered through `--le-mail` (cli-dump.feature, config-file.feature); their effects belong to the package suites | +| unknown command/subcommand suggestions, shorthand map (`v`, `check`), no suggestion | unknown-command.feature | +| composite command usage, disabled subcommand hidden | unknown-command.feature, config-file.feature (+1 `@known-bug`) | +| missing required parameters | unknown-command.feature | +| requirement checks: missing docker-compose, docker retry/backoff | requirements.feature | +| failed requirement exit status (CORE-1) | requirements.feature `@known-bug` | +| argv logged to `ee.log` with secrets (LOG-1) | global-flags.feature `@known-bug` | +| `BEHAT_RUN`, `EE_PACKAGES_DIR`, `EE_DISABLE_AUTO_CHECK_UPDATE`, `EE_AUTO_CHECK_UPDATE_DAYS`, `EE_PHP` | not covered: only affect error reporting, a packages dir nobody uses, or the phar+tty auto update | + +### Config files + +| Item | Covered by | +|---|---| +| `EE_CONFIG_PATH` file created with `locale`/`ee_installer_version` defaults | config-file.feature | +| existing values and extra keys kept when defaults are added | config-file.feature | +| path resolved before the defaults file exists (first run reads nothing) | config-file.feature | +| project config `ee.yml`, from a subdirectory, `ee.local.yml` priority, runtime override, none found | config-file.feature | +| `_: inherit:` | config-file.feature | +| `disabled_commands` (run, help, usage, alias) | config-file.feature, cli-has-command.feature | +| unreadable/unwritable config errors | not covered: root bypasses file permissions | +| `_: merge:` for extra config | not covered: only used by package config keys | + +### Migration trigger + +| Item | Covered by | +|---|---| +| cli/help/config/`site cmd-dump`/no-args never check (14 commands) | migration.feature `@destructive` outline | +| DB version newer: warning for `cli info`, error otherwise, before command lookup, nightly suffix ignored | migration.feature `@destructive` (3) | +| DB version older: full pipeline runs, version bumped | migration.feature `@destructive @migration` | +| versions equal: nothing runs | cli-info.feature | +| fresh install (`ee successfully setup.`), nightly VERSION always migrating, 5 GB disk check, rollback | not covered: needs a wiped machine, a modified VERSION or a full disk; MIG-1..19 are architectural and need the live upgrade tests | + +### Phar + +| Item | Covered by | +|---|---| +| build from the checkout, version, help, bundled packages registered, `cli info` root | phar.feature `@phar` | +| `EE_PHAR_PATH` is the working directory | phar.feature `@known-bug` | + +### Coverage estimate + +The inventory has 97 rows: 85 covered and 12 not covered (about 87% of rows, about 80% when rows with several flags are counted per flag). What is left is the real self-update, ssh aliases (dead code), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. + +## How to run + +The suite runs from the core root as root (see `/root/workspace/tools/run-behat.sh core`). `behat.yml` excludes `@destructive` by default. + +| Command | Runs | +|---|---| +| `vendor/bin/behat` | everything except `@destructive` (229 scenarios) | +| `vendor/bin/behat -p destructive` | only `@destructive` (20 scenarios: self-uninstall, DB version changes, a real migration) | +| `vendor/bin/behat -p all` | all 249 scenarios | +| `vendor/bin/behat --tags='~@destructive&&~@known-bug&&~@network&&~@phar'` | fast, deterministic subset | + +A CLI `--tags` replaces the profile filter, so always add `&&~@destructive` to it unless you mean to run destructive scenarios. + +Tags: `@destructive` (20, changes global state; restored where possible), `@migration` (1, runs the real upgrade pipeline, also `@destructive`), `@known-bug` (14, assert the correct behaviour and fail today), `@network` (7, query the GitHub releases API), `@phar` (5, build `ee.phar`, about 1 min). No `@le` or `@remote` scenarios: core has no Let's Encrypt or backup code. + +Safety rules built into the suite: + +- `Given no EasyEngine migration is pending` guards every scenario whose command reaches `maybe_trigger_migration()` (`cli info`, unknown top-level commands, phar `cli info`). It fails the scenario instead of running a migration when `options.version` differs from `VERSION`, when `VERSION` is a nightly, or when `/opt/easyengine/db/ee.sqlite` is missing (set `EE_BEHAT_ALLOW_INIT=1` to allow the first command to initialise a fresh machine, as the old CI flow did). +- Scenarios that change `options.version` restore it in `@AfterScenario`. +- Config scenarios point `EE_CONFIG_PATH` at a per-scenario file under `/var/tmp/ee-behat-core/` (`{TMP}`), removed after the scenario; the real `/opt/easyengine/config/config.yml` is never written. +- Requirement scenarios hide docker/docker-compose with a private `PATH` instead of stopping anything. +- The suite creates no sites. The `@AfterFeature` cleanup only deletes sites named `bh-easyengine-*`. +- Commands are run through `EE\Process` directly, not `EE::launch()`, so the harness does not write test command lines into `ee.log`. + +Estimated runtime: default profile about 3 to 4 minutes (help outlines about 40 s, two 7 s docker-retry scenarios, about 1 min for the phar build, a few seconds of GitHub API calls). `-p destructive` adds the migration scenario (1 to 3 min, image pulls and a cron container restart) and the self-uninstall (about 1 min, after which the next site command re-initialises EasyEngine and pulls about 8 GB of images). + +## Known bugs exposed (`@known-bug`) + +| Scenario | Issue | +|---|---| +| requirements.feature: failed requirement exits non-zero | CORE-1 | +| global-flags.feature: secrets not written to ee.log | LOG-1 | +| global-flags.feature: `--color`, `--no-color`, `color:` config (3) | new: `php/bootstrap.php` runs InitializeColorization and InitializeLogger before ConfigureRunner, so the runner config is still empty when colors and the logger are chosen | +| global-flags.feature: `--quiet`, `quiet:` config, success with `--quiet` (3) | new: same bootstrap order bug, the Quiet logger is never selected | +| cli-has-command.feature: trailing words / `site ` (2) | new: `find_command_to_run()` stops at the first leaf; `site` is a leaf (`Site_Command::__invoke`) | +| config-file.feature: composite usage prefix | new: `CompositeCommand::show_usage()` counts a skipped disabled subcommand, so the first line loses `usage:` | +| requirements.feature: help without `less` | new: help pipes through `less -r` unless `PAGER` is set; without it nothing is printed and the exit code is 0 | +| cli-alias.feature: ssh alias runs locally | new: `Runner::run_ssh_command()` is never called, so `ee @prod ` with `ssh:` runs on the local machine | +| phar.feature: phar path | new: `php/boot-phar.php` defines `EE_PHAR_PATH` as `getcwd()`, so `cli info` shows the working directory (and `phar_safe_path()` uses it) | From be0d33215aee89913a44580a94c6068826df4947 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 13:02:07 +0530 Subject: [PATCH 09/30] test(easyengine): fix the optional "not" in output steps When `(not )?` is the last capture group and doesn't match, preg_match drops it, so Behat can't fill `$negate`/`$text`: every positive "STDOUT should contain:" and "should contain ANSI color codes" step failed (all help outline examples among them). A dry run can't catch it. Capture an empty string instead. --- features/bootstrap/FeatureContext.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index f0b47a644..7c82118c6 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -488,7 +488,7 @@ public function theCommandShouldFail() } /** - * @Then /^(STDOUT|STDERR) should (not )?contain "(.*)"$/ + * @Then /^(STDOUT|STDERR) should ((?:not )?)contain "(.*)"$/ */ public function streamShouldContainText( $stream, $negate, $text ) { @@ -496,7 +496,7 @@ public function streamShouldContainText( $stream, $negate, $text ) } /** - * @Then /^(STDOUT|STDERR) should (not )?contain:$/ + * @Then /^(STDOUT|STDERR) should ((?:not )?)contain:$/ */ public function streamShouldContainBlock( $stream, $negate, PyStringNode $text ) { @@ -526,7 +526,7 @@ public function streamShouldBeEmpty( $stream ) } /** - * @Then /^(STDOUT|STDERR) should (not )?match "(.*)"$/ + * @Then /^(STDOUT|STDERR) should ((?:not )?)match "(.*)"$/ */ public function streamShouldMatch( $stream, $negate, $regex ) { @@ -541,7 +541,7 @@ public function streamShouldMatch( $stream, $negate, $regex ) } /** - * @Then /^(STDOUT|STDERR) should (not )?contain ANSI color codes$/ + * @Then /^(STDOUT|STDERR) should ((?:not )?)contain ANSI color codes$/ */ public function streamShouldContainAnsiCodes( $stream, $negate ) { @@ -630,7 +630,7 @@ public function theJsonAtShouldMatch( $path, $regex ) } /** - * @Then /^the json in STDOUT at "([^"]*)" should (not )?contain "([^"]*)"$/ + * @Then /^the json in STDOUT at "([^"]*)" should ((?:not )?)contain "([^"]*)"$/ */ public function theJsonAtShouldContain( $path, $negate, $value ) { From bae335ab2761fdbe2132ff36c9a4e3801f96eaa7 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 13:17:05 +0530 Subject: [PATCH 10/30] test(easyengine): keep destructive scenarios and the phar build off the live stack - Refuse @destructive, @migration and @uninstall scenarios unless EE_BEHAT_DESTRUCTIVE, EE_BEHAT_ALLOW_MIGRATION or EE_BEHAT_ALLOW_UNINSTALL is set: a CLI --tags replaces the behat.yml filter. - Split the migration and self-uninstall scenarios into their own profiles; the uninstall ran before later features and broke them (or re-initialised EasyEngine). - Only the @migration scenario writes an older options.version; the others use 99.0.0 so a regression errors instead of migrating. Restore changed options on shutdown and SIGTERM too. - Guard the self-uninstall's `cli info` and check networks, volumes and images after it. - Build the phar from a --no-dev copy under /var/tmp: the dev vendor produced a phar that fatals on start, and the old step wrote and deleted ee.phar/ee-old.phar in the checkout. --- behat.yml | 24 ++++- features/bootstrap/FeatureContext.php | 130 +++++++++++++++++++++----- features/cli.feature | 7 +- features/migration.feature | 31 ++++-- features/phar.feature | 22 ++--- 5 files changed, 169 insertions(+), 45 deletions(-) diff --git a/behat.yml b/behat.yml index 6dfb0b0be..e5bfb3d3d 100644 --- a/behat.yml +++ b/behat.yml @@ -1,5 +1,10 @@ -# Destructive scenarios (self-uninstall, DB version changes, real migrations) are skipped by default. -# Note: a CLI --tags option replaces these filters, so add `&&~@destructive` to it yourself. +# The default profile skips everything that changes the live EasyEngine state. Those scenarios are also refused +# at runtime unless their env var is set (see FeatureContext::GUARDED_TAGS), because a CLI --tags option +# replaces these filters: +# @destructive EE_BEHAT_DESTRUCTIVE=1 DB version changes, the self-uninstall prompt +# @migration EE_BEHAT_ALLOW_MIGRATION=1 a real upgrade migration (image pulls, cron restart, log rotation) +# @uninstall EE_BEHAT_ALLOW_UNINSTALL=1 `ee cli self-uninstall --yes`; run it alone and last +# On a fresh machine (no /opt/easyengine/db/ee.sqlite) also set EE_BEHAT_ALLOW_INIT=1. default: gherkin: filters: @@ -8,9 +13,20 @@ default: destructive: gherkin: filters: - tags: "@destructive" + tags: "@destructive&&~@migration&&~@uninstall" +migration: + gherkin: + filters: + tags: "@migration" + +uninstall: + gherkin: + filters: + tags: "@uninstall" + +# Everything that can run in one pass (the uninstall would break every scenario after it). all: gherkin: filters: - tags: "@destructive,~@destructive" + tags: "~@migration&&~@uninstall" diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 7c82118c6..9f74a3e19 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -58,6 +58,7 @@ use Behat\Behat\Context\Context; use Behat\Behat\Hook\Scope\AfterFeatureScope; use Behat\Behat\Hook\Scope\AfterScenarioScope; +use Behat\Behat\Hook\Scope\BeforeScenarioScope; use Behat\Gherkin\Node\PyStringNode, Behat\Gherkin\Node\TableNode; @@ -80,6 +81,30 @@ class FeatureContext implements Context */ const TMP_BASE = '/var/tmp/ee-behat-core'; + /** + * Tags that change the live EasyEngine state, each with the env var that must be set to run them. + * A CLI --tags replaces the behat.yml filters, so the tags alone can't keep these scenarios off. + */ + const GUARDED_TAGS = [ + 'destructive' => 'EE_BEHAT_DESTRUCTIVE', + 'migration' => 'EE_BEHAT_ALLOW_MIGRATION', + 'uninstall' => 'EE_BEHAT_ALLOW_UNINSTALL', + ]; + + /** + * Phar built by "ee phar is generated", shared by the whole run ({PHAR}). + * + * @var string|null + */ + private static $phar; + + /** + * Whether the shutdown/signal restore of changed options is registered. + * + * @var bool + */ + private $restore_registered = false; + /** * Per-scenario scratch directory, created on first use of {TMP}. * @@ -122,18 +147,32 @@ public function __construct() */ public function eePharIsPresent() { - // Building the phar is slow, so reuse the copy cached by an earlier scenario. - if(file_exists('ee-old.phar')) { - @unlink('ee.phar'); - copy('ee-old.phar','ee.phar'); - return 0; - } - exec("php -dphar.readonly=0 utils/make-phar.php ee.phar --quiet", $output, $return_status); - if (0 !== $return_status) { - throw new Exception("Unable to generate phar" . $return_status); - } - - copy('ee.phar','ee-old.phar'); + // Building takes about a minute, so the phar is shared by the whole run. + if ( self::$phar && file_exists( self::$phar ) ) { + return; + } + // Build from a private copy with a --no-dev autoloader: a phar built on a dev vendor requires + // require-dev files it doesn't contain. The checkout and /usr/local/bin/ee are never touched. + $root = realpath( EE_ROOT ); + $dir = self::TMP_BASE . '/phar-' . getmypid(); + $src = "$dir/src"; + $phar = "$dir/ee.phar"; + $cmd = sprintf( + 'rm -rf %1$s && mkdir -p %1$s && tar -C %2$s --exclude=.git --exclude=./features --exclude="./*.phar" -cf - . | tar -C %1$s -xf -' + . ' && cd %1$s && COMPOSER_ALLOW_SUPERUSER=1 %3$scomposer dump-autoload --no-dev -q' + . ' && %4$s -dphar.readonly=0 utils/make-phar.php %5$s --quiet', + escapeshellarg( $src ), + escapeshellarg( $root ), + file_exists( "$root/composer.dev.json" ) ? 'COMPOSER=composer.dev.json ' : '', + escapeshellarg( PHP_BINARY ), + escapeshellarg( $phar ) + ); + exec( "( $cmd ) 2>&1", $output, $status ); + exec( 'rm -rf ' . escapeshellarg( $src ) ); + if ( 0 !== $status || ! file_exists( $phar ) ) { + throw new Exception( "Unable to generate the phar (exit $status):\n" . implode( "\n", $output ) ); + } + self::$phar = $phar; } /** @@ -434,6 +473,12 @@ public function eeShouldBeDeleted() if( trim($result->stdout) !== '0' ) { throw new Exception("All containers have not been removed."); } + foreach ( [ 'network', 'volume', 'image' ] as $object ) { + $left = trim( EE::launch( "docker $object ls -q -f label=org.label-schema.vendor=EasyEngine | wc -l", false, true )->stdout ); + if ( '0' !== $left ) { + throw new Exception( "$left EasyEngine docker {$object}(s) have not been removed." ); + } + } $home = getenv('HOME'); if(file_exists("$home/.ee/")){ throw new Exception("~/.ee/ has not been removed"); @@ -453,7 +498,7 @@ public function eeShouldBeDeleted() * Output steps act on the LAST command run in the scenario. Step strings * support the placeholders {TMP} (per-scenario scratch dir under * /var/tmp), {EE} (absolute bin/ee), {EE_ROOT}, {EE_ROOT_DIR}, - * {EE_VERSION}, {PHP} (PHP binary running Behat) and {RAND}. + * {EE_VERSION}, {PHP} (PHP binary running Behat), {PHAR} (the built phar) and {RAND}. * --------------------------------------------------------------------- */ @@ -765,6 +810,17 @@ public function theEasyEngineOptionIsSetTo( $key, $value ) if ( ! array_key_exists( $key, $this->saved_options ) ) { $this->saved_options[ $key ] = self::get_option( $key ); } + if ( ! $this->restore_registered ) { + // Also restore when the run dies mid-scenario (Behat's SIGINT handler exits, fatals, SIGTERM). + register_shutdown_function( [ $this, 'restoreOptions' ] ); + if ( function_exists( 'pcntl_signal' ) ) { + pcntl_async_signals( true ); + pcntl_signal( SIGTERM, function () { + exit( 143 ); + } ); + } + $this->restore_registered = true; + } self::set_option( $key, $this->replace_variables( $value ) ); } @@ -803,9 +859,22 @@ public function theNumberOfContainersWithLabelShouldBeUnchanged( $label ) } /** - * @AfterScenario + * @BeforeScenario */ - public function cleanupScenario(AfterScenarioScope $scope) + public function refuseGuardedScenarios( BeforeScenarioScope $scope ) + { + $tags = array_merge( $scope->getFeature()->getTags(), $scope->getScenario()->getTags() ); + foreach ( self::GUARDED_TAGS as $tag => $env ) { + if ( in_array( $tag, $tags, true ) && ! getenv( $env ) ) { + throw new Exception( "Refusing to run a @$tag scenario: it changes the live EasyEngine state. Set $env=1 to allow it." ); + } + } + } + + /** + * Puts back the options changed by "the EasyEngine option :key is set to :value". + */ + public function restoreOptions() { foreach ( $this->saved_options as $key => $value ) { if ( false === $value ) { @@ -814,7 +883,15 @@ public function cleanupScenario(AfterScenarioScope $scope) self::set_option( $key, $value ); } } - $this->saved_options = []; + $this->saved_options = []; + } + + /** + * @AfterScenario + */ + public function cleanupScenario(AfterScenarioScope $scope) + { + $this->restoreOptions(); $this->container_counts = []; $this->commands = []; $this->rand = null; @@ -837,11 +914,16 @@ public static function cleanup(AfterFeatureScope $scope) exec( 'bin/ee site delete ' . escapeshellarg( basename( $site_dir ) ) . ' --yes' ); } - if(file_exists('ee.phar')) { - unlink('ee.phar'); - } - if(file_exists('ee-old.phar')) { - unlink('ee-old.phar'); + } + + /** + * @AfterSuite + */ + public static function removePhar() + { + if ( self::$phar ) { + exec( 'rm -rf ' . escapeshellarg( dirname( self::$phar ) ) ); + self::$phar = null; } } @@ -857,6 +939,12 @@ private function replace_variables( $string ) if ( false !== strpos( $string, '{TMP}' ) ) { $map['{TMP}'] = $this->tmp_dir(); } + if ( false !== strpos( $string, '{PHAR}' ) ) { + if ( null === self::$phar ) { + throw new Exception( '{PHAR} needs "Given ee phar is generated" first.' ); + } + $map['{PHAR}'] = self::$phar; + } if ( false !== strpos( $string, '{RAND}' ) ) { if ( null === $this->rand ) { $this->rand = bin2hex( random_bytes( 6 ) ); diff --git a/features/cli.feature b/features/cli.feature index 144ca08f4..8dd0b3a37 100644 --- a/features/cli.feature +++ b/features/cli.feature @@ -1,5 +1,6 @@ -# Wipes every EasyEngine container, volume, network, image, /opt/easyengine and all sites (TEST-3). -# Excluded by default (behat.yml); run with `vendor/bin/behat -p destructive`. +# The @uninstall scenario wipes every EasyEngine container, volume, network, image, /opt/easyengine and all +# sites (TEST-3). It is excluded from every profile but `uninstall`, refused without EE_BEHAT_ALLOW_UNINSTALL=1, +# and must run alone and last (later scenarios would re-initialise EasyEngine). @destructive Feature: CLI Command @@ -11,7 +12,9 @@ Feature: CLI Command And the number of containers with label "org.label-schema.vendor=EasyEngine" should be unchanged And the file "{EE_ROOT_DIR}/db/ee.sqlite" should exist + @uninstall Scenario: ee uninstall works properly + Given no EasyEngine migration is pending When I run 'sudo bin/ee cli info' And I run 'sudo bin/ee cli self-uninstall --yes' Then ee should be deleted diff --git a/features/migration.feature b/features/migration.feature index fe0ab643f..b0ab423fd 100644 --- a/features/migration.feature +++ b/features/migration.feature @@ -3,17 +3,22 @@ # - a DB version newer than VERSION: warning for `cli`, error for everything else, nothing migrated; # - an older DB version: the full upgrade pipeline runs (db, custom container and image migrations, # docker-compose update, cron config rewrite, logrotate) and the DB version is bumped. -# Every scenario here changes options.version, which the context restores after the scenario. +# Every scenario here changes options.version, which the context restores after the scenario (also on +# Ctrl-C/SIGTERM). Only the @migration scenario ever writes an OLDER version, so a regression elsewhere +# (or a concurrent `ee` run by cron or another user) gets the "not running latest version" error instead +# of a real image/container migration. Needs EE_BEHAT_DESTRUCTIVE=1 (and EE_BEHAT_ALLOW_MIGRATION=1 for @migration). @destructive Feature: Migration triggering - Scenario Outline: Commands that skip the requirement checks never run the migration + # The skip decision doesn't depend on the version, so a newer DB version detects a wrongly checked + # command (warning or error) without ever migrating. + Scenario Outline: Commands that skip the requirement checks never run the version check Given no EasyEngine migration is pending - And the EasyEngine option "version" is set to "4.0.0" + And the EasyEngine option "version" is set to "99.0.0" When I run '' - Then STDOUT should not contain "Executing migrations" - And STDERR should not contain "It seems you're not running latest version" - And the EasyEngine option "version" should be "4.0.0" + Then STDERR should not contain "It seems you're not running latest version" + And STDOUT should not contain "Executing migrations" + And the EasyEngine option "version" should be "99.0.0" Examples: | command | @@ -32,6 +37,15 @@ Feature: Migration triggering | bin/ee site cmd-dump | | bin/ee config list | + Scenario: cli info skips the version check when docker is not available + Given no EasyEngine migration is pending + And the EasyEngine option "version" is set to "99.0.0" + And the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli info' + Then the command should exit with code 0 + And STDERR should not contain "It seems you're not running latest version" + And STDOUT should contain "EE version" + Scenario: A newer DB version makes cli info warn and other commands refuse to run Given no EasyEngine migration is pending And the EasyEngine option "version" is set to "99.0.0" @@ -58,7 +72,9 @@ Feature: Migration triggering And STDERR should contain "Error: It seems you're not running latest version." And STDERR should not contain "is not a registered ee command" - Scenario: The nightly suffix is ignored when comparing versions + # A test that tells whether the suffix is stripped would risk a real migration on failure, so this only + # checks that a newer nightly DB version is refused like a stable one. + Scenario: A newer nightly DB version also makes checked commands refuse to run Given no EasyEngine migration is pending And the EasyEngine option "version" is set to "99.0.0-nightly-bh" When I run 'bin/ee site list' @@ -66,6 +82,7 @@ Feature: Migration triggering And STDERR should contain "Error: It seems you're not running latest version." # Runs a real migration: image pulls (MIG-4), cron container restart and log rotation (MIG-5). + # Never on a stack in use: excluded from every profile but `migration`, and refused without EE_BEHAT_ALLOW_MIGRATION=1. @migration Scenario: An older DB version triggers the upgrade on the next checked command Given no EasyEngine migration is pending diff --git a/features/phar.feature b/features/phar.feature index 08a1582c8..413155df6 100644 --- a/features/phar.feature +++ b/features/phar.feature @@ -1,5 +1,6 @@ -# Builds ee.phar from the checkout (~1 min, cached for the feature) and checks the phar bootstrap. +# Builds a phar from a private copy of the checkout into /var/tmp (~1 min, once per run) and checks the phar bootstrap. # `cli update` is not exercised: from a phar it would really download and swap the phar. +# EE_DISABLE_AUTO_CHECK_UPDATE keeps `help` from ever offering a self-update of the built phar. @phar Feature: Phar build @@ -7,7 +8,7 @@ Feature: Phar build Given ee phar is generated Scenario: The phar reports the source VERSION - When I run 'php ee.phar cli version' + When I run '{PHP} {PHAR} cli version' Then the command should exit with code 0 And STDOUT should be: """ @@ -15,31 +16,30 @@ Feature: Phar build """ Scenario: The phar serves help pages - When I run 'php ee.phar help cli' + When I run 'EE_DISABLE_AUTO_CHECK_UPDATE=1 {PHP} {PHAR} help cli' Then the command should exit with code 0 And STDOUT should contain "ee cli" And STDOUT should contain "self-uninstall" Scenario: The phar registers the bundled command packages - When I run 'php ee.phar cli has-command "site create"' + When I run '{PHP} {PHAR} cli has-command "site create"' Then the command should exit with code 0 - When I run 'php ee.phar cli has-command "service restart"' + When I run '{PHP} {PHAR} cli has-command "service restart"' Then the command should exit with code 0 - When I run 'php ee.phar cli has-command "admin-tools enable"' + When I run '{PHP} {PHAR} cli has-command "admin-tools enable"' Then the command should exit with code 0 Scenario: The phar cli info reports the phar as EE root Given no EasyEngine migration is pending - When I run 'php ee.phar cli info --format=json' + When I run '{PHP} {PHAR} cli info --format=json' Then the command should exit with code 0 And the json in STDOUT at "ee_dir_path" should be "phar://ee.phar" And the json in STDOUT at "ee_version" should be "{EE_VERSION}" - # New (suspected): php/boot-phar.php defines EE_PHAR_PATH as getcwd(), so "EE phar path" shows the - # working directory; phar_safe_path() relies on the same constant. + # CORE-8: php/boot-phar.php defines EE_PHAR_PATH as getcwd(), so "EE phar path" shows the working directory. @known-bug Scenario: The phar cli info shows the path of the phar file Given no EasyEngine migration is pending - When I run 'cd {TMP} && php {EE_ROOT}/ee.phar cli info' + When I run 'cd {TMP} && COLUMNS=250 {PHP} {PHAR} cli info' Then the command should exit with code 0 - And STDOUT should match "#^\| EE phar path\s+\| {EE_ROOT}/ee\.phar\s+\|#m" + And STDOUT should match "#^\| EE phar path\s+\| {PHAR}\s+\|#m" From ecc39f797376527f66201ee054bdd8faeea56f2d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 14:24:29 +0530 Subject: [PATCH 11/30] test(easyengine): tighten assertions and cover config defaults, packages and alias groups - Reference CORE-3..13 in the @known-bug scenarios; drop the "site nosuchcommand" example, which contradicts the passing "site create" rows. - Add scenarios for per-command config defaults, disabled commands in help lists and whole namespaces, EE_PACKAGES_DIR, site cmd-dump without docker, alias group flags, the self-uninstall refusal, check-update HTTP errors, and known bugs CORE-9..13. - Check exit codes before negative assertions, make the PAGER check observable, set COLUMNS for cli info tables, clear no_proxy for the dead-proxy checks, and fold check-update --major into the type outline. --- features/cli-alias.feature | 60 +++++++++++++++++++++++++++++- features/cli-completions.feature | 10 ++++- features/cli-dump.feature | 11 +++++- features/cli-has-command.feature | 14 +++---- features/cli-info.feature | 5 ++- features/cli-update.feature | 20 +++++----- features/config-file.feature | 63 +++++++++++++++++++++++++++++++- features/global-flags.feature | 6 +-- features/help.feature | 15 ++++++-- features/requirements.feature | 17 ++++++++- features/unknown-command.feature | 10 ++++- 11 files changed, 197 insertions(+), 34 deletions(-) diff --git a/features/cli-alias.feature b/features/cli-alias.feature index ec2108ccf..a945b5dea 100644 --- a/features/cli-alias.feature +++ b/features/cli-alias.feature @@ -190,7 +190,7 @@ Feature: ee cli alias and @alias execution Then the command should exit with code 1 And STDERR should contain "Error: Cannot use '@all' when no aliases are registered." - # New (suspected): Runner::run_ssh_command() is never called, so an alias with `ssh:` silently runs the + # CORE-4: Runner::run_ssh_command() is never called, so an alias with `ssh:` silently runs the # command on the local machine instead of the remote one. @known-bug Scenario: A command for an ssh alias is not run locally @@ -204,3 +204,61 @@ Feature: ee cli alias and @alias execution When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-remote cli version' Then the command should fail And STDOUT should not contain "EE {EE_VERSION}" + + Scenario: An alias group passes the command's flags to every member + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-two: + path: /var/tmp + @bh-group: + - @bh-one + - @bh-two + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-group cli param-dump --with-values --le-mail=bh-group@example.test' + Then the command should exit with code 0 + And STDOUT should match "#^@bh-one\n\{.*"current":"bh-group@example\.test".*@bh-two\n\{.*"current":"bh-group@example\.test"#s" + + # CORE-12: run_alias_group() ignores the children's exit codes. + @known-bug + Scenario: @all fails when the command fails for an alias + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-two: + path: /var/tmp + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli verison' + Then STDERR should contain "Error: 'verison' is not a registered subcommand of 'cli'." + And the command should fail + + # CORE-12: @all also runs the group aliases, so their members run twice. + @known-bug + Scenario: @all runs the command once per alias when groups exist + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-two: + path: /var/tmp + @bh-group: + - @bh-one + - @bh-two + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli version' + Then the command should exit with code 0 + And STDOUT should be: + """ + @bh-one + EE {EE_VERSION} + @bh-two + EE {EE_VERSION} + """ diff --git a/features/cli-completions.feature b/features/cli-completions.feature index 011597138..efa9cdd64 100644 --- a/features/cli-completions.feature +++ b/features/cli-completions.feature @@ -79,7 +79,8 @@ Feature: ee cli completions When I run 'bin/ee cli completions --line="ee site create --type=wp --admin-u" --point=100' Then STDOUT should match "#^--admin-user=$#m" When I run 'bin/ee cli completions --line="ee site create --type=html --admin-u" --point=100' - Then STDOUT should be empty + Then the command should exit with code 0 + And STDOUT should be empty Scenario: Aliases from the config file are completed Given a file "{TMP}/config.yml" with: @@ -111,3 +112,10 @@ Feature: ee cli completions When I run 'bin/ee cli completions --line="ee cli bh{RAND}" --point=100' Then the command should exit with code 0 And the file "{EE_ROOT_DIR}/logs/ee.log" should not contain "bh{RAND}" + + # CORE-13: an unknown --type makes CommandFactory reflect an empty class name (uncaught ReflectionException). + @known-bug + Scenario: Completing options for an unknown site type does not crash + When I run 'bin/ee cli completions --line="ee site create --type=bogus --" --point=100' + Then the command should exit with code 0 + And STDOUT should be empty diff --git a/features/cli-dump.feature b/features/cli-dump.feature index 55f5f3ce4..a7e5d53bb 100644 --- a/features/cli-dump.feature +++ b/features/cli-dump.feature @@ -13,7 +13,9 @@ Feature: ee cli param-dump and cmd-dump Scenario: param-dump without --with-values has no current values When I run 'bin/ee cli param-dump' - Then STDOUT should not contain "current" + Then the command should exit with code 0 + And STDOUT should be valid json + And STDOUT should not contain "current" Scenario: param-dump --format=var_export prints a PHP array When I run 'bin/ee cli param-dump --format=var_export' @@ -97,3 +99,10 @@ Feature: ee cli param-dump and cmd-dump And the json in STDOUT at "subcommands.name=site.subcommands.*.name" should contain "create --type=wp" And the json in STDOUT at "subcommands.name=site.subcommands.*.name" should contain "create --type=php" And the json in STDOUT at "subcommands.*.name" should not contain "site_type_wp" + + Scenario: site cmd-dump works without docker + Given the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php site cmd-dump --debug' + Then the command should exit with code 0 + And STDOUT should be valid json + And STDERR should not contain "Docker check failed" diff --git a/features/cli-has-command.feature b/features/cli-has-command.feature index 66a14583a..0edb55488 100644 --- a/features/cli-has-command.feature +++ b/features/cli-has-command.feature @@ -29,18 +29,14 @@ Feature: ee cli has-command | "foo bar" | | "" | - # New (suspected): find_command_to_run() stops at the first leaf and ignores the remaining words. - # `site` is a leaf too (Site_Command::__invoke routes site types), so any "site " is "registered". + # CORE-6: find_command_to_run() stops at the first leaf and ignores the remaining words. `site` is a leaf too + # (Site_Command::__invoke routes site types), so "site create" above only passes through the same shortcut and + # "site " returns 0 as well; it is left out here because a fix has to special-case site first. @known-bug - Scenario Outline: Extra words after a leaf command are not a registered command - When I run 'bin/ee cli has-command ' + Scenario: Extra words after a leaf command are not a registered command + When I run 'bin/ee cli has-command "cli version extra"' Then the command should exit with code 1 - Examples: - | command | - | "cli version extra" | - | "site nosuchcommand" | - Scenario: has-command without a command prints its usage When I run 'bin/ee cli has-command' Then the command should exit with code 1 diff --git a/features/cli-info.feature b/features/cli-info.feature index 4ce923b91..10b92cfc7 100644 --- a/features/cli-info.feature +++ b/features/cli-info.feature @@ -1,11 +1,12 @@ # `cli info` runs the migration check when docker is available, so every scenario is guarded. +# COLUMNS keeps cli-tools from wrapping the table to the terminal width. Feature: ee cli info Background: Given no EasyEngine migration is pending Scenario: cli info lists every diagnostic row - When I run 'bin/ee cli info' + When I run 'COLUMNS=250 bin/ee cli info' Then the command should exit with code 0 And STDOUT should not contain "Executing migrations" And STDOUT should match "#^\| OS\s+\| \S+#m" @@ -39,7 +40,7 @@ Feature: ee cli info When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli info --format=json' Then the command should exit with code 0 And the json in STDOUT at "global_config_path" should be "{TMP}/config.yml" - When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli info' + When I run 'COLUMNS=250 EE_CONFIG_PATH={TMP}/config.yml bin/ee cli info' Then STDOUT should match "#^\| EE global config\s+\| {TMP}/config\.yml\s+\|#m" Scenario: cli info rejects an unsupported format diff --git a/features/cli-update.feature b/features/cli-update.feature index 92a55221c..d001b85af 100644 --- a/features/cli-update.feature +++ b/features/cli-update.feature @@ -6,15 +6,6 @@ Feature: ee cli check-update and ee cli update Then the command should exit with code 0 And STDOUT should match "#(^Success: EasyEngine is at the latest version\.$|^\| version\s+\| update_type\s+\| package_url\s+\|$)#m" - @network - Scenario: check-update --major reports that no major release exists - When I run 'bin/ee cli check-update --major' - Then the command should exit with code 0 - And STDOUT should be: - """ - Success: EasyEngine is at the latest major version. - """ - @network Scenario Outline: check-update filters by update type When I run 'bin/ee cli check-update --' @@ -23,6 +14,7 @@ Feature: ee cli check-update and ee cli update Examples: | type | + | major | | minor | | patch | @@ -40,12 +32,12 @@ Feature: ee cli check-update and ee cli update And STDOUT should match "#^\s*\d*\s*$#" Scenario: check-update rejects an unsupported format before any request - When I run 'https_proxy=http://127.0.0.1:9 bin/ee cli check-update --format=xml' + When I run 'no_proxy= NO_PROXY= https_proxy=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 bin/ee cli check-update --format=xml' Then the command should exit with code 1 And STDERR should contain "Invalid value specified for 'format'" Scenario: check-update fails cleanly when GitHub is unreachable - When I run 'https_proxy=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 bin/ee cli check-update' + When I run 'no_proxy= NO_PROXY= https_proxy=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 bin/ee cli check-update' Then the command should exit with code 1 And STDERR should contain "Error: Failed to get url 'https://api.github.com/repos/EasyEngine/easyengine/releases?per_page=100'" @@ -63,3 +55,9 @@ Feature: ee cli check-update and ee cli update | --yes | | --stable --yes | | --nightly --yes | + + @network + Scenario: check-update reports the HTTP status of a failed GitHub request + When I run 'GITHUB_TOKEN=bh-invalid-token bin/ee cli check-update' + Then the command should exit with code 1 + And STDERR should contain "Error: Failed to get latest version (HTTP code 401)." diff --git a/features/config-file.feature b/features/config-file.feature index 516cce478..fb2452d77 100644 --- a/features/config-file.feature +++ b/features/config-file.feature @@ -150,7 +150,7 @@ Feature: Global and project config files And STDOUT should contain "ee cli update" And STDOUT should not contain "ee cli version" - # New (suspected): CompositeCommand::show_usage() counts the skipped disabled subcommand, so the + # CORE-7: CompositeCommand::show_usage() counts the skipped disabled subcommand, so the # first printed line loses its "usage:" prefix when the first subcommand is disabled. @known-bug Scenario: The composite usage still starts with "usage:" when its first subcommand is disabled @@ -175,3 +175,64 @@ Feature: Global and project config files When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli aliases' Then the command should exit with code 1 And STDERR should contain "has been disabled from the config file." + + Scenario: A command section in the config file sets that command's default flags + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + cli param-dump: + format: var_export + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli param-dump' + Then the command should exit with code 0 + And STDOUT should match "#^array \(#" + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli param-dump --format=json' + Then the command should exit with code 0 + And STDOUT should be valid json + + Scenario: help leaves disabled commands out of the command lists + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli version + - mailhog + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee help cli' + Then the command should exit with code 0 + And STDOUT should match "#^ update\s#m" + And STDOUT should not match "#^ version\s#m" + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee help' + Then the command should exit with code 0 + And STDOUT should match "#^ log\s#m" + And STDOUT should not match "#^ mailhog\s#m" + + Scenario: Disabling a namespace disables its subcommands + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + disabled_commands: + - cli + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli version' + Then the command should exit with code 1 + And STDERR should contain "Error: The 'cli' command has been disabled from the config file." + + Scenario: Commands from EE_PACKAGES_DIR are loaded + Given a file "{TMP}/pkgs/vendor/autoload.php" with: + """ + " When I run 'bin/ee help site create --type=html' - Then STDOUT should not contain "--admin-user=" + Then the command should exit with code 0 + And STDOUT should contain "Runs the standard HTML site installation." + And STDOUT should not contain "--admin-user=" Scenario: help for an unknown site type fails When I run 'bin/ee help site create --type=bogus' Then the command should exit with code 1 And STDERR should contain "Error: 'bogus' is not a registered site type" + # CORE-10 (site-command): the hint repeats the bogus type, and that command fails the same way. + @known-bug + Scenario: help for an unknown site type points to a working help command + When I run 'bin/ee help site create --type=bogus' + Then the command should exit with code 1 + And STDERR should not contain "ee help site --type=bogus" + Scenario: help for an unknown command fails with a hint When I run 'bin/ee help nosuchcommand' Then the command should exit with code 1 @@ -214,9 +223,9 @@ Feature: ee help Then STDERR should contain "Did you mean 'version'?" Scenario: help honours the PAGER environment variable - When I run 'PAGER=cat bin/ee help cli version' + When I run 'PAGER="sed s/^/bh-pager:/" bin/ee help cli version' Then the command should exit with code 0 - And STDOUT should contain "ee cli version" + And STDOUT should match "#^bh-pager: ee cli version#m" Scenario: help pages include the command examples When I run 'bin/ee help cli has-command' diff --git a/features/requirements.feature b/features/requirements.feature index 8d7309098..d5f4276cd 100644 --- a/features/requirements.feature +++ b/features/requirements.feature @@ -9,7 +9,7 @@ Feature: Requirement checks Then the command should exit with code 0 And STDOUT should contain "ee cli" - # New: help pipes through `less -r` unless PAGER is set; without less it prints nothing and exits 0. + # CORE-5: help pipes through `less -r` unless PAGER is set; without less it prints nothing and exits 0. @known-bug Scenario: help still prints when the default pager is not installed Given the directory "{TMP}/emptybin" exists @@ -44,3 +44,18 @@ Feature: Requirement checks Then STDERR should contain "Docker check failed (attempt 1/4), retrying in 1s..." And STDERR should contain "Docker check failed (attempt 3/4), retrying in 4s..." And STDERR should contain "Error: Docker not installed or not running (checked 4 times)." + + # Safe by construction: without docker the command stops before the prompt, which `n` would decline anyway. + Scenario: self-uninstall refuses to run when docker is not available + Given the directory "{TMP}/emptybin" exists + When I run 'echo n | PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli self-uninstall' + Then STDERR should contain "Error: Unable to proceed with uninstallation. Seems there is a dependency down." + And STDOUT should not contain "Are you sure" + + # CORE-11: the refusal uses `EE::error( ..., false ); die;`, so it exits 0 like CORE-1. + @known-bug + Scenario: self-uninstall refuses with a non-zero status when docker is not available + Given the directory "{TMP}/emptybin" exists + When I run 'echo n | PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli self-uninstall' + Then STDERR should contain "Unable to proceed with uninstallation" + And the command should fail diff --git a/features/unknown-command.feature b/features/unknown-command.feature index a2ea459a3..8f6bdc06c 100644 --- a/features/unknown-command.feature +++ b/features/unknown-command.feature @@ -29,7 +29,7 @@ Feature: Unknown commands and composite commands And STDERR should contain "Error: 'zzzzzzzzzz' is not a registered subcommand of 'cli'." And STDERR should not contain "Did you mean" - # Unknown top-level commands pass the requirement and migration checks before the lookup. + # CORE-9: unknown top-level commands pass the requirement and migration checks before the lookup. Scenario: An unknown top-level command suggests the closest one Given no EasyEngine migration is pending When I run 'bin/ee sitee' @@ -69,3 +69,11 @@ Feature: Unknown commands and composite commands Then the command should exit with code 1 And STDERR should contain "missing --line parameter" And STDERR should contain "missing --point parameter" + + # CORE-9: without docker the requirement check fails first and hides the typo. + @known-bug + Scenario: An unknown top-level command is reported even when docker is not available + Given the directory "{TMP}/emptybin" exists + When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php sitee' + Then the command should exit with code 1 + And STDERR should contain "Error: 'sitee' is not a registered ee command." From 126815ca6ad0f54b2446a414dc577765b798d44b Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 14:37:52 +0530 Subject: [PATCH 12/30] docs(easyengine): update behat coverage inventory after review --- features/COVERAGE.md | 110 +++++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 46 deletions(-) diff --git a/features/COVERAGE.md b/features/COVERAGE.md index 25ada00ef..a5e79ec10 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -24,7 +24,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | global config path from `EE_CONFIG_PATH` (json and list) | cli-info.feature: "cli info reports the global config ..." | | project config path | config-file.feature: "cli info reports the project config file" | | invalid `--format`, typo suggestion, extra positional | cli-info.feature (3 scenarios) | -| runs the migration check only when docker is up | requirements.feature: "cli info still works when docker is not available"; migration.feature: newer/older version scenarios | +| runs the migration check only when docker is up | migration.feature `@destructive`: "cli info skips the version check when docker is not available" and the newer/older version scenarios; requirements.feature: "cli info still works when docker is not available" | | no migration when versions match | cli-info.feature: "cli info lists every diagnostic row" (no "Executing migrations") | | phar values (`ee_dir_path` = `phar://ee.phar`, phar path) | phar.feature (1 + 1 `@known-bug`) | @@ -33,20 +33,20 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | Item | Covered by | |---|---| | default table or "latest version" success | cli-update.feature `@network` | -| `--major`, `--minor`, `--patch` | cli-update.feature `@network` (outline + `--major`) | +| `--major`, `--minor`, `--patch` | cli-update.feature `@network` outline | | `--format=json`, `--format=count` | cli-update.feature `@network` | | invalid `--format` (validated before any request) | cli-update.feature | -| GitHub unreachable: clean error, exit 1 | cli-update.feature (proxy to a closed port, no network needed) | -| `--field`, `--fields`, `--format=csv|yaml` | not covered: output depends on live release data; formatter covered by package suites | +| GitHub unreachable: clean error, exit 1 | cli-update.feature (proxy to a closed port, `no_proxy` cleared, no network needed) | +| HTTP error status (`GITHUB_TOKEN` sent, invalid token gives 401) | cli-update.feature `@network` | +| `--field`, `--fields`, `--format=csv|yaml`, precedence of several type flags | not covered: output depends on live release data; formatter covered by package suites | | nightly branch (`-alpha-` versions, `NIGHTLY_VERSION`) | not covered: needs a nightly VERSION | -| `GITHUB_TOKEN` header | not covered: no observable output | ### `ee cli update [--stable] [--nightly] [--yes]` | Item | Covered by | |---|---| | refuses to run from source (`You can only self-update Phar files.`) with every flag combination | cli-update.feature outline (4 rows) | -| real update (download, md5 check, child `cli info`, phar swap), macOS brew message, not-writable errors | not covered: would replace the phar and run the new version's migrations (MIG-6); root can write everything; not on macOS | +| real update (prompt, download, md5 check, child `cli info`, phar swap), macOS brew message, not-writable errors | not covered on purpose: an accepted prompt downloads a phar whose `cli info` runs the new version's migrations on the live DB (MIG-6); a declined prompt or an immutable (`chattr +i`) phar would be testable but a regression there would do exactly that; not on macOS | | `auto_check_update()` on help (phar + tty only) | not covered: needs a tty and a writable phar | ### `ee cli param-dump [--with-values] [--format=json|var_export]` @@ -65,7 +65,8 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | json tree, root name/description, all 12 top-level commands | cli-dump.feature | | nested subcommands, synopsis, description | cli-dump.feature | -| `site cmd-dump` bypasses requirement checks, merges site types (`create --type=wp`) | cli-dump.feature | +| `site cmd-dump` merges site types (`create --type=wp`) | cli-dump.feature | +| `site cmd-dump` bypasses the requirement checks (works without docker) | cli-dump.feature: "site cmd-dump works without docker"; migration.feature outline | ### `ee cli completions --line --point [--shell=bash|zsh]` @@ -79,6 +80,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | site subcommands and `--type`-aware site options | cli-completions.feature | | aliases (`ee @`, `ee @alias cli ...`) | cli-completions.feature | | unknown command, invalid shell, missing `--line/--point` | cli-completions.feature, unknown-command.feature | +| unknown `--type` in site completions doesn't crash | cli-completions.feature `@known-bug` (CORE-13) | | does not log to `ee.log` | cli-completions.feature | | `` positional completion, site type from an existing site (`Site::find`) | not covered: no core command has a `` positional; needs a site | @@ -93,7 +95,9 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | invalid format | cli-alias.feature | | `EE_RUNTIME_ALIAS` replaces file aliases, filters keys and names | cli-alias.feature | | run through a local alias, `@all`, a group, a group with an unknown member, unknown alias suggestion, `@all` without aliases | cli-alias.feature | -| aliases with `ssh:` run remotely | cli-alias.feature `@known-bug` (they run locally) | +| a group passes the command's flags to every member | cli-alias.feature | +| `@all`/group exit status, `@all` running group members twice | cli-alias.feature `@known-bug` (2, CORE-12) | +| aliases with `ssh:` run remotely | cli-alias.feature `@known-bug` (CORE-4: they run locally) | | ssh schemes (docker, docker-compose, vagrant, ssh), `EE_SSH_PRE_CMD` | not covered: `run_ssh_command()` is dead code in EE; would need a remote host | ### `ee cli has-command ...` @@ -102,7 +106,8 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | registered commands (quoted and split forms, packages' commands) exit 0 silently | cli-has-command.feature outline (8 rows) | | unknown commands and empty name exit 1 silently | cli-has-command.feature outline (3 rows) | -| trailing words after a leaf, `site ` | cli-has-command.feature `@known-bug` outline (2 rows) | +| trailing words after a leaf (`cli version extra`) | cli-has-command.feature `@known-bug` (CORE-6) | +| `site ` is "registered" (site is a leaf) | not covered: documented in CORE-6; the passing `site create` rows use the same shortcut | | missing argument prints usage | cli-has-command.feature | | disabled command reported as missing | cli-has-command.feature | @@ -111,8 +116,9 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | Item | Covered by | |---|---| | declining the prompt removes nothing | cli.feature `@destructive` | -| full uninstall (containers, networks, volumes, images, site dirs, `/opt/easyengine`) | cli.feature `@destructive` (existing scenario, TEST-3) | -| refuses when docker is down, phar self-delete | not covered: needs docker stopped / a phar install | +| full uninstall (containers, networks, volumes, images, `/opt/easyengine`; site dirs not checked) | cli.feature `@destructive @uninstall` (existing scenario, TEST-3) | +| refuses when docker is down (and its exit status) | requirements.feature (+1 `@known-bug`, CORE-11) | +| phar self-delete | not covered: needs a phar install, and would delete it | ### `ee help [...]` @@ -120,12 +126,12 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | `ee`, `ee help`, `ee --help` print the same top-level page, with global parameters | help.feature | | every top-level command (12) and subcommand (54) has a help page equal to ` --help` | help.feature outlines | -| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature | +| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature (+1 `@known-bug`, CORE-10) | | unknown command/subcommand errors and suggestions | help.feature | -| `PAGER`, examples section | help.feature | +| `PAGER` (a `sed` pager that prefixes every line), examples section | help.feature | | works without docker | requirements.feature | -| no output without `less` | requirements.feature `@known-bug` | -| disabled command | config-file.feature | +| no output without `less` | requirements.feature `@known-bug` (CORE-5) | +| disabled command: error, left out of `ee help` and `ee help cli` lists | config-file.feature | ### Global parameters and runner behaviour @@ -134,17 +140,19 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | `--debug`, `--debug` before the command, `--debug=`, unknown group, `debug:` in config | global-flags.feature | | `EE_STRICT_ARGS_MODE` | global-flags.feature | | color auto-detection (piped, `SHELL_PIPE=0`) | global-flags.feature | -| `--color`, `--no-color`, `color:` in config | global-flags.feature `@known-bug` (3) | -| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (1 + 3 `@known-bug`) | +| `--color`, `--no-color`, `color:` in config | global-flags.feature `@known-bug` (3, CORE-3) | +| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (1 + 3 `@known-bug`, CORE-3) | | unknown parameter suggestions | cli-info.feature, cli-dump.feature | -| `--sites_path`, `--locale`, `--le-mail`, `--env`, `--wp-mail`, `--sysctl`, `--skip-tty`, `--custom-compose` | runtime parsing covered through `--le-mail` (cli-dump.feature, config-file.feature); their effects belong to the package suites | +| `--sites_path`, `--locale`, `--le-mail`, `--env`, `--wp-mail`, `--sysctl`, `--skip-tty`, `--custom-compose` | partly: only `--le-mail` (runtime) and `locale`/`le-mail`/`wp-mail` (file) are exercised (cli-dump.feature, config-file.feature); their effects belong to the package suites | | unknown command/subcommand suggestions, shorthand map (`v`, `check`), no suggestion | unknown-command.feature | +| unknown top-level command runs the checks before the lookup | unknown-command.feature (guarded) + `@known-bug` (CORE-9), migration.feature `@destructive` | | composite command usage, disabled subcommand hidden | unknown-command.feature, config-file.feature (+1 `@known-bug`) | | missing required parameters | unknown-command.feature | | requirement checks: missing docker-compose, docker retry/backoff | requirements.feature | | failed requirement exit status (CORE-1) | requirements.feature `@known-bug` | | argv logged to `ee.log` with secrets (LOG-1) | global-flags.feature `@known-bug` | -| `BEHAT_RUN`, `EE_PACKAGES_DIR`, `EE_DISABLE_AUTO_CHECK_UPDATE`, `EE_AUTO_CHECK_UPDATE_DAYS`, `EE_PHP` | not covered: only affect error reporting, a packages dir nobody uses, or the phar+tty auto update | +| `EE_PACKAGES_DIR` (package autoloader loaded, commands registered) | config-file.feature | +| `BEHAT_RUN`, `EE_DISABLE_AUTO_CHECK_UPDATE`, `EE_AUTO_CHECK_UPDATE_DAYS`, `EE_PHP` | not covered: only affect error reporting or the phar+tty auto update (the phar scenarios set `EE_DISABLE_AUTO_CHECK_UPDATE`) | ### Config files @@ -155,17 +163,19 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | path resolved before the defaults file exists (first run reads nothing) | config-file.feature | | project config `ee.yml`, from a subdirectory, `ee.local.yml` priority, runtime override, none found | config-file.feature | | `_: inherit:` | config-file.feature | -| `disabled_commands` (run, help, usage, alias) | config-file.feature, cli-has-command.feature | -| unreadable/unwritable config errors | not covered: root bypasses file permissions | -| `_: merge:` for extra config | not covered: only used by package config keys | +| `disabled_commands` (run, help, usage, alias, a whole namespace) | config-file.feature, cli-has-command.feature | +| per-command defaults (a `cli param-dump:` section), runtime flag wins | config-file.feature | +| unreadable/unwritable config errors | not covered: root bypasses file permissions; only reachable with `chattr +i` or `/proc` paths (fragile) | +| `_: merge:` for extra config, relative `inherit:` paths | not covered | ### Migration trigger | Item | Covered by | |---|---| -| cli/help/config/`site cmd-dump`/no-args never check (14 commands) | migration.feature `@destructive` outline | -| DB version newer: warning for `cli info`, error otherwise, before command lookup, nightly suffix ignored | migration.feature `@destructive` (3) | -| DB version older: full pipeline runs, version bumped | migration.feature `@destructive @migration` | +| cli/help/config/`site cmd-dump`/no-args never check (14 commands) | migration.feature `@destructive` outline (with a NEWER DB version, so a regression errors instead of migrating) | +| DB version newer: warning for `cli info`, error otherwise, before command lookup, a newer nightly DB version | migration.feature `@destructive` (3) | +| nightly suffix stripping itself | not covered: a distinguishing test would run a real migration when it fails | +| DB version older: full pipeline runs, version bumped | migration.feature `@destructive @migration` (never on a stack in use) | | versions equal: nothing runs | cli-info.feature | | fresh install (`ee successfully setup.`), nightly VERSION always migrating, 5 GB disk check, rollback | not covered: needs a wiped machine, a modified VERSION or a full disk; MIG-1..19 are architectural and need the live upgrade tests | @@ -173,38 +183,42 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | Item | Covered by | |---|---| -| build from the checkout, version, help, bundled packages registered, `cli info` root | phar.feature `@phar` | -| `EE_PHAR_PATH` is the working directory | phar.feature `@known-bug` | +| build (from a `--no-dev` copy of the checkout into `/var/tmp`), version, help, bundled packages registered, `cli info` root | phar.feature `@phar` | +| `EE_PHAR_PATH` is the working directory | phar.feature `@known-bug` (CORE-8) | + ### Coverage estimate -The inventory has 97 rows: 85 covered and 12 not covered (about 87% of rows, about 80% when rows with several flags are counted per flag). What is left is the real self-update, ssh aliases (dead code), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. +The inventory has 107 rows: 93 covered, 1 partly covered and 13 not covered (about 87% of rows, about 78% when rows with several flags are counted per flag). What is left is the real self-update and phar self-delete (unsafe on a live stack), ssh aliases (never wired up, CORE-4), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. ## How to run -The suite runs from the core root as root (see `/root/workspace/tools/run-behat.sh core`). `behat.yml` excludes `@destructive` by default. +The suite runs from the core root as root (see `/root/workspace/tools/run-behat.sh core`). | Command | Runs | |---|---| -| `vendor/bin/behat` | everything except `@destructive` (229 scenarios) | -| `vendor/bin/behat -p destructive` | only `@destructive` (20 scenarios: self-uninstall, DB version changes, a real migration) | -| `vendor/bin/behat -p all` | all 249 scenarios | -| `vendor/bin/behat --tags='~@destructive&&~@known-bug&&~@network&&~@phar'` | fast, deterministic subset | +| `vendor/bin/behat` | everything except `@destructive` (242 scenarios) | +| `EE_BEHAT_DESTRUCTIVE=1 vendor/bin/behat -p destructive` | `@destructive` without `@migration`/`@uninstall` (19 scenarios: DB version changes, the declined self-uninstall prompt) | +| `EE_BEHAT_DESTRUCTIVE=1 EE_BEHAT_ALLOW_MIGRATION=1 vendor/bin/behat -p migration` | the real upgrade migration (1 scenario; never on a stack in use) | +| `EE_BEHAT_DESTRUCTIVE=1 EE_BEHAT_ALLOW_UNINSTALL=1 vendor/bin/behat -p uninstall` | `ee cli self-uninstall --yes` (1 scenario; run it alone and last) | +| `EE_BEHAT_DESTRUCTIVE=1 vendor/bin/behat -p all` | everything that can run in one pass (261 scenarios) | +| `vendor/bin/behat --tags='~@destructive&&~@known-bug&&~@network&&~@phar'` | fast, deterministic subset (212 scenarios) | -A CLI `--tags` replaces the profile filter, so always add `&&~@destructive` to it unless you mean to run destructive scenarios. +A CLI `--tags` replaces the profile filter, so the tags alone can't keep the dangerous scenarios off: a `@BeforeScenario` hook refuses (fails) every `@destructive`, `@migration` or `@uninstall` scenario whose env var (`EE_BEHAT_DESTRUCTIVE`, `EE_BEHAT_ALLOW_MIGRATION`, `EE_BEHAT_ALLOW_UNINSTALL`) isn't set. On a machine without `/opt/easyengine/db/ee.sqlite` (a fresh CI runner) also set `EE_BEHAT_ALLOW_INIT=1`. -Tags: `@destructive` (20, changes global state; restored where possible), `@migration` (1, runs the real upgrade pipeline, also `@destructive`), `@known-bug` (14, assert the correct behaviour and fail today), `@network` (7, query the GitHub releases API), `@phar` (5, build `ee.phar`, about 1 min). No `@le` or `@remote` scenarios: core has no Let's Encrypt or backup code. +Tags: `@destructive` (21, changes global state; restored where possible), `@migration` (1, the real upgrade pipeline), `@uninstall` (1, self-uninstall), `@known-bug` (19, assert the correct behaviour and fail today), `@network` (8, query the GitHub releases API; unauthenticated calls share GitHub's 60/hour limit), `@phar` (5, build a phar, about 1 min). No `@le` or `@remote` scenarios: core has no Let's Encrypt or backup code. Safety rules built into the suite: -- `Given no EasyEngine migration is pending` guards every scenario whose command reaches `maybe_trigger_migration()` (`cli info`, unknown top-level commands, phar `cli info`). It fails the scenario instead of running a migration when `options.version` differs from `VERSION`, when `VERSION` is a nightly, or when `/opt/easyengine/db/ee.sqlite` is missing (set `EE_BEHAT_ALLOW_INIT=1` to allow the first command to initialise a fresh machine, as the old CI flow did). -- Scenarios that change `options.version` restore it in `@AfterScenario`. -- Config scenarios point `EE_CONFIG_PATH` at a per-scenario file under `/var/tmp/ee-behat-core/` (`{TMP}`), removed after the scenario; the real `/opt/easyengine/config/config.yml` is never written. +- `Given no EasyEngine migration is pending` guards every scenario whose command reaches `maybe_trigger_migration()` with docker available (`cli info`, unknown top-level commands, phar `cli info`, the self-uninstall). It fails the scenario instead of running a migration when `options.version` differs from `VERSION`, when `VERSION` is a nightly, or when `/opt/easyengine/db/ee.sqlite` is missing (unless `EE_BEHAT_ALLOW_INIT=1`, which lets the first command initialise a fresh machine, as the old CI flow did). +- Only the `@migration` scenario writes an older `options.version`; the others use `99.0.0`, so a regression, or an `ee` run by cron or another user meanwhile, gets the "not running latest version" error instead of a migration. Changed options are restored in `@AfterScenario` and, if the run dies, by a shutdown function (Behat exits on Ctrl-C) and a SIGTERM handler when pcntl is available. +- The phar is built from a `--no-dev` copy of the checkout under `/var/tmp/ee-behat-core/phar-/` and removed after the suite; the checkout, its `vendor/` autoloader and `/usr/local/bin/ee` are never touched, `cli update` is never run from the phar and `help` runs with `EE_DISABLE_AUTO_CHECK_UPDATE=1`. +- Config scenarios point `EE_CONFIG_PATH` at a per-scenario file under `/var/tmp/ee-behat-core/` (`{TMP}`), removed after the scenario; the real `/opt/easyengine/config/config.yml` is never written (scenarios without `EE_CONFIG_PATH` read it, so `debug:`, aliases or `disabled_commands` there would break exact-output checks). - Requirement scenarios hide docker/docker-compose with a private `PATH` instead of stopping anything. - The suite creates no sites. The `@AfterFeature` cleanup only deletes sites named `bh-easyengine-*`. - Commands are run through `EE\Process` directly, not `EE::launch()`, so the harness does not write test command lines into `ee.log`. -Estimated runtime: default profile about 3 to 4 minutes (help outlines about 40 s, two 7 s docker-retry scenarios, about 1 min for the phar build, a few seconds of GitHub API calls). `-p destructive` adds the migration scenario (1 to 3 min, image pulls and a cron container restart) and the self-uninstall (about 1 min, after which the next site command re-initialises EasyEngine and pulls about 8 GB of images). +Estimated runtime: default profile about 4 minutes (help outlines about 40 s, five 7 s docker-retry scenarios, about 1 min for the phar build, a few seconds of GitHub API calls). `-p migration` takes 1 to 3 min (image pulls, a cron container restart) and `-p uninstall` about 1 min, after which the next site command re-initialises EasyEngine and pulls about 8 GB of images. ## Known bugs exposed (`@known-bug`) @@ -212,10 +226,14 @@ Estimated runtime: default profile about 3 to 4 minutes (help outlines about 40 |---|---| | requirements.feature: failed requirement exits non-zero | CORE-1 | | global-flags.feature: secrets not written to ee.log | LOG-1 | -| global-flags.feature: `--color`, `--no-color`, `color:` config (3) | new: `php/bootstrap.php` runs InitializeColorization and InitializeLogger before ConfigureRunner, so the runner config is still empty when colors and the logger are chosen | -| global-flags.feature: `--quiet`, `quiet:` config, success with `--quiet` (3) | new: same bootstrap order bug, the Quiet logger is never selected | -| cli-has-command.feature: trailing words / `site ` (2) | new: `find_command_to_run()` stops at the first leaf; `site` is a leaf (`Site_Command::__invoke`) | -| config-file.feature: composite usage prefix | new: `CompositeCommand::show_usage()` counts a skipped disabled subcommand, so the first line loses `usage:` | -| requirements.feature: help without `less` | new: help pipes through `less -r` unless `PAGER` is set; without it nothing is printed and the exit code is 0 | -| cli-alias.feature: ssh alias runs locally | new: `Runner::run_ssh_command()` is never called, so `ee @prod ` with `ssh:` runs on the local machine | -| phar.feature: phar path | new: `php/boot-phar.php` defines `EE_PHAR_PATH` as `getcwd()`, so `cli info` shows the working directory (and `phar_safe_path()` uses it) | +| global-flags.feature: `--color`, `--no-color`, `color:` config, `--quiet`, `quiet:` config, success with `--quiet` (6) | CORE-3: `php/bootstrap.php` sets up colours and the logger before reading the config | +| cli-alias.feature: ssh alias runs locally | CORE-4: `Runner::run_ssh_command()` is never called | +| requirements.feature: help without `less` | CORE-5: nothing printed, exit 0 | +| cli-has-command.feature: trailing words after a leaf | CORE-6: `has-command` stops at the first leaf | +| config-file.feature: composite usage prefix | CORE-7: `CompositeCommand::show_usage()` counts the skipped disabled subcommand | +| phar.feature: phar path | CORE-8: `EE_PHAR_PATH` is `getcwd()` | +| unknown-command.feature: typo reported without docker | CORE-9: checks run before the command lookup | +| help.feature: hint for an unknown site type | CORE-10 (site-command): the hint repeats the bogus type | +| requirements.feature: self-uninstall refusal exit status | CORE-11 | +| cli-alias.feature: `@all` exit status, `@all` running group members twice (2) | CORE-12 | +| cli-completions.feature: unknown `--type` in site completions | CORE-13: uncaught `ReflectionException` | From e850afd622695130a452fc60e4dd4c635e24108e Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 14:51:32 +0530 Subject: [PATCH 13/30] test(easyengine): cover reverting container migrations when the image migration fails A probe container migration and a blocked registry make the upgrade fail after the container migrations ran: the probe's down() runs, its row is removed, and the next attempt runs it again. --- features/bootstrap/FeatureContext.php | 95 +++++++++++++++++++++++++++ features/migration.feature | 23 +++++++ 2 files changed, 118 insertions(+) diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 9f74a3e19..8584a7cc9 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -133,6 +133,20 @@ class FeatureContext implements Context */ private $container_counts = []; + /** + * Core container migrations written by a scenario (file => created its directory), removed afterwards. + * + * @var array + */ + private $probe_migrations = []; + + /** + * /etc/hosts before "Docker Hub is unreachable", restored afterwards. + * + * @var string|null + */ + private static $hosts_backup; + /** * Initializes context. */ @@ -858,6 +872,76 @@ public function theNumberOfContainersWithLabelShouldBeUnchanged( $label ) } } + /** + * @Given the core container migration :file with class :class logs its up and down calls to :log + */ + public function theCoreContainerMigrationLogsTo( $file, $class, $log ) + { + $dir = EE_ROOT . '/migrations/container'; + $created = ! is_dir( $dir ); + if ( $created && ! mkdir( $dir, 0755, true ) ) { + throw new Exception( "Could not create $dir" ); + } + $this->probe_migrations[ $file ] = $created; + $log = var_export( $this->replace_variables( $log ), true ); + $code = "prepare( "SELECT COUNT(*) FROM migrations WHERE migration = ? AND type = 'container'" ); + $query->execute( [ $file ] ); + $count = (int) $query->fetchColumn(); + if ( $negate ? 0 !== $count : 1 !== $count ) { + throw new Exception( "Expected $file to " . ( $negate ? 'not ' : '' ) . "be recorded once, found $count rows." ); + } + } + + /** + * @Given Docker Hub is unreachable + */ + public function dockerHubIsUnreachable() + { + if ( null === self::$hosts_backup ) { + self::$hosts_backup = file_get_contents( '/etc/hosts' ); + register_shutdown_function( [ self::class, 'restoreHosts' ] ); + } + file_put_contents( '/etc/hosts', self::$hosts_backup . "\n127.0.0.1 registry-1.docker.io auth.docker.io index.docker.io\n" ); + exec( 'timeout 60 docker pull -q busybox:1.36 2>&1', $out, $rc ); + if ( 0 === $rc ) { + exec( 'docker rmi busybox:1.36' ); + self::restoreHosts(); + throw new Exception( 'Blocking Docker Hub in /etc/hosts had no effect (a registry mirror?).' ); + } + } + + /** + * @When Docker Hub is reachable again + */ + public function dockerHubIsReachableAgain() + { + self::restoreHosts(); + } + + /** + * Puts back /etc/hosts saved by "Docker Hub is unreachable". + */ + public static function restoreHosts() + { + if ( null !== self::$hosts_backup ) { + file_put_contents( '/etc/hosts', self::$hosts_backup ); + self::$hosts_backup = null; + } + } + /** * @BeforeScenario */ @@ -891,6 +975,17 @@ public function restoreOptions() */ public function cleanupScenario(AfterScenarioScope $scope) { + self::restoreHosts(); + foreach ( $this->probe_migrations as $file => $created_dir ) { + @unlink( EE_ROOT . "/migrations/container/$file" ); + if ( $created_dir ) { + @rmdir( EE_ROOT . '/migrations/container' ); + } + if ( file_exists( self::db_path() ) ) { + self::db()->prepare( 'DELETE FROM migrations WHERE migration = ?' )->execute( [ $file ] ); + } + } + $this->probe_migrations = []; $this->restoreOptions(); $this->container_counts = []; $this->commands = []; diff --git a/features/migration.feature b/features/migration.feature index b0ab423fd..5eb4e7553 100644 --- a/features/migration.feature +++ b/features/migration.feature @@ -91,3 +91,26 @@ Feature: Migration triggering Then the command should exit with code 0 And STDOUT should contain "Executing migrations. This might take some time." And the EasyEngine option "version" should be "{EE_VERSION}" + + # Registry blocked, so the image migration fails after the container migrations ran. They must be reverted + # (down() called, row removed) and run again by the next attempt; earlier ones stay recorded. + @migration + Scenario: A failed image migration reverts the container migrations of the same run + Given no EasyEngine migration is pending + And the core container migration "20990101000000_easyengine_behat_revert_probe.php" with class "BehatRevertProbe" logs its up and down calls to "{TMP}/probe.log" + And the EasyEngine option "version" is set to "4.0.0" + And Docker Hub is unreachable + When I run 'bin/ee cli info' + Then the command should exit with code 1 + And STDERR should contain "Error while executing ee-docker-image-migrations" + And the file "{TMP}/probe.log" should contain "up,down," + And the container migration "20990101000000_easyengine_behat_revert_probe.php" should not be recorded + And the container migration "20210830165250_auth-command_add_frontend_ip.php" should be recorded + And the EasyEngine option "version" should be "4.0.0" + When Docker Hub is reachable again + And I run 'bin/ee cli info' + Then the command should exit with code 0 + And the file "{TMP}/probe.log" should contain "up,down,up," + And the file "{TMP}/probe.log" should not contain "up,down,up,down" + And the container migration "20990101000000_easyengine_behat_revert_probe.php" should be recorded + And the EasyEngine option "version" should be "{EE_VERSION}" From 3e8051e067cdc08afff8e7cff42587618df3f56c Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 14:57:06 +0530 Subject: [PATCH 14/30] test(easyengine): wait for dockerd to re-read /etc/hosts after blocking Docker Hub --- features/bootstrap/FeatureContext.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 8584a7cc9..daade922b 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -915,6 +915,8 @@ public function dockerHubIsUnreachable() register_shutdown_function( [ self::class, 'restoreHosts' ] ); } file_put_contents( '/etc/hosts', self::$hosts_backup . "\n127.0.0.1 registry-1.docker.io auth.docker.io index.docker.io\n" ); + // dockerd's Go resolver re-reads /etc/hosts at most every 5 s. + sleep( 6 ); exec( 'timeout 60 docker pull -q busybox:1.36 2>&1', $out, $rc ); if ( 0 === $rc ) { exec( 'docker rmi busybox:1.36' ); @@ -929,6 +931,7 @@ public function dockerHubIsUnreachable() public function dockerHubIsReachableAgain() { self::restoreHosts(); + sleep( 6 ); } /** From cc47a16cd528d1c0ec5688ca6371a8fed7708e46 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 15:10:22 +0530 Subject: [PATCH 15/30] test(behat): expect remote aliases, --ssh/--http and an ssh config key to be refused The CORE-4 scenario becomes a regular one and checks the error; new scenarios cover a group and @all with an ssh member (a local alias still runs), --ssh/--http, and an ssh: config key that ee config unset can remove. They fail on develop until core fix/refuse-remote-aliases is merged. --- features/COVERAGE.md | 5 ++-- features/cli-alias.feature | 57 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/features/COVERAGE.md b/features/COVERAGE.md index a5e79ec10..32c3e68e2 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -97,7 +97,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | run through a local alias, `@all`, a group, a group with an unknown member, unknown alias suggestion, `@all` without aliases | cli-alias.feature | | a group passes the command's flags to every member | cli-alias.feature | | `@all`/group exit status, `@all` running group members twice | cli-alias.feature `@known-bug` (2, CORE-12) | -| aliases with `ssh:` run remotely | cli-alias.feature `@known-bug` (CORE-4: they run locally) | +| aliases with `ssh:`/`http:`, `--ssh`, `--http`, `ssh:` in config | cli-alias.feature: refused (CORE-4, core `fix/refuse-remote-aliases`; they ran locally before) | | ssh schemes (docker, docker-compose, vagrant, ssh), `EE_SSH_PRE_CMD` | not covered: `run_ssh_command()` is dead code in EE; would need a remote host | ### `ee cli has-command ...` @@ -189,7 +189,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration ### Coverage estimate -The inventory has 107 rows: 93 covered, 1 partly covered and 13 not covered (about 87% of rows, about 78% when rows with several flags are counted per flag). What is left is the real self-update and phar self-delete (unsafe on a live stack), ssh aliases (never wired up, CORE-4), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. +The inventory has 107 rows: 93 covered, 1 partly covered and 13 not covered (about 87% of rows, about 78% when rows with several flags are counted per flag). What is left is the real self-update and phar self-delete (unsafe on a live stack), ssh aliases (never wired up; refused since CORE-4), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. ## How to run @@ -227,7 +227,6 @@ Estimated runtime: default profile about 4 minutes (help outlines about 40 s, fi | requirements.feature: failed requirement exits non-zero | CORE-1 | | global-flags.feature: secrets not written to ee.log | LOG-1 | | global-flags.feature: `--color`, `--no-color`, `color:` config, `--quiet`, `quiet:` config, success with `--quiet` (6) | CORE-3: `php/bootstrap.php` sets up colours and the logger before reading the config | -| cli-alias.feature: ssh alias runs locally | CORE-4: `Runner::run_ssh_command()` is never called | | requirements.feature: help without `less` | CORE-5: nothing printed, exit 0 | | cli-has-command.feature: trailing words after a leaf | CORE-6: `has-command` stops at the first leaf | | config-file.feature: composite usage prefix | CORE-7: `CompositeCommand::show_usage()` counts the skipped disabled subcommand | diff --git a/features/cli-alias.feature b/features/cli-alias.feature index a945b5dea..3bbf05b08 100644 --- a/features/cli-alias.feature +++ b/features/cli-alias.feature @@ -190,9 +190,7 @@ Feature: ee cli alias and @alias execution Then the command should exit with code 1 And STDERR should contain "Error: Cannot use '@all' when no aliases are registered." - # CORE-4: Runner::run_ssh_command() is never called, so an alias with `ssh:` silently runs the - # command on the local machine instead of the remote one. - @known-bug + # CORE-4: remote targets were accepted but never dispatched, so the command ran on the local machine. Scenario: A command for an ssh alias is not run locally Given a file "{TMP}/config.yml" with: """ @@ -204,6 +202,59 @@ Feature: ee cli alias and @alias execution When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-remote cli version' Then the command should fail And STDOUT should not contain "EE {EE_VERSION}" + And STDERR should contain "The alias '@bh-remote' has a remote target (ssh), which is not supported." + + Scenario: A group or @all with an ssh alias is refused before any member runs + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + @bh-one: + path: /var/tmp + @bh-remote: + ssh: bh-nobody@127.0.0.1:1 + @bh-group: + - @bh-one + - @bh-remote + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-group cli version' + Then the command should fail + And STDOUT should not contain "EE {EE_VERSION}" + And STDERR should contain "The alias '@bh-group' has a remote target (ssh)" + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli version' + Then the command should fail + And STDOUT should not contain "EE {EE_VERSION}" + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-one cli version' + Then the command should exit with code 0 + And STDOUT should contain "EE {EE_VERSION}" + + Scenario: --ssh and --http are refused + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee --ssh=bh-nobody@127.0.0.1:1 site list' + Then the command should fail + And STDERR should contain "The --ssh parameter is not supported." + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee site list --http=https://bh.example.test' + Then the command should fail + And STDERR should contain "The --http parameter is not supported." + + Scenario: An ssh key in the config is refused, but ee config can remove it + Given a file "{TMP}/config.yml" with: + """ + locale: en_US + ee_installer_version: stable + ssh: bh-nobody@127.0.0.1:1 + """ + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee site list' + Then the command should fail + And STDERR should contain "The 'ssh' config key is not supported." + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee config unset ssh' + Then the command should exit with code 0 + When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee site list' + Then STDERR should not contain "config key is not supported" Scenario: An alias group passes the command's flags to every member Given a file "{TMP}/config.yml" with: From f9f0543bd7fc2353c804a7d9c0104de966839466 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 17 Oct 2026 15:18:01 +0530 Subject: [PATCH 16/30] test(easyengine): run the LOG-1 argv scenario as a regular one --- features/COVERAGE.md | 5 ++--- features/global-flags.feature | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/features/COVERAGE.md b/features/COVERAGE.md index 32c3e68e2..b3530029b 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -97,7 +97,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | run through a local alias, `@all`, a group, a group with an unknown member, unknown alias suggestion, `@all` without aliases | cli-alias.feature | | a group passes the command's flags to every member | cli-alias.feature | | `@all`/group exit status, `@all` running group members twice | cli-alias.feature `@known-bug` (2, CORE-12) | -| aliases with `ssh:`/`http:`, `--ssh`, `--http`, `ssh:` in config | cli-alias.feature: refused (CORE-4, core `fix/refuse-remote-aliases`; they ran locally before) | +| aliases with `ssh:`/`http:`, `--ssh`, `--http`, `ssh:` in config | cli-alias.feature: refused (CORE-4, fixed in v4.13.2 by easyengine#1945; they ran locally before) | | ssh schemes (docker, docker-compose, vagrant, ssh), `EE_SSH_PRE_CMD` | not covered: `run_ssh_command()` is dead code in EE; would need a remote host | ### `ee cli has-command ...` @@ -150,7 +150,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | missing required parameters | unknown-command.feature | | requirement checks: missing docker-compose, docker retry/backoff | requirements.feature | | failed requirement exit status (CORE-1) | requirements.feature `@known-bug` | -| argv logged to `ee.log` with secrets (LOG-1) | global-flags.feature `@known-bug` | +| argv logged to `ee.log` with secrets (LOG-1) | global-flags.feature, regular since the v4.13.2 fix (easyengine#1944) | | `EE_PACKAGES_DIR` (package autoloader loaded, commands registered) | config-file.feature | | `BEHAT_RUN`, `EE_DISABLE_AUTO_CHECK_UPDATE`, `EE_AUTO_CHECK_UPDATE_DAYS`, `EE_PHP` | not covered: only affect error reporting or the phar+tty auto update (the phar scenarios set `EE_DISABLE_AUTO_CHECK_UPDATE`) | @@ -225,7 +225,6 @@ Estimated runtime: default profile about 4 minutes (help outlines about 40 s, fi | Scenario | Issue | |---|---| | requirements.feature: failed requirement exits non-zero | CORE-1 | -| global-flags.feature: secrets not written to ee.log | LOG-1 | | global-flags.feature: `--color`, `--no-color`, `color:` config, `--quiet`, `quiet:` config, success with `--quiet` (6) | CORE-3: `php/bootstrap.php` sets up colours and the logger before reading the config | | requirements.feature: help without `less` | CORE-5: nothing printed, exit 0 | | cli-has-command.feature: trailing words after a leaf | CORE-6: `has-command` stops at the first leaf | diff --git a/features/global-flags.feature b/features/global-flags.feature index aae50582e..b6e5bfbd0 100644 --- a/features/global-flags.feature +++ b/features/global-flags.feature @@ -117,8 +117,7 @@ Feature: Global parameters (--debug, --quiet, --color) Then the command should exit with code 1 And STDERR should contain "Error: 'verison' is not a registered subcommand of 'cli'." - # LOG-1 (won't fix for now): the full argv is logged at debug level to ee.log, including secrets. - @known-bug + # LOG-1 (fixed in v4.13.2, easyengine#1944), regression check: the full argv used to be logged at debug level to ee.log, secrets included. Scenario: Secrets passed on the command line are not written to ee.log When I run 'bin/ee cli version --pass=bh-secret-{RAND}' Then the command should exit with code 0 From 300bd39545f223fd95cec33f0c13bf6a7f55a00e Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 19 Oct 2026 16:55:52 +0530 Subject: [PATCH 17/30] ci: run the behat suite on GitHub Actions Runs core's features/ on a throwaway runner, with every easyengine/* package on dev-develop. The main job runs the default profile plus @destructive on a fresh runner (EE_BEHAT_ALLOW_INIT=1, the first checked command initialises EasyEngine); the real upgrade migration and the self-uninstall each get their own job on an initialised install. @known-bug scenarios run in a separate non-gating job. Needs the suite's behat.yml profiles and guards from test/behat-coverage. --- .github/workflows/behat.yml | 155 ++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 .github/workflows/behat.yml diff --git a/.github/workflows/behat.yml b/.github/workflows/behat.yml new file mode 100644 index 000000000..874db5d2b --- /dev/null +++ b/.github/workflows/behat.yml @@ -0,0 +1,155 @@ +# Behat suite of EasyEngine core (features/), run on a throwaway runner. +name: Behat + +on: + pull_request: + push: + branches: + - develop + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: behat-${{ github.ref }} + cancel-in-progress: true + +env: + COMPOSE_VERSION: v5.5.1 + # No public DNS (Let's Encrypt) or real backup storage on hosted runners. + EXCLUDE_TAGS: '~@le&&~@remote-real' + COMPOSE_INTERACTIVE_NO_CLI: 1 + # The suite's guards: a fresh runner has no ee.sqlite, and every lane may change global state. + EE_BEHAT_ALLOW_INIT: 1 + EE_BEHAT_DESTRUCTIVE: 1 + +jobs: + behat: + name: Behat ${{ matrix.lane }} (PHP ${{ matrix.php }}) + runs-on: ubuntu-latest + timeout-minutes: 90 + # Known bugs are expected to fail; they're reported, not gating. + continue-on-error: ${{ matrix.lane == 'known-bug' }} + strategy: + fail-fast: false + matrix: + php: ['7.4', '8.5'] + lane: [main, migration, uninstall, known-bug] + include: + # Default profile plus @destructive; the first checked command initialises EasyEngine. + - lane: main + args: --tags="${EXCLUDE_TAGS}&&~@migration&&~@uninstall&&~@known-bug" + # The real upgrade pipeline and the self-uninstall need an initialised install, and a job each. + - lane: migration + args: -p migration + init: true + allow_migration: 1 + - lane: uninstall + args: -p uninstall + init: true + allow_uninstall: 1 + - lane: known-bug + args: --tags="${EXCLUDE_TAGS}&&@known-bug&&~@migration&&~@uninstall" + env: + EE_BEHAT_ALLOW_MIGRATION: ${{ matrix.allow_migration }} + EE_BEHAT_ALLOW_UNINSTALL: ${{ matrix.allow_uninstall }} + steps: + - name: Check out source code + uses: actions/checkout@v7 + + - name: Set up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + coverage: none + tools: composer + extensions: pcntl, curl, sqlite3, zip, dom, mbstring, json + + - name: Get Composer cache directory + id: composer-cache + run: echo "dir=$(composer config -g cache-files-dir)" >> "$GITHUB_OUTPUT" + + - name: Set up Composer caching + uses: actions/cache@v6 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-php${{ matrix.php }}-${{ github.sha }} + restore-keys: ${{ runner.os }}-composer-php${{ matrix.php }}- + + - name: Prepare the host + run: | + # The runner image ships web and database servers; EE needs 80/443. + sudo systemctl stop nginx apache2 mysql 2>/dev/null || true + sudo ss -ltnp | grep -E ':(80|443|3306) ' || true + sudo curl -fsSL "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker version --format 'docker {{.Server.Version}}' + docker-compose version + openssl version + echo "umask $(umask), under sudo $(sudo sh -c umask)" + + - name: Install dependencies + run: | + # Every easyengine/* package on dev-develop, as core CI does on PRs. + php -r ' + $j = json_decode(file_get_contents("composer.json"), true); + foreach ($j["require"] as $k => $v) if (strpos($k, "easyengine/") === 0) $j["require"][$k] = "dev-develop"; + file_put_contents("composer.json", json_encode($j, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n");' + composer update --prefer-dist --no-progress --no-interaction + # Some package commands shell out to `ee` from PATH. + sudo ln -sf "$PWD/bin/ee" /usr/local/bin/ee + cat VERSION + + - name: Initialise EasyEngine + if: matrix.init + run: | + # The first non-cli command runs the initial setup (DB, config, image pulls). + sudo -E bin/ee site list || true + sudo -E bin/ee cli info + + - name: Run Behat + shell: 'script -q -e -c "bash {0}"' + env: + # check-update calls the GitHub API; authenticated requests get a higher rate limit. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + mkdir -p "$RUNNER_TEMP/behat/junit" + # A CLI --tags replaces the profile filter, so the exclusions only apply to the --tags lanes. + sudo -E vendor/bin/behat --strict ${{ matrix.args }} \ + --format=progress --out=std \ + --format=junit --out="$RUNNER_TEMP/behat/junit" \ + --format=pretty --out="$RUNNER_TEMP/behat/pretty.log" + + - name: Collect logs + if: always() + run: | + out="$RUNNER_TEMP/behat" + mkdir -p "$out" + sudo cp /opt/easyengine/logs/ee.log "$out/" 2>/dev/null || echo 'No ee.log.' + sudo stat -c '%a %U:%G %n' /opt/easyengine /opt/easyengine/logs > "$out/dirs.txt" 2>/dev/null || true + sudo docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' > "$out/containers.txt" || true + sudo docker images --format '{{.Repository}}:{{.Tag}}' > "$out/images.txt" || true + sudo chown -R "$USER" "$out" + # One block per failed scenario, from the junit files. + python3 - "$out" <<'PY' + import glob, sys, xml.etree.ElementTree as ET + out = sys.argv[1] + lines = [] + for f in sorted(glob.glob(out + '/junit/*.xml')): + for case in ET.parse(f).getroot().iter('testcase'): + for bad in list(case.findall('failure')) + list(case.findall('error')): + lines.append('### %s :: %s\n%s\n%s\n' % (case.get('classname'), case.get('name'), bad.get('message') or '', bad.text or '')) + open(out + '/failures.txt', 'w').write('\n'.join(lines) or 'No failures.\n') + print('%d failed scenarios' % len(lines)) + PY + head -c 20000 "$out/failures.txt" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v7 + with: + name: behat-${{ matrix.lane }}-php${{ matrix.php }} + path: ${{ runner.temp }}/behat + retention-days: 14 From 5a4f6dcfd9223f08b00e193ccd1f277bc4aae6b2 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 19 Oct 2026 16:58:36 +0530 Subject: [PATCH 18/30] ci: finalize the behat workflow Run PHP 7.4 and 8.5, strip control characters from the junit files before building failures.txt, and cap the Behat step at 75 minutes so a hung run still uploads its logs. --- .github/workflows/behat.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/behat.yml b/.github/workflows/behat.yml index 874db5d2b..85e57d0ed 100644 --- a/.github/workflows/behat.yml +++ b/.github/workflows/behat.yml @@ -110,6 +110,8 @@ jobs: - name: Run Behat shell: 'script -q -e -c "bash {0}"' + # A hung step (e.g. a command stopped on the TTY) should still leave logs. + timeout-minutes: 75 env: # check-update calls the GitHub API; authenticated requests get a higher rate limit. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -134,11 +136,13 @@ jobs: sudo chown -R "$USER" "$out" # One block per failed scenario, from the junit files. python3 - "$out" <<'PY' - import glob, sys, xml.etree.ElementTree as ET + import glob, re, sys, xml.etree.ElementTree as ET out = sys.argv[1] lines = [] for f in sorted(glob.glob(out + '/junit/*.xml')): - for case in ET.parse(f).getroot().iter('testcase'): + # Behat copies raw output (docker logs with ANSI escapes) into the junit file; XML 1.0 forbids those bytes. + data = re.sub(rb'[\x00-\x08\x0b\x0c\x0e-\x1f]', b'', open(f, 'rb').read()) + for case in ET.fromstring(data).iter('testcase'): for bad in list(case.findall('failure')) + list(case.findall('error')): lines.append('### %s :: %s\n%s\n%s\n' % (case.get('classname'), case.get('name'), bad.get('message') or '', bad.text or '')) open(out + '/failures.txt', 'w').write('\n'.join(lines) or 'No failures.\n') From 116aec58548be5965ef388b89ea9877f9c6df6b2 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 19 Oct 2026 17:01:59 +0530 Subject: [PATCH 19/30] ci: re-fetch docker-compose when the first run leaves a broken binary EE's first run downloads docker-compose 2.27.0 without --fail (MIG-10); on a GitHub error it moves the HTML error page over the binary, as on an admin-tools-command runner. Fetch 2.27.0 again with --fail and retries in that case. --- .github/workflows/behat.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/behat.yml b/.github/workflows/behat.yml index 85e57d0ed..a0d2a0eef 100644 --- a/.github/workflows/behat.yml +++ b/.github/workflows/behat.yml @@ -107,6 +107,12 @@ jobs: # The first non-cli command runs the initial setup (DB, config, image pulls). sudo -E bin/ee site list || true sudo -E bin/ee cli info + # The first run's compose download has no --fail (MIG-10): a GitHub error page can replace the binary. + if ! docker-compose version; then + sudo curl -fsSL --retry 5 "https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose version + fi - name: Run Behat shell: 'script -q -e -c "bash {0}"' From 4a90a87a54df127a7bb67b9e685b66273a271304 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 19 Oct 2026 17:04:37 +0530 Subject: [PATCH 20/30] ci(test): stop running the site-command and site-type-wp suites in core's test job Both packages now run their own Behat workflow on every pull request, with tag-filtered lanes: known-bug scenarios never gate, and @le runs only against a local Pebble. Run unfiltered here, their new suites would fail on the known-bug scenarios and on the ones that need Pebble or the local rclone remote. --- .github/workflows/test_and_build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 3884028ae..96e264558 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -151,10 +151,9 @@ jobs: set -e cd $GITHUB_WORKSPACE sudo -E ./vendor/bin/behat + # site-command and site-type-wp run their own Behat suites (with known-bug and Pebble lanes) on every PR. sub_commands=( - vendor/easyengine/site-command/features vendor/easyengine/site-type-php/features - vendor/easyengine/site-type-wp/features ) for command in "${sub_commands[@]}"; do IFS='/' read -r -a array <<< "$command" From 80ba60e7b1c03800ee5f9fe6c24bc200ccccedb2 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 19 Oct 2026 17:07:07 +0530 Subject: [PATCH 21/30] ci(test): let core's suite initialise EasyEngine in the test job The test job runs core's Behat suite on a fresh runner with no ee.sqlite, and the suite refuses to start the first-run setup unless EE_BEHAT_ALLOW_INIT is set. GITHUB_TOKEN keeps the check-update scenarios clear of the unauthenticated API rate limit when all PHP versions run at once. --- .github/workflows/test_and_build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 96e264558..b5005c12f 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -165,6 +165,10 @@ jobs: done env: COMPOSE_INTERACTIVE_NO_CLI: 1 + # Core's suite refuses to initialise EasyEngine unless allowed; the runner starts without ee.sqlite. + EE_BEHAT_ALLOW_INIT: 1 + # check-update calls the GitHub API; authenticated requests get a higher rate limit. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Output logs if: ${{ always() }} From ac5b9cd8fc6adc5db92db612bd92c354d57eb84d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 19 Oct 2026 17:12:39 +0530 Subject: [PATCH 22/30] test(behat): leave the known-bug scenarios out until their fixes land The 17 scenario rows written for open bugs assert the fixed behaviour and fail today, so they stay out of the suite until their fixes land, and COVERAGE.md lists those bugs as not covered yet. Every scenario that ships now gates, and the non-gating known-bug CI lane goes with them. --- .github/workflows/behat.yml | 6 +--- features/COVERAGE.md | 42 +++++++++++++------------- features/cli-alias.feature | 41 ------------------------- features/cli-completions.feature | 7 ----- features/cli-has-command.feature | 8 ----- features/config-file.feature | 14 --------- features/global-flags.feature | 51 -------------------------------- features/help.feature | 7 ----- features/phar.feature | 8 ----- features/requirements.feature | 23 -------------- features/unknown-command.feature | 8 ----- 11 files changed, 23 insertions(+), 192 deletions(-) diff --git a/.github/workflows/behat.yml b/.github/workflows/behat.yml index a0d2a0eef..1a1665d48 100644 --- a/.github/workflows/behat.yml +++ b/.github/workflows/behat.yml @@ -29,13 +29,11 @@ jobs: name: Behat ${{ matrix.lane }} (PHP ${{ matrix.php }}) runs-on: ubuntu-latest timeout-minutes: 90 - # Known bugs are expected to fail; they're reported, not gating. - continue-on-error: ${{ matrix.lane == 'known-bug' }} strategy: fail-fast: false matrix: php: ['7.4', '8.5'] - lane: [main, migration, uninstall, known-bug] + lane: [main, migration, uninstall] include: # Default profile plus @destructive; the first checked command initialises EasyEngine. - lane: main @@ -49,8 +47,6 @@ jobs: args: -p uninstall init: true allow_uninstall: 1 - - lane: known-bug - args: --tags="${EXCLUDE_TAGS}&&@known-bug&&~@migration&&~@uninstall" env: EE_BEHAT_ALLOW_MIGRATION: ${{ matrix.allow_migration }} EE_BEHAT_ALLOW_UNINSTALL: ${{ matrix.allow_uninstall }} diff --git a/features/COVERAGE.md b/features/COVERAGE.md index b3530029b..63ddf27f0 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -26,7 +26,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | invalid `--format`, typo suggestion, extra positional | cli-info.feature (3 scenarios) | | runs the migration check only when docker is up | migration.feature `@destructive`: "cli info skips the version check when docker is not available" and the newer/older version scenarios; requirements.feature: "cli info still works when docker is not available" | | no migration when versions match | cli-info.feature: "cli info lists every diagnostic row" (no "Executing migrations") | -| phar values (`ee_dir_path` = `phar://ee.phar`, phar path) | phar.feature (1 + 1 `@known-bug`) | +| phar values (`ee_dir_path` = `phar://ee.phar`, phar path) | phar.feature (the phar path is not covered yet: CORE-8, its scenario comes back with the fix) | ### `ee cli check-update [--patch] [--minor] [--major] [--field] [--fields] [--format]` @@ -80,7 +80,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | site subcommands and `--type`-aware site options | cli-completions.feature | | aliases (`ee @`, `ee @alias cli ...`) | cli-completions.feature | | unknown command, invalid shell, missing `--line/--point` | cli-completions.feature, unknown-command.feature | -| unknown `--type` in site completions doesn't crash | cli-completions.feature `@known-bug` (CORE-13) | +| unknown `--type` in site completions doesn't crash | not covered yet: CORE-13 (its scenario comes back with the fix) | | does not log to `ee.log` | cli-completions.feature | | `` positional completion, site type from an existing site (`Site::find`) | not covered: no core command has a `` positional; needs a site | @@ -96,7 +96,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | `EE_RUNTIME_ALIAS` replaces file aliases, filters keys and names | cli-alias.feature | | run through a local alias, `@all`, a group, a group with an unknown member, unknown alias suggestion, `@all` without aliases | cli-alias.feature | | a group passes the command's flags to every member | cli-alias.feature | -| `@all`/group exit status, `@all` running group members twice | cli-alias.feature `@known-bug` (2, CORE-12) | +| `@all`/group exit status, `@all` running group members twice | not covered yet: CORE-12 (its 2 scenarios come back with the fix) | | aliases with `ssh:`/`http:`, `--ssh`, `--http`, `ssh:` in config | cli-alias.feature: refused (CORE-4, fixed in v4.13.2 by easyengine#1945; they ran locally before) | | ssh schemes (docker, docker-compose, vagrant, ssh), `EE_SSH_PRE_CMD` | not covered: `run_ssh_command()` is dead code in EE; would need a remote host | @@ -106,7 +106,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | registered commands (quoted and split forms, packages' commands) exit 0 silently | cli-has-command.feature outline (8 rows) | | unknown commands and empty name exit 1 silently | cli-has-command.feature outline (3 rows) | -| trailing words after a leaf (`cli version extra`) | cli-has-command.feature `@known-bug` (CORE-6) | +| trailing words after a leaf (`cli version extra`) | not covered yet: CORE-6 (its scenario comes back with the fix) | | `site ` is "registered" (site is a leaf) | not covered: documented in CORE-6; the passing `site create` rows use the same shortcut | | missing argument prints usage | cli-has-command.feature | | disabled command reported as missing | cli-has-command.feature | @@ -117,7 +117,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | declining the prompt removes nothing | cli.feature `@destructive` | | full uninstall (containers, networks, volumes, images, `/opt/easyengine`; site dirs not checked) | cli.feature `@destructive @uninstall` (existing scenario, TEST-3) | -| refuses when docker is down (and its exit status) | requirements.feature (+1 `@known-bug`, CORE-11) | +| refuses when docker is down (and its exit status) | requirements.feature (the exit status is not covered yet: CORE-11, its scenario comes back with the fix) | | phar self-delete | not covered: needs a phar install, and would delete it | ### `ee help [...]` @@ -126,11 +126,11 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | `ee`, `ee help`, `ee --help` print the same top-level page, with global parameters | help.feature | | every top-level command (12) and subcommand (54) has a help page equal to ` --help` | help.feature outlines | -| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature (+1 `@known-bug`, CORE-10) | +| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature (the unknown-type hint is not covered yet: CORE-10, its scenario comes back with the fix) | | unknown command/subcommand errors and suggestions | help.feature | | `PAGER` (a `sed` pager that prefixes every line), examples section | help.feature | | works without docker | requirements.feature | -| no output without `less` | requirements.feature `@known-bug` (CORE-5) | +| no output without `less` | not covered yet: CORE-5 (its scenario comes back with the fix) | | disabled command: error, left out of `ee help` and `ee help cli` lists | config-file.feature | ### Global parameters and runner behaviour @@ -140,16 +140,16 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | `--debug`, `--debug` before the command, `--debug=`, unknown group, `debug:` in config | global-flags.feature | | `EE_STRICT_ARGS_MODE` | global-flags.feature | | color auto-detection (piped, `SHELL_PIPE=0`) | global-flags.feature | -| `--color`, `--no-color`, `color:` in config | global-flags.feature `@known-bug` (3, CORE-3) | -| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (1 + 3 `@known-bug`, CORE-3) | +| `--color`, `--no-color`, `color:` in config | not covered yet: CORE-3 (its 3 scenarios come back with the fix) | +| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (errors still shown; hiding warnings and success is not covered yet: CORE-3, its 3 scenarios come back with the fix) | | unknown parameter suggestions | cli-info.feature, cli-dump.feature | | `--sites_path`, `--locale`, `--le-mail`, `--env`, `--wp-mail`, `--sysctl`, `--skip-tty`, `--custom-compose` | partly: only `--le-mail` (runtime) and `locale`/`le-mail`/`wp-mail` (file) are exercised (cli-dump.feature, config-file.feature); their effects belong to the package suites | | unknown command/subcommand suggestions, shorthand map (`v`, `check`), no suggestion | unknown-command.feature | -| unknown top-level command runs the checks before the lookup | unknown-command.feature (guarded) + `@known-bug` (CORE-9), migration.feature `@destructive` | -| composite command usage, disabled subcommand hidden | unknown-command.feature, config-file.feature (+1 `@known-bug`) | +| unknown top-level command runs the checks before the lookup | unknown-command.feature (guarded), migration.feature `@destructive`; the typo without docker is not covered yet: CORE-9 (its scenario comes back with the fix) | +| composite command usage, disabled subcommand hidden | unknown-command.feature, config-file.feature (the usage prefix with a disabled first subcommand is not covered yet: CORE-7, its scenario comes back with the fix) | | missing required parameters | unknown-command.feature | | requirement checks: missing docker-compose, docker retry/backoff | requirements.feature | -| failed requirement exit status (CORE-1) | requirements.feature `@known-bug` | +| failed requirement exit status (CORE-1) | not covered yet: CORE-1 (its scenario comes back with the fix) | | argv logged to `ee.log` with secrets (LOG-1) | global-flags.feature, regular since the v4.13.2 fix (easyengine#1944) | | `EE_PACKAGES_DIR` (package autoloader loaded, commands registered) | config-file.feature | | `BEHAT_RUN`, `EE_DISABLE_AUTO_CHECK_UPDATE`, `EE_AUTO_CHECK_UPDATE_DAYS`, `EE_PHP` | not covered: only affect error reporting or the phar+tty auto update (the phar scenarios set `EE_DISABLE_AUTO_CHECK_UPDATE`) | @@ -184,7 +184,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | Item | Covered by | |---|---| | build (from a `--no-dev` copy of the checkout into `/var/tmp`), version, help, bundled packages registered, `cli info` root | phar.feature `@phar` | -| `EE_PHAR_PATH` is the working directory | phar.feature `@known-bug` (CORE-8) | +| `EE_PHAR_PATH` is the working directory | not covered yet: CORE-8 (its scenario comes back with the fix) | ### Coverage estimate @@ -197,16 +197,18 @@ The suite runs from the core root as root (see `/root/workspace/tools/run-behat. | Command | Runs | |---|---| -| `vendor/bin/behat` | everything except `@destructive` (242 scenarios) | +| `vendor/bin/behat` | everything except `@destructive` (228 scenarios) | | `EE_BEHAT_DESTRUCTIVE=1 vendor/bin/behat -p destructive` | `@destructive` without `@migration`/`@uninstall` (19 scenarios: DB version changes, the declined self-uninstall prompt) | -| `EE_BEHAT_DESTRUCTIVE=1 EE_BEHAT_ALLOW_MIGRATION=1 vendor/bin/behat -p migration` | the real upgrade migration (1 scenario; never on a stack in use) | +| `EE_BEHAT_DESTRUCTIVE=1 EE_BEHAT_ALLOW_MIGRATION=1 vendor/bin/behat -p migration` | the real upgrade migration (2 scenarios; never on a stack in use) | | `EE_BEHAT_DESTRUCTIVE=1 EE_BEHAT_ALLOW_UNINSTALL=1 vendor/bin/behat -p uninstall` | `ee cli self-uninstall --yes` (1 scenario; run it alone and last) | -| `EE_BEHAT_DESTRUCTIVE=1 vendor/bin/behat -p all` | everything that can run in one pass (261 scenarios) | -| `vendor/bin/behat --tags='~@destructive&&~@known-bug&&~@network&&~@phar'` | fast, deterministic subset (212 scenarios) | +| `EE_BEHAT_DESTRUCTIVE=1 vendor/bin/behat -p all` | everything that can run in one pass (247 scenarios) | +| `vendor/bin/behat --tags='~@destructive&&~@network&&~@phar'` | fast, deterministic subset (217 scenarios) | A CLI `--tags` replaces the profile filter, so the tags alone can't keep the dangerous scenarios off: a `@BeforeScenario` hook refuses (fails) every `@destructive`, `@migration` or `@uninstall` scenario whose env var (`EE_BEHAT_DESTRUCTIVE`, `EE_BEHAT_ALLOW_MIGRATION`, `EE_BEHAT_ALLOW_UNINSTALL`) isn't set. On a machine without `/opt/easyengine/db/ee.sqlite` (a fresh CI runner) also set `EE_BEHAT_ALLOW_INIT=1`. -Tags: `@destructive` (21, changes global state; restored where possible), `@migration` (1, the real upgrade pipeline), `@uninstall` (1, self-uninstall), `@known-bug` (19, assert the correct behaviour and fail today), `@network` (8, query the GitHub releases API; unauthenticated calls share GitHub's 60/hour limit), `@phar` (5, build a phar, about 1 min). No `@le` or `@remote` scenarios: core has no Let's Encrypt or backup code. +Tags: `@destructive` (22, changes global state; restored where possible), `@migration` (2, the real upgrade pipeline), `@uninstall` (1, self-uninstall), `@network` (7, query the GitHub releases API; unauthenticated calls share GitHub's 60/hour limit), `@phar` (4, build a phar, about 1 min). No `@le` or `@remote` scenarios: core has no Let's Encrypt or backup code. + +No `@known-bug` scenarios ship here: the 17 scenarios written for open bugs (CORE-1, CORE-3, CORE-5 to CORE-13) are held back until their fixes land and then return as regular scenarios. Safety rules built into the suite: @@ -220,9 +222,9 @@ Safety rules built into the suite: Estimated runtime: default profile about 4 minutes (help outlines about 40 s, five 7 s docker-retry scenarios, about 1 min for the phar build, a few seconds of GitHub API calls). `-p migration` takes 1 to 3 min (image pulls, a cron container restart) and `-p uninstall` about 1 min, after which the next site command re-initialises EasyEngine and pulls about 8 GB of images. -## Known bugs exposed (`@known-bug`) +## Known bugs (scenarios held back until fixed) -| Scenario | Issue | +| Held-back scenario | Issue | |---|---| | requirements.feature: failed requirement exits non-zero | CORE-1 | | global-flags.feature: `--color`, `--no-color`, `color:` config, `--quiet`, `quiet:` config, success with `--quiet` (6) | CORE-3: `php/bootstrap.php` sets up colours and the logger before reading the config | diff --git a/features/cli-alias.feature b/features/cli-alias.feature index 3bbf05b08..0cabc114c 100644 --- a/features/cli-alias.feature +++ b/features/cli-alias.feature @@ -272,44 +272,3 @@ Feature: ee cli alias and @alias execution When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @bh-group cli param-dump --with-values --le-mail=bh-group@example.test' Then the command should exit with code 0 And STDOUT should match "#^@bh-one\n\{.*"current":"bh-group@example\.test".*@bh-two\n\{.*"current":"bh-group@example\.test"#s" - - # CORE-12: run_alias_group() ignores the children's exit codes. - @known-bug - Scenario: @all fails when the command fails for an alias - Given a file "{TMP}/config.yml" with: - """ - locale: en_US - ee_installer_version: stable - @bh-one: - path: /var/tmp - @bh-two: - path: /var/tmp - """ - When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli verison' - Then STDERR should contain "Error: 'verison' is not a registered subcommand of 'cli'." - And the command should fail - - # CORE-12: @all also runs the group aliases, so their members run twice. - @known-bug - Scenario: @all runs the command once per alias when groups exist - Given a file "{TMP}/config.yml" with: - """ - locale: en_US - ee_installer_version: stable - @bh-one: - path: /var/tmp - @bh-two: - path: /var/tmp - @bh-group: - - @bh-one - - @bh-two - """ - When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee @all cli version' - Then the command should exit with code 0 - And STDOUT should be: - """ - @bh-one - EE {EE_VERSION} - @bh-two - EE {EE_VERSION} - """ diff --git a/features/cli-completions.feature b/features/cli-completions.feature index efa9cdd64..cc48c2bd7 100644 --- a/features/cli-completions.feature +++ b/features/cli-completions.feature @@ -112,10 +112,3 @@ Feature: ee cli completions When I run 'bin/ee cli completions --line="ee cli bh{RAND}" --point=100' Then the command should exit with code 0 And the file "{EE_ROOT_DIR}/logs/ee.log" should not contain "bh{RAND}" - - # CORE-13: an unknown --type makes CommandFactory reflect an empty class name (uncaught ReflectionException). - @known-bug - Scenario: Completing options for an unknown site type does not crash - When I run 'bin/ee cli completions --line="ee site create --type=bogus --" --point=100' - Then the command should exit with code 0 - And STDOUT should be empty diff --git a/features/cli-has-command.feature b/features/cli-has-command.feature index 0edb55488..3bc9f8d68 100644 --- a/features/cli-has-command.feature +++ b/features/cli-has-command.feature @@ -29,14 +29,6 @@ Feature: ee cli has-command | "foo bar" | | "" | - # CORE-6: find_command_to_run() stops at the first leaf and ignores the remaining words. `site` is a leaf too - # (Site_Command::__invoke routes site types), so "site create" above only passes through the same shortcut and - # "site " returns 0 as well; it is left out here because a fix has to special-case site first. - @known-bug - Scenario: Extra words after a leaf command are not a registered command - When I run 'bin/ee cli has-command "cli version extra"' - Then the command should exit with code 1 - Scenario: has-command without a command prints its usage When I run 'bin/ee cli has-command' Then the command should exit with code 1 diff --git a/features/config-file.feature b/features/config-file.feature index fb2452d77..e60243f9d 100644 --- a/features/config-file.feature +++ b/features/config-file.feature @@ -150,20 +150,6 @@ Feature: Global and project config files And STDOUT should contain "ee cli update" And STDOUT should not contain "ee cli version" - # CORE-7: CompositeCommand::show_usage() counts the skipped disabled subcommand, so the - # first printed line loses its "usage:" prefix when the first subcommand is disabled. - @known-bug - Scenario: The composite usage still starts with "usage:" when its first subcommand is disabled - Given a file "{TMP}/config.yml" with: - """ - locale: en_US - ee_installer_version: stable - disabled_commands: - - cli alias - """ - When I run 'EE_CONFIG_PATH={TMP}/config.yml bin/ee cli' - Then STDOUT should match "#\Ausage: ee cli check-update#" - Scenario: Disabling a command also disables its alias Given a file "{TMP}/config.yml" with: """ diff --git a/features/global-flags.feature b/features/global-flags.feature index b6e5bfbd0..1c8d28070 100644 --- a/features/global-flags.feature +++ b/features/global-flags.feature @@ -61,57 +61,6 @@ Feature: Global parameters (--debug, --quiet, --color) When I run 'SHELL_PIPE=0 bin/ee cli verison' Then STDERR should contain ANSI color codes - # CORE-3: InitializeColorization/InitializeLogger run before ConfigureRunner in - # php/bootstrap.php, so --color, --no-color and `color:` in the config file are ignored. - @known-bug - Scenario: --color forces colors even when piped - When I run 'bin/ee cli verison --color' - Then STDERR should contain ANSI color codes - - @known-bug - Scenario: --no-color disables colors - When I run 'SHELL_PIPE=0 bin/ee cli verison --no-color' - Then STDERR should not contain ANSI color codes - - @known-bug - Scenario: color can be disabled from the config file - Given a file "{TMP}/config.yml" with: - """ - locale: en_US - ee_installer_version: stable - color: false - """ - When I run 'SHELL_PIPE=0 EE_CONFIG_PATH={TMP}/config.yml bin/ee cli verison' - Then STDERR should not contain ANSI color codes - - # CORE-3: same bootstrap order bug, the Quiet logger is never selected, so --quiet - # and `quiet: true` do not hide warnings. The missing docker-compose path warns without a migration. - @known-bug - Scenario: --quiet hides warnings - Given the executable "docker" is linked into "{TMP}/bin" - When I run 'PATH={TMP}/bin {PHP} {EE_ROOT}/php/boot-fs.php site list --quiet' - Then STDERR should contain "Error: EasyEngine requires docker-compose." - And STDERR should not contain "Warning:" - - @known-bug - Scenario: quiet can be enabled from the config file - Given the executable "docker" is linked into "{TMP}/bin" - And a file "{TMP}/config.yml" with: - """ - locale: en_US - ee_installer_version: stable - quiet: true - """ - When I run 'PATH={TMP}/bin EE_CONFIG_PATH={TMP}/config.yml {PHP} {EE_ROOT}/php/boot-fs.php site list' - Then STDERR should contain "Error: EasyEngine requires docker-compose." - And STDERR should not contain "Warning:" - - @known-bug @network - Scenario: --quiet hides success messages - When I run 'bin/ee cli check-update --major --quiet' - Then the command should exit with code 0 - And STDOUT should not contain "Success:" - Scenario: Errors are still printed with --quiet When I run 'bin/ee cli verison --quiet' Then the command should exit with code 1 diff --git a/features/help.feature b/features/help.feature index a19827ba1..38fed9310 100644 --- a/features/help.feature +++ b/features/help.feature @@ -188,13 +188,6 @@ Feature: ee help Then the command should exit with code 1 And STDERR should contain "Error: 'bogus' is not a registered site type" - # CORE-10 (site-command): the hint repeats the bogus type, and that command fails the same way. - @known-bug - Scenario: help for an unknown site type points to a working help command - When I run 'bin/ee help site create --type=bogus' - Then the command should exit with code 1 - And STDERR should not contain "ee help site --type=bogus" - Scenario: help for an unknown command fails with a hint When I run 'bin/ee help nosuchcommand' Then the command should exit with code 1 diff --git a/features/phar.feature b/features/phar.feature index 413155df6..7ea22a2bd 100644 --- a/features/phar.feature +++ b/features/phar.feature @@ -35,11 +35,3 @@ Feature: Phar build Then the command should exit with code 0 And the json in STDOUT at "ee_dir_path" should be "phar://ee.phar" And the json in STDOUT at "ee_version" should be "{EE_VERSION}" - - # CORE-8: php/boot-phar.php defines EE_PHAR_PATH as getcwd(), so "EE phar path" shows the working directory. - @known-bug - Scenario: The phar cli info shows the path of the phar file - Given no EasyEngine migration is pending - When I run 'cd {TMP} && COLUMNS=250 {PHP} {PHAR} cli info' - Then the command should exit with code 0 - And STDOUT should match "#^\| EE phar path\s+\| {PHAR}\s+\|#m" diff --git a/features/requirements.feature b/features/requirements.feature index d5f4276cd..290c9409f 100644 --- a/features/requirements.feature +++ b/features/requirements.feature @@ -9,13 +9,6 @@ Feature: Requirement checks Then the command should exit with code 0 And STDOUT should contain "ee cli" - # CORE-5: help pipes through `less -r` unless PAGER is set; without less it prints nothing and exits 0. - @known-bug - Scenario: help still prints when the default pager is not installed - Given the directory "{TMP}/emptybin" exists - When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php help cli' - Then STDOUT should contain "ee cli" - Scenario: cli info still works when docker is not available # Requirements are only probed (4 attempts, ~7s of backoff), so no migration can run. Given the directory "{TMP}/emptybin" exists @@ -30,14 +23,6 @@ Feature: Requirement checks And STDERR should contain "Error: EasyEngine requires docker-compose." And STDOUT should not contain "Executing migrations" - # CORE-1: a failed requirement exits with status 0 (`EE::error( ..., false ); die;`). - @known-bug - Scenario: A failed requirement exits with a non-zero status - Given the executable "docker" is linked into "{TMP}/bin" - When I run 'PATH={TMP}/bin {PHP} {EE_ROOT}/php/boot-fs.php site list' - Then STDERR should contain "Error: EasyEngine requires docker-compose." - And the command should fail - Scenario: A checked command retries the docker probe before giving up Given the directory "{TMP}/emptybin" exists When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php site list --debug' @@ -51,11 +36,3 @@ Feature: Requirement checks When I run 'echo n | PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli self-uninstall' Then STDERR should contain "Error: Unable to proceed with uninstallation. Seems there is a dependency down." And STDOUT should not contain "Are you sure" - - # CORE-11: the refusal uses `EE::error( ..., false ); die;`, so it exits 0 like CORE-1. - @known-bug - Scenario: self-uninstall refuses with a non-zero status when docker is not available - Given the directory "{TMP}/emptybin" exists - When I run 'echo n | PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php cli self-uninstall' - Then STDERR should contain "Unable to proceed with uninstallation" - And the command should fail diff --git a/features/unknown-command.feature b/features/unknown-command.feature index 8f6bdc06c..90c6da831 100644 --- a/features/unknown-command.feature +++ b/features/unknown-command.feature @@ -69,11 +69,3 @@ Feature: Unknown commands and composite commands Then the command should exit with code 1 And STDERR should contain "missing --line parameter" And STDERR should contain "missing --point parameter" - - # CORE-9: without docker the requirement check fails first and hides the typo. - @known-bug - Scenario: An unknown top-level command is reported even when docker is not available - Given the directory "{TMP}/emptybin" exists - When I run 'PATH={TMP}/emptybin {PHP} {EE_ROOT}/php/boot-fs.php sitee' - Then the command should exit with code 1 - And STDERR should contain "Error: 'sitee' is not a registered ee command." From ac937b70f388ad2789f3be848c1307c552cf4add Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 01:50:40 +0530 Subject: [PATCH 23/30] test(easyengine): guard the last site list of the ssh-key scenario and drop the unused site cleanup After ee config unset ssh the scenario runs a full site list, which passes the requirement checks and could start a migration or the first-run setup; it now starts with the no-pending-migration guard. The @AfterFeature hook deleted bh-easyengine-* sites although the suite creates none, and its site delete could run a migration too, so it is gone. --- features/COVERAGE.md | 2 +- features/bootstrap/FeatureContext.php | 18 ------------------ features/cli-alias.feature | 4 +++- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/features/COVERAGE.md b/features/COVERAGE.md index 63ddf27f0..f0b28b802 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -217,7 +217,7 @@ Safety rules built into the suite: - The phar is built from a `--no-dev` copy of the checkout under `/var/tmp/ee-behat-core/phar-/` and removed after the suite; the checkout, its `vendor/` autoloader and `/usr/local/bin/ee` are never touched, `cli update` is never run from the phar and `help` runs with `EE_DISABLE_AUTO_CHECK_UPDATE=1`. - Config scenarios point `EE_CONFIG_PATH` at a per-scenario file under `/var/tmp/ee-behat-core/` (`{TMP}`), removed after the scenario; the real `/opt/easyengine/config/config.yml` is never written (scenarios without `EE_CONFIG_PATH` read it, so `debug:`, aliases or `disabled_commands` there would break exact-output checks). - Requirement scenarios hide docker/docker-compose with a private `PATH` instead of stopping anything. -- The suite creates no sites. The `@AfterFeature` cleanup only deletes sites named `bh-easyengine-*`. +- The suite creates no sites and never deletes any. - Commands are run through `EE\Process` directly, not `EE::launch()`, so the harness does not write test command lines into `ee.log`. Estimated runtime: default profile about 4 minutes (help outlines about 40 s, five 7 s docker-retry scenarios, about 1 min for the phar build, a few seconds of GitHub API calls). `-p migration` takes 1 to 3 min (image pulls, a cron container restart) and `-p uninstall` about 1 min, after which the next site command re-initialises EasyEngine and pulls about 8 GB of images. diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index daade922b..14e247ba7 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -56,7 +56,6 @@ /* End. Loading required files to enable EE::launch() in tests. */ use Behat\Behat\Context\Context; -use Behat\Behat\Hook\Scope\AfterFeatureScope; use Behat\Behat\Hook\Scope\AfterScenarioScope; use Behat\Behat\Hook\Scope\BeforeScenarioScope; @@ -71,11 +70,6 @@ class FeatureContext implements Context public $webroot_path; public $ee_path; - /** - * Prefix of every site this suite may create; cleanup never touches other sites. - */ - const SITE_PREFIX = 'bh-easyengine-'; - /** * Base directory of the per-scenario scratch directories ({TMP}). */ @@ -1002,18 +996,6 @@ public function cleanupScenario(AfterScenarioScope $scope) chdir($this->ee_path); } - /** - * @AfterFeature - */ - public static function cleanup(AfterFeatureScope $scope) - { - // Only this suite's sites, never every site on the host. - foreach ( glob( EE_SITE_ROOT . '/' . self::SITE_PREFIX . '*', GLOB_ONLYDIR ) ?: [] as $site_dir ) { - exec( 'bin/ee site delete ' . escapeshellarg( basename( $site_dir ) ) . ' --yes' ); - } - - } - /** * @AfterSuite */ diff --git a/features/cli-alias.feature b/features/cli-alias.feature index 0cabc114c..98262ee8e 100644 --- a/features/cli-alias.feature +++ b/features/cli-alias.feature @@ -242,7 +242,9 @@ Feature: ee cli alias and @alias execution And STDERR should contain "The --http parameter is not supported." Scenario: An ssh key in the config is refused, but ee config can remove it - Given a file "{TMP}/config.yml" with: + # The last site list passes the refusal and runs the requirement checks. + Given no EasyEngine migration is pending + And a file "{TMP}/config.yml" with: """ locale: en_US ee_installer_version: stable From a6b1c58f2b12bf04d9a90e4fd6c43c5dec64e5f9 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 02:09:08 +0530 Subject: [PATCH 24/30] test(easyengine): make sure the image migration rollback scenario has an image to pull With Docker Hub blocked the image migration only fails if it pulls something. On a fresh runner mailhog and php have no recorded tag, but a host where every pullable image already runs at its target tag pulls nothing, so the scenario would fail against correct code. A new step records a stale tag (restored after the scenario) for one image that isn't running when nothing would be pulled. --- features/bootstrap/FeatureContext.php | 34 +++++++++++++++++++++++++++ features/migration.feature | 1 + 2 files changed, 35 insertions(+) diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 14e247ba7..075899a61 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -832,6 +832,40 @@ public function theEasyEngineOptionIsSetTo( $key, $value ) self::set_option( $key, $this->replace_variables( $value ) ); } + /** + * Without a pull, a blocked registry can't fail the image migration: if every image it would pull + * already runs (or is recorded) at its target tag, record a stale tag for one that isn't running. + * + * @Given the next image migration pulls at least one image + */ + public function theNextImageMigrationPullsAtLeastOneImage() + { + $running = []; + foreach ( array_filter( explode( "\n", (string) shell_exec( "docker ps --format '{{.Image}}'" ) ) ) as $image ) { + $parts = explode( ':', $image, 2 ); + $running[ $parts[0] ] = isset( $parts[1] ) ? $parts[1] : null; + } + $free = null; + foreach ( json_decode( file_get_contents( EE_ROOT . '/img-versions.json' ), true ) as $image => $version ) { + // Same exclusions as Containers::start_container_migration(). + if ( preg_match( '#^easyengine/(php\d|newrelic-daemon)#', $image ) ) { + continue; + } + // Running containers take precedence over the options table, as in the migration. + $current = array_key_exists( $image, $running ) ? $running[ $image ] : self::get_option( $image ); + if ( $current !== $version ) { + return; + } + if ( null === $free && ! array_key_exists( $image, $running ) ) { + $free = $image; + } + } + if ( null === $free ) { + throw new Exception( 'Every image the migration could pull runs at its target tag, so no pull can fail.' ); + } + $this->theEasyEngineOptionIsSetTo( $free, 'v0.0.0-behat-stale' ); + } + /** * @Then the EasyEngine option :key should be :value */ diff --git a/features/migration.feature b/features/migration.feature index 5eb4e7553..78d96707b 100644 --- a/features/migration.feature +++ b/features/migration.feature @@ -99,6 +99,7 @@ Feature: Migration triggering Given no EasyEngine migration is pending And the core container migration "20990101000000_easyengine_behat_revert_probe.php" with class "BehatRevertProbe" logs its up and down calls to "{TMP}/probe.log" And the EasyEngine option "version" is set to "4.0.0" + And the next image migration pulls at least one image And Docker Hub is unreachable When I run 'bin/ee cli info' Then the command should exit with code 1 From 188e210e145920cd9b1a6c28d0abbe2d14dadc21 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 02:25:56 +0530 Subject: [PATCH 25/30] test(easyengine): drop the unused subsite step No scenario uses it, and it swapped the site and the subsite and still used the pre-compose-v2 container name. --- features/bootstrap/FeatureContext.php | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 075899a61..2d1693da9 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -203,16 +203,6 @@ public function iRun($command) $this->commands[] = \EE\Process::create( $this->replace_variables( $command ), null, null )->run(); } - /** - * @When I create subsite :subsite in :site - */ - public function iCreateSubsiteInOfType($subsite, $site ) - { - $php_container = implode( explode( '.', $subsite ) ) . '_php_1'; - - EE::launch( "docker exec --user='www-data' $php_container sh -c 'wp site create --slug=$site'", false, true ); - } - /** * @Then return value should be 0 */ From 3a8bad5135544101edaa3dea848f4265aecfc45b Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 09:22:32 +0530 Subject: [PATCH 26/30] docs(easyengine): describe coverage gaps without workspace references Replace internal tracker IDs and local paths in the coverage notes and test comments with short descriptions of each bug or links to the pull requests that fixed it. --- .github/workflows/behat.yml | 2 +- features/COVERAGE.md | 66 ++++++++++++++++---------------- features/cli-alias.feature | 2 +- features/cli.feature | 2 +- features/global-flags.feature | 2 +- features/migration.feature | 2 +- features/unknown-command.feature | 2 +- 7 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/behat.yml b/.github/workflows/behat.yml index 1a1665d48..f3e34b6ef 100644 --- a/.github/workflows/behat.yml +++ b/.github/workflows/behat.yml @@ -103,7 +103,7 @@ jobs: # The first non-cli command runs the initial setup (DB, config, image pulls). sudo -E bin/ee site list || true sudo -E bin/ee cli info - # The first run's compose download has no --fail (MIG-10): a GitHub error page can replace the binary. + # The first run's compose download has no --fail: a GitHub error page can replace the binary. if ! docker-compose version; then sudo curl -fsSL --retry 5 "https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose diff --git a/features/COVERAGE.md b/features/COVERAGE.md index f0b28b802..a64ad1a00 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -26,7 +26,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | invalid `--format`, typo suggestion, extra positional | cli-info.feature (3 scenarios) | | runs the migration check only when docker is up | migration.feature `@destructive`: "cli info skips the version check when docker is not available" and the newer/older version scenarios; requirements.feature: "cli info still works when docker is not available" | | no migration when versions match | cli-info.feature: "cli info lists every diagnostic row" (no "Executing migrations") | -| phar values (`ee_dir_path` = `phar://ee.phar`, phar path) | phar.feature (the phar path is not covered yet: CORE-8, its scenario comes back with the fix) | +| phar values (`ee_dir_path` = `phar://ee.phar`, phar path) | phar.feature (the phar path is not covered yet: `EE_PHAR_PATH` is the working directory, its scenario comes back with the fix) | ### `ee cli check-update [--patch] [--minor] [--major] [--field] [--fields] [--format]` @@ -46,7 +46,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | Item | Covered by | |---|---| | refuses to run from source (`You can only self-update Phar files.`) with every flag combination | cli-update.feature outline (4 rows) | -| real update (prompt, download, md5 check, child `cli info`, phar swap), macOS brew message, not-writable errors | not covered on purpose: an accepted prompt downloads a phar whose `cli info` runs the new version's migrations on the live DB (MIG-6); a declined prompt or an immutable (`chattr +i`) phar would be testable but a regression there would do exactly that; not on macOS | +| real update (prompt, download, md5 check, child `cli info`, phar swap), macOS brew message, not-writable errors | not covered on purpose: an accepted prompt downloads a phar whose `cli info` runs the new version's migrations on the live DB; a declined prompt or an immutable (`chattr +i`) phar would be testable but a regression there would do exactly that; not on macOS | | `auto_check_update()` on help (phar + tty only) | not covered: needs a tty and a writable phar | ### `ee cli param-dump [--with-values] [--format=json|var_export]` @@ -80,7 +80,7 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | site subcommands and `--type`-aware site options | cli-completions.feature | | aliases (`ee @`, `ee @alias cli ...`) | cli-completions.feature | | unknown command, invalid shell, missing `--line/--point` | cli-completions.feature, unknown-command.feature | -| unknown `--type` in site completions doesn't crash | not covered yet: CORE-13 (its scenario comes back with the fix) | +| unknown `--type` in site completions doesn't crash | not covered yet: it dies with an uncaught `ReflectionException` (its scenario comes back with the fix) | | does not log to `ee.log` | cli-completions.feature | | `` positional completion, site type from an existing site (`Site::find`) | not covered: no core command has a `` positional; needs a site | @@ -96,8 +96,8 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | `EE_RUNTIME_ALIAS` replaces file aliases, filters keys and names | cli-alias.feature | | run through a local alias, `@all`, a group, a group with an unknown member, unknown alias suggestion, `@all` without aliases | cli-alias.feature | | a group passes the command's flags to every member | cli-alias.feature | -| `@all`/group exit status, `@all` running group members twice | not covered yet: CORE-12 (its 2 scenarios come back with the fix) | -| aliases with `ssh:`/`http:`, `--ssh`, `--http`, `ssh:` in config | cli-alias.feature: refused (CORE-4, fixed in v4.13.2 by easyengine#1945; they ran locally before) | +| `@all`/group exit status, `@all` running group members twice | not covered yet: alias groups and `@all` ignore the children's exit codes (its 2 scenarios come back with the fix) | +| aliases with `ssh:`/`http:`, `--ssh`, `--http`, `ssh:` in config | cli-alias.feature: refused (fixed in v4.13.2 by EasyEngine/easyengine#1945; they ran locally before) | | ssh schemes (docker, docker-compose, vagrant, ssh), `EE_SSH_PRE_CMD` | not covered: `run_ssh_command()` is dead code in EE; would need a remote host | ### `ee cli has-command ...` @@ -106,8 +106,8 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | registered commands (quoted and split forms, packages' commands) exit 0 silently | cli-has-command.feature outline (8 rows) | | unknown commands and empty name exit 1 silently | cli-has-command.feature outline (3 rows) | -| trailing words after a leaf (`cli version extra`) | not covered yet: CORE-6 (its scenario comes back with the fix) | -| `site ` is "registered" (site is a leaf) | not covered: documented in CORE-6; the passing `site create` rows use the same shortcut | +| trailing words after a leaf (`cli version extra`) | not covered yet: `has-command` stops at the first leaf (its scenario comes back with the fix) | +| `site ` is "registered" (site is a leaf) | not covered: same first-leaf shortcut as the row above; the passing `site create` rows use the same shortcut | | missing argument prints usage | cli-has-command.feature | | disabled command reported as missing | cli-has-command.feature | @@ -116,8 +116,8 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | Item | Covered by | |---|---| | declining the prompt removes nothing | cli.feature `@destructive` | -| full uninstall (containers, networks, volumes, images, `/opt/easyengine`; site dirs not checked) | cli.feature `@destructive @uninstall` (existing scenario, TEST-3) | -| refuses when docker is down (and its exit status) | requirements.feature (the exit status is not covered yet: CORE-11, its scenario comes back with the fix) | +| full uninstall (containers, networks, volumes, images, `/opt/easyengine`; site dirs not checked) | cli.feature `@destructive @uninstall` (the existing scenario) | +| refuses when docker is down (and its exit status) | requirements.feature (the exit status is not covered yet: the refusal exits 0, its scenario comes back with the fix) | | phar self-delete | not covered: needs a phar install, and would delete it | ### `ee help [...]` @@ -126,11 +126,11 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration |---|---| | `ee`, `ee help`, `ee --help` print the same top-level page, with global parameters | help.feature | | every top-level command (12) and subcommand (54) has a help page equal to ` --help` | help.feature outlines | -| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature (the unknown-type hint is not covered yet: CORE-10, its scenario comes back with the fix) | +| site type routing (`--type=html|wp|php`, default html, wp-only options, unknown type) | help.feature (the unknown-type hint is not covered yet: it repeats the bogus type, its scenario comes back with the fix) | | unknown command/subcommand errors and suggestions | help.feature | | `PAGER` (a `sed` pager that prefixes every line), examples section | help.feature | | works without docker | requirements.feature | -| no output without `less` | not covered yet: CORE-5 (its scenario comes back with the fix) | +| no output without `less` | not covered yet: nothing is printed and the exit status is 0 (its scenario comes back with the fix) | | disabled command: error, left out of `ee help` and `ee help cli` lists | config-file.feature | ### Global parameters and runner behaviour @@ -140,17 +140,17 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | `--debug`, `--debug` before the command, `--debug=`, unknown group, `debug:` in config | global-flags.feature | | `EE_STRICT_ARGS_MODE` | global-flags.feature | | color auto-detection (piped, `SHELL_PIPE=0`) | global-flags.feature | -| `--color`, `--no-color`, `color:` in config | not covered yet: CORE-3 (its 3 scenarios come back with the fix) | -| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (errors still shown; hiding warnings and success is not covered yet: CORE-3, its 3 scenarios come back with the fix) | +| `--color`, `--no-color`, `color:` in config | not covered yet: colours and the logger are set up before the config is read (its 3 scenarios come back with the fix) | +| `--quiet`, `quiet:` in config (warnings, success), errors still shown | global-flags.feature (errors still shown; hiding warnings and success is not covered yet for the same reason, its 3 scenarios come back with the fix) | | unknown parameter suggestions | cli-info.feature, cli-dump.feature | | `--sites_path`, `--locale`, `--le-mail`, `--env`, `--wp-mail`, `--sysctl`, `--skip-tty`, `--custom-compose` | partly: only `--le-mail` (runtime) and `locale`/`le-mail`/`wp-mail` (file) are exercised (cli-dump.feature, config-file.feature); their effects belong to the package suites | | unknown command/subcommand suggestions, shorthand map (`v`, `check`), no suggestion | unknown-command.feature | -| unknown top-level command runs the checks before the lookup | unknown-command.feature (guarded), migration.feature `@destructive`; the typo without docker is not covered yet: CORE-9 (its scenario comes back with the fix) | -| composite command usage, disabled subcommand hidden | unknown-command.feature, config-file.feature (the usage prefix with a disabled first subcommand is not covered yet: CORE-7, its scenario comes back with the fix) | +| unknown top-level command runs the checks before the lookup | unknown-command.feature (guarded), migration.feature `@destructive`; the typo without docker is not covered yet: the checks run before the command lookup (its scenario comes back with the fix) | +| composite command usage, disabled subcommand hidden | unknown-command.feature, config-file.feature (the usage prefix with a disabled first subcommand is not covered yet: the prefix counts the skipped subcommand, its scenario comes back with the fix) | | missing required parameters | unknown-command.feature | | requirement checks: missing docker-compose, docker retry/backoff | requirements.feature | -| failed requirement exit status (CORE-1) | not covered yet: CORE-1 (its scenario comes back with the fix) | -| argv logged to `ee.log` with secrets (LOG-1) | global-flags.feature, regular since the v4.13.2 fix (easyengine#1944) | +| failed requirement exit status | not covered yet: a failed requirement exits 0 (its scenario comes back with the fix) | +| argv logged to `ee.log` with secrets | global-flags.feature, regular since the v4.13.2 fix (EasyEngine/easyengine#1944) | | `EE_PACKAGES_DIR` (package autoloader loaded, commands registered) | config-file.feature | | `BEHAT_RUN`, `EE_DISABLE_AUTO_CHECK_UPDATE`, `EE_AUTO_CHECK_UPDATE_DAYS`, `EE_PHP` | not covered: only affect error reporting or the phar+tty auto update (the phar scenarios set `EE_DISABLE_AUTO_CHECK_UPDATE`) | @@ -177,23 +177,23 @@ Core owns the runner (bootstrap, config, aliases, requirement checks, migration | nightly suffix stripping itself | not covered: a distinguishing test would run a real migration when it fails | | DB version older: full pipeline runs, version bumped | migration.feature `@destructive @migration` (never on a stack in use) | | versions equal: nothing runs | cli-info.feature | -| fresh install (`ee successfully setup.`), nightly VERSION always migrating, 5 GB disk check, rollback | not covered: needs a wiped machine, a modified VERSION or a full disk; MIG-1..19 are architectural and need the live upgrade tests | +| fresh install (`ee successfully setup.`), nightly VERSION always migrating, 5 GB disk check, rollback | not covered: needs a wiped machine, a modified VERSION or a full disk; the rest of the upgrade pipeline needs live upgrade tests | ### Phar | Item | Covered by | |---|---| | build (from a `--no-dev` copy of the checkout into `/var/tmp`), version, help, bundled packages registered, `cli info` root | phar.feature `@phar` | -| `EE_PHAR_PATH` is the working directory | not covered yet: CORE-8 (its scenario comes back with the fix) | +| `EE_PHAR_PATH` is the working directory | not covered yet (its scenario comes back with the fix) | ### Coverage estimate -The inventory has 107 rows: 93 covered, 1 partly covered and 13 not covered (about 87% of rows, about 78% when rows with several flags are counted per flag). What is left is the real self-update and phar self-delete (unsafe on a live stack), ssh aliases (never wired up; refused since CORE-4), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. +The inventory has 107 rows: 93 covered, 1 partly covered and 13 not covered (about 87% of rows, about 78% when rows with several flags are counted per flag). What is left is the real self-update and phar self-delete (unsafe on a live stack), ssh aliases (never wired up; refused since EasyEngine/easyengine#1945), permission errors that root cannot hit, fresh-install and nightly migration paths, and output that depends on live GitHub release data. ## How to run -The suite runs from the core root as root (see `/root/workspace/tools/run-behat.sh core`). +The suite runs from the core root as root. | Command | Runs | |---|---| @@ -208,7 +208,7 @@ A CLI `--tags` replaces the profile filter, so the tags alone can't keep the dan Tags: `@destructive` (22, changes global state; restored where possible), `@migration` (2, the real upgrade pipeline), `@uninstall` (1, self-uninstall), `@network` (7, query the GitHub releases API; unauthenticated calls share GitHub's 60/hour limit), `@phar` (4, build a phar, about 1 min). No `@le` or `@remote` scenarios: core has no Let's Encrypt or backup code. -No `@known-bug` scenarios ship here: the 17 scenarios written for open bugs (CORE-1, CORE-3, CORE-5 to CORE-13) are held back until their fixes land and then return as regular scenarios. +No `@known-bug` scenarios ship here: the 17 scenarios written for open bugs (listed under "Known bugs" below) are held back until their fixes land and then return as regular scenarios. Safety rules built into the suite: @@ -226,14 +226,14 @@ Estimated runtime: default profile about 4 minutes (help outlines about 40 s, fi | Held-back scenario | Issue | |---|---| -| requirements.feature: failed requirement exits non-zero | CORE-1 | -| global-flags.feature: `--color`, `--no-color`, `color:` config, `--quiet`, `quiet:` config, success with `--quiet` (6) | CORE-3: `php/bootstrap.php` sets up colours and the logger before reading the config | -| requirements.feature: help without `less` | CORE-5: nothing printed, exit 0 | -| cli-has-command.feature: trailing words after a leaf | CORE-6: `has-command` stops at the first leaf | -| config-file.feature: composite usage prefix | CORE-7: `CompositeCommand::show_usage()` counts the skipped disabled subcommand | -| phar.feature: phar path | CORE-8: `EE_PHAR_PATH` is `getcwd()` | -| unknown-command.feature: typo reported without docker | CORE-9: checks run before the command lookup | -| help.feature: hint for an unknown site type | CORE-10 (site-command): the hint repeats the bogus type | -| requirements.feature: self-uninstall refusal exit status | CORE-11 | -| cli-alias.feature: `@all` exit status, `@all` running group members twice (2) | CORE-12 | -| cli-completions.feature: unknown `--type` in site completions | CORE-13: uncaught `ReflectionException` | +| requirements.feature: failed requirement exits non-zero | `check_requirements()` exits 0 after the error | +| global-flags.feature: `--color`, `--no-color`, `color:` config, `--quiet`, `quiet:` config, success with `--quiet` (6) | `php/bootstrap.php` sets up colours and the logger before reading the config | +| requirements.feature: help without `less` | nothing printed, exit 0 | +| cli-has-command.feature: trailing words after a leaf | `has-command` stops at the first leaf | +| config-file.feature: composite usage prefix | `CompositeCommand::show_usage()` counts the skipped disabled subcommand | +| phar.feature: phar path | `EE_PHAR_PATH` is `getcwd()` | +| unknown-command.feature: typo reported without docker | checks run before the command lookup | +| help.feature: hint for an unknown site type | site-command: the hint repeats the bogus type | +| requirements.feature: self-uninstall refusal exit status | the refusal exits 0 | +| cli-alias.feature: `@all` exit status, `@all` running group members twice (2) | the children's exit codes are ignored; `@all` also runs the group keys | +| cli-completions.feature: unknown `--type` in site completions | uncaught `ReflectionException` | diff --git a/features/cli-alias.feature b/features/cli-alias.feature index 98262ee8e..39db9c35c 100644 --- a/features/cli-alias.feature +++ b/features/cli-alias.feature @@ -190,7 +190,7 @@ Feature: ee cli alias and @alias execution Then the command should exit with code 1 And STDERR should contain "Error: Cannot use '@all' when no aliases are registered." - # CORE-4: remote targets were accepted but never dispatched, so the command ran on the local machine. + # Fixed in v4.13.2 (EasyEngine/easyengine#1945): remote targets were accepted but never dispatched, so the command ran on the local machine. Scenario: A command for an ssh alias is not run locally Given a file "{TMP}/config.yml" with: """ diff --git a/features/cli.feature b/features/cli.feature index 8dd0b3a37..0f969531e 100644 --- a/features/cli.feature +++ b/features/cli.feature @@ -1,5 +1,5 @@ # The @uninstall scenario wipes every EasyEngine container, volume, network, image, /opt/easyengine and all -# sites (TEST-3). It is excluded from every profile but `uninstall`, refused without EE_BEHAT_ALLOW_UNINSTALL=1, +# sites. It is excluded from every profile but `uninstall`, refused without EE_BEHAT_ALLOW_UNINSTALL=1, # and must run alone and last (later scenarios would re-initialise EasyEngine). @destructive Feature: CLI Command diff --git a/features/global-flags.feature b/features/global-flags.feature index 1c8d28070..4289a77ac 100644 --- a/features/global-flags.feature +++ b/features/global-flags.feature @@ -66,7 +66,7 @@ Feature: Global parameters (--debug, --quiet, --color) Then the command should exit with code 1 And STDERR should contain "Error: 'verison' is not a registered subcommand of 'cli'." - # LOG-1 (fixed in v4.13.2, easyengine#1944), regression check: the full argv used to be logged at debug level to ee.log, secrets included. + # Fixed in v4.13.2 (EasyEngine/easyengine#1944), regression check: the full argv used to be logged at debug level to ee.log, secrets included. Scenario: Secrets passed on the command line are not written to ee.log When I run 'bin/ee cli version --pass=bh-secret-{RAND}' Then the command should exit with code 0 diff --git a/features/migration.feature b/features/migration.feature index 78d96707b..a75a73b66 100644 --- a/features/migration.feature +++ b/features/migration.feature @@ -81,7 +81,7 @@ Feature: Migration triggering Then the command should exit with code 1 And STDERR should contain "Error: It seems you're not running latest version." - # Runs a real migration: image pulls (MIG-4), cron container restart and log rotation (MIG-5). + # Runs a real migration: image pulls, cron container restart and log rotation. # Never on a stack in use: excluded from every profile but `migration`, and refused without EE_BEHAT_ALLOW_MIGRATION=1. @migration Scenario: An older DB version triggers the upgrade on the next checked command diff --git a/features/unknown-command.feature b/features/unknown-command.feature index 90c6da831..a2ea459a3 100644 --- a/features/unknown-command.feature +++ b/features/unknown-command.feature @@ -29,7 +29,7 @@ Feature: Unknown commands and composite commands And STDERR should contain "Error: 'zzzzzzzzzz' is not a registered subcommand of 'cli'." And STDERR should not contain "Did you mean" - # CORE-9: unknown top-level commands pass the requirement and migration checks before the lookup. + # Unknown top-level commands pass the requirement and migration checks before the lookup. Scenario: An unknown top-level command suggests the closest one Given no EasyEngine migration is pending When I run 'bin/ee sitee' From 0610274a1a04070565da743ae952d2086ed5cfd0 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 09:34:40 +0530 Subject: [PATCH 27/30] ci(easyengine): run the behat lanes on every supported PHP version Test PHP 7.4 and 8.0 through 8.5 instead of only the lowest and highest supported versions. --- .github/workflows/behat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/behat.yml b/.github/workflows/behat.yml index f3e34b6ef..0855ae834 100644 --- a/.github/workflows/behat.yml +++ b/.github/workflows/behat.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - php: ['7.4', '8.5'] + php: ['7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5'] lane: [main, migration, uninstall] include: # Default profile plus @destructive; the first checked command initialises EasyEngine. From 5fa5fc90dba5a97a94c2c9aa77acb433ce380708 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 09:46:21 +0530 Subject: [PATCH 28/30] test(easyengine): match lists by membership in compareContents compareContents() compared list elements by position, so [1,3,5] wasn't found in [1,2,3,4,5], and a missing key or property raised a notice. The output steps already compare the whole expected text. --- features/bootstrap/support.php | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/features/bootstrap/support.php b/features/bootstrap/support.php index ed9a5b565..a5f1a1e67 100644 --- a/features/bootstrap/support.php +++ b/features/bootstrap/support.php @@ -81,13 +81,27 @@ function compareContents( $expected, $actual ) { if ( is_object( $expected ) ) { foreach ( get_object_vars( $expected ) as $name => $value ) { - if ( ! compareContents( $value, $actual->$name ) ) { + if ( ! property_exists( $actual, $name ) || ! compareContents( $value, $actual->$name ) ) { + return false; + } + } + } else if ( is_array( $expected ) && array_values( $expected ) === $expected ) { + // Lists match by membership, so [1,3,5] is contained in [1,2,3,4,5]. + foreach ( $expected as $value ) { + $found = false; + foreach ( $actual as $candidate ) { + if ( compareContents( $value, $candidate ) ) { + $found = true; + break; + } + } + if ( ! $found ) { return false; } } } else if ( is_array( $expected ) ) { foreach ( $expected as $key => $value ) { - if ( ! compareContents( $value, $actual[$key] ) ) { + if ( ! array_key_exists( $key, $actual ) || ! compareContents( $value, $actual[ $key ] ) ) { return false; } } From c9d26f18ec9e14f58aa0563854d6836e814cb3c7 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 09:59:45 +0530 Subject: [PATCH 29/30] test(easyengine): trust only a private scratch dir and force the self-uninstall prompt /var/tmp/ee-behat-core was used as found, so a pre-created directory or symlink could redirect root's writes and rm -rf. It is now created 0700 and refused unless it is a real directory owned by the running user and not writable by others. The declined self-uninstall scenario passes --no-yes, so a yes: true default in the config can't skip the prompt. --- features/COVERAGE.md | 1 + features/bootstrap/FeatureContext.php | 30 ++++++++++++++++++++++++--- features/cli.feature | 3 ++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/features/COVERAGE.md b/features/COVERAGE.md index a64ad1a00..4b3de251a 100644 --- a/features/COVERAGE.md +++ b/features/COVERAGE.md @@ -214,6 +214,7 @@ Safety rules built into the suite: - `Given no EasyEngine migration is pending` guards every scenario whose command reaches `maybe_trigger_migration()` with docker available (`cli info`, unknown top-level commands, phar `cli info`, the self-uninstall). It fails the scenario instead of running a migration when `options.version` differs from `VERSION`, when `VERSION` is a nightly, or when `/opt/easyengine/db/ee.sqlite` is missing (unless `EE_BEHAT_ALLOW_INIT=1`, which lets the first command initialise a fresh machine, as the old CI flow did). - Only the `@migration` scenario writes an older `options.version`; the others use `99.0.0`, so a regression, or an `ee` run by cron or another user meanwhile, gets the "not running latest version" error instead of a migration. Changed options are restored in `@AfterScenario` and, if the run dies, by a shutdown function (Behat exits on Ctrl-C) and a SIGTERM handler when pcntl is available. +- Scratch files live under `/var/tmp/ee-behat-core`, which is created 0700 and refused when it is a symlink, owned by another user or writable by group or others. - The phar is built from a `--no-dev` copy of the checkout under `/var/tmp/ee-behat-core/phar-/` and removed after the suite; the checkout, its `vendor/` autoloader and `/usr/local/bin/ee` are never touched, `cli update` is never run from the phar and `help` runs with `EE_DISABLE_AUTO_CHECK_UPDATE=1`. - Config scenarios point `EE_CONFIG_PATH` at a per-scenario file under `/var/tmp/ee-behat-core/` (`{TMP}`), removed after the scenario; the real `/opt/easyengine/config/config.yml` is never written (scenarios without `EE_CONFIG_PATH` read it, so `debug:`, aliases or `disabled_commands` there would break exact-output checks). - Requirement scenarios hide docker/docker-compose with a private `PATH` instead of stopping anything. diff --git a/features/bootstrap/FeatureContext.php b/features/bootstrap/FeatureContext.php index 2d1693da9..fc6237e83 100644 --- a/features/bootstrap/FeatureContext.php +++ b/features/bootstrap/FeatureContext.php @@ -162,7 +162,7 @@ public function eePharIsPresent() // Build from a private copy with a --no-dev autoloader: a phar built on a dev vendor requires // require-dev files it doesn't contain. The checkout and /usr/local/bin/ee are never touched. $root = realpath( EE_ROOT ); - $dir = self::TMP_BASE . '/phar-' . getmypid(); + $dir = self::tmp_base() . '/phar-' . getmypid(); $src = "$dir/src"; $phar = "$dir/ee.phar"; $cmd = sprintf( @@ -1062,8 +1062,8 @@ private function replace_variables( $string ) private function tmp_dir() { if ( null === $this->tmp_dir ) { - $this->tmp_dir = self::TMP_BASE . '/' . uniqid( 'scenario-', true ); - if ( ! mkdir( $this->tmp_dir, 0755, true ) ) { + $this->tmp_dir = self::tmp_base() . '/' . uniqid( 'scenario-', true ); + if ( ! mkdir( $this->tmp_dir, 0755 ) ) { throw new Exception( "Could not create {$this->tmp_dir}" ); } } @@ -1071,6 +1071,30 @@ private function tmp_dir() return $this->tmp_dir; } + /** + * TMP_BASE is in world-writable /var/tmp, so it is only used as a real directory owned by the running + * user and not writable by others; anything else could redirect root's writes and rm -rf through symlinks. + */ + private static function tmp_base() + { + $base = self::TMP_BASE; + if ( ! file_exists( $base ) && ! is_link( $base ) ) { + $umask = umask( 077 ); + @mkdir( $base, 0700 ); + umask( $umask ); + } + clearstatcache(); + $stat = @lstat( $base ); + if ( false === $stat || is_link( $base ) || ! is_dir( $base ) ) { + throw new Exception( "$base is not a real directory." ); + } + if ( $stat['uid'] !== posix_geteuid() || ( $stat['mode'] & 0022 ) ) { + throw new Exception( "$base must be owned by the running user and not writable by group or others." ); + } + + return $base; + } + private function last_command() { if ( empty( $this->commands ) ) { diff --git a/features/cli.feature b/features/cli.feature index 0f969531e..7edc26a8c 100644 --- a/features/cli.feature +++ b/features/cli.feature @@ -6,7 +6,8 @@ Feature: CLI Command Scenario: Declining the self-uninstall prompt removes nothing Given I record the number of containers with label "org.label-schema.vendor=EasyEngine" - When I run 'echo n | bin/ee cli self-uninstall' + # --no-yes overrides a `yes: true` default for self-uninstall in the config, which would skip the prompt. + When I run 'echo n | bin/ee cli self-uninstall --no-yes' Then the command should exit with code 0 And STDOUT should contain "Are you sure you want to remove EasyEngine and all its sites(along with their data)?" And the number of containers with label "org.label-schema.vendor=EasyEngine" should be unchanged From 3263f81d0d26cba20984df2d98e3e0121c339214 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Tue, 20 Oct 2026 11:23:04 +0530 Subject: [PATCH 30/30] ci(test): stop running the site-type-php suite in core's test job --- .github/workflows/test_and_build.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index b5005c12f..092b32367 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -150,19 +150,8 @@ jobs: run: | set -e cd $GITHUB_WORKSPACE + # site-command, site-type-php and site-type-wp run their own Behat suites (with Pebble lanes) on every PR. sudo -E ./vendor/bin/behat - # site-command and site-type-wp run their own Behat suites (with known-bug and Pebble lanes) on every PR. - sub_commands=( - vendor/easyengine/site-type-php/features - ) - for command in "${sub_commands[@]}"; do - IFS='/' read -r -a array <<< "$command" - rm -rf features/* - rsync -av --delete $command/ features/ > /dev/null - for file in features/*.feature; do mv "$file" "${file%.feature}_${array[2]}.feature"; done - echo "Running tests for $command" - sudo -E ./vendor/bin/behat - done env: COMPOSE_INTERACTIVE_NO_CLI: 1 # Core's suite refuses to initialise EasyEngine unless allowed; the runner starts without ee.sqlite.