From 32b5b48add78fe6c5f747db9532076f7e1439c5e Mon Sep 17 00:00:00 2001
From: Chris Hallberg <chris.hallberg@hey.com>
Date: Tue, 22 Sep 2020 16:03:13 -0400
Subject: [PATCH] Take a screenshot on integrated test failure. (#1562)
- Includes some related fixes to AutoRetryTrait.
---
build.xml | 7 +++--
.../src/VuFindTest/Unit/AutoRetryTrait.php | 29 ++++++++++++++-----
.../src/VuFindTest/Unit/MinkTestCase.php | 18 ++++++++++++
3 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/build.xml b/build.xml
index c9feb21392e..bde0ae62e08 100644
--- a/build.xml
+++ b/build.xml
@@ -24,6 +24,7 @@
<property name="composer_version" value="1.10.8" />
<property name="composer_extra_params" value="" />
<property name="mink_driver" value="selenium" />
+ <property name="screenshot_dir" value="" /><!-- set to a directory name to capture screenshots of failed tests -->
<property name="selenium_browser" value="firefox" />
<property name="snooze_multiplier" value="1" /><!-- can be used to slow down tests (selenium only) -->
<property name="solr_startup_sleep" value="0" />
@@ -176,17 +177,17 @@
<!-- PHPUnit -->
<target name="phpunit" description="Run tests">
- <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --log-junit ${builddir}/reports/phpunit.xml --coverage-clover ${builddir}/reports/coverage/clover.xml --coverage-html ${builddir}/reports/coverage/ ${phpunit_extra_params}" passthru="true" checkreturn="true" />
+ <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SCREENSHOT_DIR=${screenshot_dir} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --log-junit ${builddir}/reports/phpunit.xml --coverage-clover ${builddir}/reports/coverage/clover.xml --coverage-html ${builddir}/reports/coverage/ ${phpunit_extra_params}" passthru="true" checkreturn="true" />
</target>
<!-- PHPUnit without logging output -->
<target name="phpunitfast" description="Run tests">
- <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 ${phpunit_extra_params}" passthru="true" checkreturn="true" />
+ <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SCREENSHOT_DIR=${screenshot_dir} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 ${phpunit_extra_params}" passthru="true" checkreturn="true" />
</target>
<!-- PHPUnit without logging output, stopping at first error or failure -->
<target name="phpunitfaster" description="Run tests until first failure">
- <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --stop-on-failure ${phpunit_extra_params}" passthru="true" checkreturn="true" />
+ <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SCREENSHOT_DIR=${screenshot_dir} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --stop-on-failure ${phpunit_extra_params}" passthru="true" checkreturn="true" />
</target>
<target name="installsolr" description="Install Solr">
diff --git a/module/VuFind/src/VuFindTest/Unit/AutoRetryTrait.php b/module/VuFind/src/VuFindTest/Unit/AutoRetryTrait.php
index 42a3dbbb7bc..fcf31dcc094 100644
--- a/module/VuFind/src/VuFindTest/Unit/AutoRetryTrait.php
+++ b/module/VuFind/src/VuFindTest/Unit/AutoRetryTrait.php
@@ -52,6 +52,16 @@ trait AutoRetryTrait
*/
protected static $failedAfterRetries = false;
+ /**
+ * Count of remaining retry attempts (updated during the retry loop). This is
+ * exposed as a class property rather than a local variable so that classes
+ * using the trait can be aware of the retry state. This is used, for example,
+ * in the VuFindTest\Unit\MinkTestCase class to control screenshot behavior.
+ *
+ * @var int
+ */
+ protected $retriesLeft;
+
/**
* Override PHPUnit's main run method, introducing annotation-based retry
* behavior.
@@ -78,10 +88,9 @@ trait AutoRetryTrait
$retryCallbacks = $annotations['method']['retryCallback'] ?? [];
$retryCallbacks[] = 'tearDown';
- // Run through all of the attempts... Note that even if retryCount is 0,
- // we still need to run the test once (single attempt, no retries)...
- // hence the $retryCount + 1 below.
- for ($i = 0; $i < $retryCount + 1; $i++) {
+ // Run through all of the attempts...
+ $this->retriesLeft = $retryCount;
+ while ($this->retriesLeft >= 0) {
try {
parent::runBare();
// No exception thrown? We can return as normal.
@@ -91,13 +100,17 @@ trait AutoRetryTrait
if (get_class($e) == SkippedTestError::class) {
throw $e;
}
- // Execute callbacks for interrupted test.
- foreach ($retryCallbacks as $callback) {
- if (is_callable([$this, $callback])) {
- $this->{$callback}();
+ // Execute callbacks for interrupted test, unless this is the
+ // last round of testing:
+ if ($this->retriesLeft > 0) {
+ foreach ($retryCallbacks as $callback) {
+ if (is_callable([$this, $callback])) {
+ $this->{$callback}();
+ }
}
}
}
+ $this->retriesLeft--;
}
// If we got this far, something went wrong... under healthy circumstances,
diff --git a/module/VuFind/src/VuFindTest/Unit/MinkTestCase.php b/module/VuFind/src/VuFindTest/Unit/MinkTestCase.php
index df3025e5379..9dabd5fc6b2 100644
--- a/module/VuFind/src/VuFindTest/Unit/MinkTestCase.php
+++ b/module/VuFind/src/VuFindTest/Unit/MinkTestCase.php
@@ -381,6 +381,24 @@ abstract class MinkTestCase extends DbTestCase
*/
public function tearDown(): void
{
+ // Take screenshot of failed test, if we have a screenshot directory set
+ // and we have run out of retries ($this->retriesLeft is set by the
+ // AutoRetryTrait when it is use, and we'll default it to 0 to cover
+ // cases where that trait is not in play):
+ if ($this->hasFailed() && ($imageDir = getenv('VUFIND_SCREENSHOT_DIR'))
+ && ($this->retriesLeft ?? 0) === 0
+ ) {
+ $imageData = $this->getMinkSession()->getDriver()->getScreenshot();
+ if (!empty($imageData)) {
+ $filename = $this->getName() . '-' . hrtime(true) . '.png';
+
+ if (!file_exists($imageDir)) {
+ mkdir($imageDir);
+ }
+ file_put_contents($imageDir . '/' . $filename, $imageData);
+ }
+ }
+
$this->stopMinkSession();
$this->restoreConfigs();
}
--
GitLab