Take a screenshot on integrated test failure. (#1562)

- Includes some related fixes to AutoRetryTrait.

build.xml

@@ -24,6 +24,7 @@
   <property name="composer_version" value="1.10.8" />
   <property name="composer_extra_params" value="" />
   <property name="mink_driver" value="selenium" />
+  <property name="screenshot_dir" value="" /><!-- set to a directory name to capture screenshots of failed tests -->
   <property name="selenium_browser" value="firefox" />
   <property name="snooze_multiplier" value="1" /><!-- can be used to slow down tests (selenium only) -->
   <property name="solr_startup_sleep" value="0" />
@@ -176,17 +177,17 @@
 
   <!-- PHPUnit -->
   <target name="phpunit" description="Run tests">
-    <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --log-junit ${builddir}/reports/phpunit.xml --coverage-clover ${builddir}/reports/coverage/clover.xml --coverage-html ${builddir}/reports/coverage/ ${phpunit_extra_params}" passthru="true" checkreturn="true" />
+    <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SCREENSHOT_DIR=${screenshot_dir} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --log-junit ${builddir}/reports/phpunit.xml --coverage-clover ${builddir}/reports/coverage/clover.xml --coverage-html ${builddir}/reports/coverage/ ${phpunit_extra_params}" passthru="true" checkreturn="true" />
   </target>
 
   <!-- PHPUnit without logging output -->
   <target name="phpunitfast" description="Run tests">
-    <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 ${phpunit_extra_params}" passthru="true" checkreturn="true" />
+    <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SCREENSHOT_DIR=${screenshot_dir} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 ${phpunit_extra_params}" passthru="true" checkreturn="true" />
   </target>
 
   <!-- PHPUnit without logging output, stopping at first error or failure -->
   <target name="phpunitfaster" description="Run tests until first failure">
-    <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --stop-on-failure ${phpunit_extra_params}" passthru="true" checkreturn="true" />
+    <exec dir="${srcdir}/module/VuFind/tests" command="VUFIND_MINK_DRIVER=${mink_driver} VUFIND_SCREENSHOT_DIR=${screenshot_dir} VUFIND_SELENIUM_BROWSER=${selenium_browser} VUFIND_SNOOZE_MULTIPLIER=${snooze_multiplier} VUFIND_LOCAL_DIR=${srcdir}/local VUFIND_URL=${vufindurl} ${srcdir}/vendor/bin/phpunit -dzend.enable_gc=0 --stop-on-failure ${phpunit_extra_params}" passthru="true" checkreturn="true" />
   </target>
 
   <target name="installsolr" description="Install Solr">

module/VuFind/src/VuFindTest/Unit/AutoRetryTrait.php

@@ -52,6 +52,16 @@ trait AutoRetryTrait
      */
     protected static $failedAfterRetries = false;
 
+    /**
+     * Count of remaining retry attempts (updated during the retry loop). This is
+     * exposed as a class property rather than a local variable so that classes
+     * using the trait can be aware of the retry state. This is used, for example,
+     * in the VuFindTest\Unit\MinkTestCase class to control screenshot behavior.
+     *
+     * @var int
+     */
+    protected $retriesLeft;
+
     /**
      * Override PHPUnit's main run method, introducing annotation-based retry
      * behavior.
@@ -78,10 +88,9 @@ trait AutoRetryTrait
         $retryCallbacks = $annotations['method']['retryCallback'] ?? [];
         $retryCallbacks[] = 'tearDown';
 
-        // Run through all of the attempts... Note that even if retryCount is 0,
-        // we still need to run the test once (single attempt, no retries)...
-        // hence the $retryCount + 1 below.
-        for ($i = 0; $i < $retryCount + 1; $i++) {
+        // Run through all of the attempts...
+        $this->retriesLeft = $retryCount;
+        while ($this->retriesLeft >= 0) {
             try {
                 parent::runBare();
                 // No exception thrown? We can return as normal.
@@ -91,13 +100,17 @@ trait AutoRetryTrait
                 if (get_class($e) == SkippedTestError::class) {
                     throw $e;
                 }
-                // Execute callbacks for interrupted test.
-                foreach ($retryCallbacks as $callback) {
-                    if (is_callable([$this, $callback])) {
-                        $this->{$callback}();
+                // Execute callbacks for interrupted test, unless this is the
+                // last round of testing:
+                if ($this->retriesLeft > 0) {
+                    foreach ($retryCallbacks as $callback) {
+                        if (is_callable([$this, $callback])) {
+                            $this->{$callback}();
+                        }
                     }
                 }
             }
+            $this->retriesLeft--;
         }
 
         // If we got this far, something went wrong... under healthy circumstances,

module/VuFind/src/VuFindTest/Unit/MinkTestCase.php

@@ -381,6 +381,24 @@ abstract class MinkTestCase extends DbTestCase
      */
     public function tearDown(): void
     {
+        // Take screenshot of failed test, if we have a screenshot directory set
+        // and we have run out of retries ($this->retriesLeft is set by the
+        // AutoRetryTrait when it is use, and we'll default it to 0 to cover
+        // cases where that trait is not in play):
+        if ($this->hasFailed() && ($imageDir = getenv('VUFIND_SCREENSHOT_DIR'))
+            && ($this->retriesLeft ?? 0) === 0
+        ) {
+            $imageData = $this->getMinkSession()->getDriver()->getScreenshot();
+            if (!empty($imageData)) {
+                $filename = $this->getName() . '-' . hrtime(true) . '.png';
+
+                if (!file_exists($imageDir)) {
+                    mkdir($imageDir);
+                }
+                file_put_contents($imageDir . '/' . $filename, $imageData);
+            }
+        }
+
         $this->stopMinkSession();
         $this->restoreConfigs();
     }