Added database encoding conversion to upgrade script (resolving VUFIND-294).

module/VuFind/config/module.config.php

@@ -601,7 +601,8 @@ $staticRoutes = array(
     'Tag/Home',
     'Upgrade/Home', 'Upgrade/FixAnonymousTags', 'Upgrade/FixConfig',
     'Upgrade/FixDatabase', 'Upgrade/FixMetadata', 'Upgrade/GetDBCredentials',
-    'Upgrade/GetSourceDir', 'Upgrade/Reset', 'Upgrade/ShowSQL',
+    'Upgrade/GetDbEncodingPreference', 'Upgrade/GetSourceDir', 'Upgrade/Reset',
+    'Upgrade/ShowSQL',
     'VuDL/Browse', 'VuDL/DSRecord', 'VuDL/Record',
     'Worldcat/Advanced', 'Worldcat/Home', 'Worldcat/Search'
 );

module/VuFind/src/VuFind/Controller/Plugin/DbUpgrade.php

@@ -111,10 +111,11 @@ class DbUpgrade extends AbstractPlugin
     public function query($sql, $logsql)
     {
         if ($logsql) {
-            return $sql . ";\n";
+            return rtrim($sql, ';') . ";\n";
         } else {
             $this->getAdapter()->query($sql, DbAdapter::QUERY_MODE_EXECUTE);
         }
+        return '';
     }
 
     /**
@@ -148,6 +149,116 @@ class DbUpgrade extends AbstractPlugin
         return array_keys($this->getTableInfo());
     }
 
+    /**
+     * Support method for getEncodingProblems() -- get column details
+     *
+     * @param string $table Table to check
+     *
+     * @throws \Exception
+     * @return array
+     */
+    protected function getEncodingProblemsForTable($table)
+    {
+        // Get column summary:
+        $sql = "SHOW FULL COLUMNS FROM `{$table}`";
+        $results = $this->getAdapter()->query($sql, DbAdapter::QUERY_MODE_EXECUTE);
+
+        // Load details:
+        $retVal = array();
+        foreach ($results as $current) {
+            if (strtolower(substr($current->Collation, 0, 6)) == 'latin1') {
+                $retVal[$current->Field] = (array)$current;
+            }
+        }
+        return $retVal;
+    }
+
+    /**
+     * Get information on incorrectly encoded tables/columns.
+     *
+     * @throws \Exception
+     * @return array
+     */
+    public function getEncodingProblems()
+    {
+        // Get table summary:
+        $sql = "SHOW TABLE STATUS";
+        $results = $this->getAdapter()->query($sql, DbAdapter::QUERY_MODE_EXECUTE);
+
+        // Load details:
+        $retVal = array();
+        foreach ($results as $current) {
+            if (strtolower(substr($current->Collation, 0, 6)) == 'latin1') {
+                $retVal[$current->Name]
+                    = $this->getEncodingProblemsForTable($current->Name);
+            }
+        }
+
+        return $retVal;
+    }
+
+    /**
+     * Fix encoding problems based on the output of getEncodingProblems().
+     *
+     * @param array $tables Output of getEncodingProblems()
+     * @param bool  $logsql Should we return the SQL as a string rather than
+     * execute it?
+     *
+     * @throws \Exception
+     * @return string       SQL if $logsql is true, empty string otherwise
+     */
+    public function fixEncodingProblems($tables, $logsql = false)
+    {
+        $newCollation = "utf8_general_ci";
+        $sqlcommands = '';
+
+        // Database conversion routines inspired by:
+        //     https://github.com/nicjansma/mysql-convert-latin1-to-utf8
+        foreach ($tables as $table => $columns) {
+            foreach ($columns as $column => $details) {
+                $oldType = $details['Type'];
+                $parts = explode('(', $oldType);
+                switch ($parts[0]) {
+                case 'text':
+                    $newType = 'blob';
+                    break;
+                case 'varchar':
+                    $newType = 'varbinary(' . $parts[1];
+                    break;
+                default:
+                    throw new \Exception('Unexpected column type: ' . $parts[0]);
+                }
+                // Set up default:
+                if (null !== $details['Default']) {
+                    $safeDefault = mysql_real_escape_string($details['Default']);
+                    $currentDefault = " DEFAULT '{$safeDefault}'";
+                } else {
+                    $currentDefault = '';
+                }
+
+                // Change to binary equivalent:
+                $sql = "ALTER TABLE `$table` MODIFY `$column` $newType"
+                    . (strtoupper($details['Null']) == 'NO' ? ' NOT NULL' : '')
+                    . $currentDefault
+                    . ";";
+                $sqlcommands .= $this->query($sql, $logsql);
+
+                // Change back to appropriate character data with fixed encoding:
+                $sql = "ALTER TABLE `$table` MODIFY `$column` $oldType"
+                    . " COLLATE $newCollation"
+                    . (strtoupper($details['Null']) == 'NO' ? ' NOT NULL' : '')
+                    . $currentDefault
+                    . ";";
+                $sqlcommands .= $this->query($sql, $logsql);
+            }
+
+            // Adjust default table collation:
+            $sql = "ALTER TABLE `$table` DEFAULT COLLATE $newCollation;";
+            $sqlcommands .= $this->query($sql, $logsql);
+        }
+        return $sqlcommands;
+    }
+
     /**
      * Get information on all columns in a table, keyed by column name.
      *

module/VuFind/src/VuFind/Controller/UpgradeController.php

@@ -189,6 +189,24 @@ class UpgradeController extends AbstractBase
             && isset($this->session->dbRootPass);
     }
 
+    /**
+     * Configure the database encoding.
+     *
+     * @param string $charset Encoding setting to use.
+     *
+     * @throws \Exception
+     * @return void
+     */
+    protected function setDbEncodingConfiguration($charset)
+    {
+        $config = ConfigReader::getLocalConfigPath('config.ini', null, true);
+        $writer = new \VuFind\Config\Writer($config);
+        $writer->set('Database', 'charset', $charset);
+        if (!$writer->save()) {
+            throw new \Exception('Problem writing DB encoding to config.ini');
+        }
+    }
+
     /**
      * Upgrade the database.
      *
@@ -261,6 +279,34 @@ class UpgradeController extends AbstractBase
                     ->updateModifiedColumns($modifiedCols, $this->logsql);
             }
 
+            // Check for encoding problems.
+            $encProblems = $this->dbUpgrade()->getEncodingProblems();
+            if (!empty($encProblems)) {
+                if (!isset($this->session->dbChangeEncoding)) {
+                    return $this->forwardTo('Upgrade', 'GetDbEncodingPreference');
+                }
+
+                if ($this->session->dbChangeEncoding) {
+                    // Only manipulate DB if we're not in logging mode:
+                    if (!$this->logsql) {
+                        if (!$this->hasDatabaseRootCredentials()) {
+                            return $this->forwardTo('Upgrade', 'GetDbCredentials');
+                        }
+                        $this->dbUpgrade()->setAdapter($this->getRootDbAdapter());
+                        $this->session->warnings->append(
+                            "Modified encoding settings in table(s): "
+                            . implode(', ', array_keys($encProblems))
+                        );
+                    }
+                    $sql .= $this->dbUpgrade()
+                        ->fixEncodingProblems($encProblems, $this->logsql);
+                    $this->setDbEncodingConfiguration('utf8');
+                } else {
+                    // User has requested that we skip encoding conversion:
+                    $this->setDbEncodingConfiguration('latin1');
+                }
+            }
+
             // Don't keep DB credentials in session longer than necessary:
             unset($this->session->dbRootUser);
             unset($this->session->dbRootPass);
@@ -338,6 +384,24 @@ class UpgradeController extends AbstractBase
         return $this->createViewModel(array('dbrootuser' => $dbrootuser));
     }
 
+    /**
+     * Prompt the user for action on encoding problems.
+     *
+     * @return mixed
+     */
+    public function getdbencodingpreferenceAction()
+    {
+        $action = $this->params()->fromPost('encodingaction', '');
+        if ($action == 'Change') {
+            $this->session->dbChangeEncoding = true;
+            return $this->forwardTo('Upgrade', 'FixDatabase');
+        } else if ($action == 'Keep') {
+            $this->session->dbChangeEncoding = false;
+            return $this->forwardTo('Upgrade', 'FixDatabase');
+        }
+        return $this->createViewModel();
+    }
+
     /**
      * Prompt the user about fixing anonymous tags.
      *

themes/blueprint/templates/upgrade/getdbencodingpreference.phtml

+<?
+    // Set page title.
+    $this->headTitle($this->translate('Upgrade VuFind'));
+
+    // Set up breadcrumbs:
+    $this->layout()->breadcrumbs = '<em>' . $this->transEsc('Upgrade VuFind') . '</em>';
+?>
+<h1><?=$this->transEsc('Upgrade VuFind')?></h1>
+<?=$this->flashmessages()?>
+
+<p>Your current VuFind database is encoded in Latin-1 format.  This may cause incorrect sorting and
+display of records containing characters outside of the basic ASCII character set.</p>
+
+<p>It is <b>STRONGLY RECOMMENDED</b> that you convert your database to UTF-8.  However, this will
+prevent older versions of VuFind from reading the database correctly.</p>
+
+<p>If you need to maintain backward compatibility with 1.x, choose "Keep."  You can return to this
+upgrade tool later to perform UTF-8 conversion.</p>
+
+<p>If backward compatibility is not necessary, choose "Change" now.
+(You should make a backup first if you have not already!)</p>
+
+<form method="post" action="<?=$this->url('upgrade-getdbencodingpreference')?>">
+  <input type="submit" name="encodingaction" value="Change" /> encoding to UTF-8<br />
+  <input type="submit" name="encodingaction" value="Keep" /> Latin-1 encoding
+</form>
\ No newline at end of file