Skip to content
Snippets Groups Projects
Commit d7e6b4d3 authored by Ere Maijala's avatar Ere Maijala Committed by Demian Katz
Browse files

Refactored search/session expiration to delete records in batches (#739) 

- Avoid locking the tables for too long or creating a massive transaction in case the table has accumulated records for a longer time.

* Reverted deletion of getExpiredQuery() and added command line parameters for specifying batch size and sleep time.

* Don't sleep after last batch.
parent c154a7be
No related merge requests found
Hide whitespace changes
Inline Side-by-side
module/VuFind/src/VuFind/Db/Table/Search.php
+ 59
− 0
View file @ d7e6b4d3
......@@ -31,6 +31,7 @@ namespace VuFind\Db\Table;
use minSO;
use Zend\Db\Adapter\ParameterContainer;
use Zend\Db\TableGateway\Feature;
use Zend\Db\Sql\Expression;
/**
* Table Definition for search
......@@ -131,6 +132,64 @@ class Search extends Gateway
return $this->select($callback);
}
/**
* Delete expired searches. Allows setting of 'from' and 'to' ID's so that rows
* can be deleted in small batches.
*
* @param int $daysOld Age in days of an "expired" search.
* @param int $idFrom Lowest id of rows to delete.
* @param int $idTo Highest id of rows to delete.
*
* @return int Number of rows deleted
*/
public function deleteExpired($daysOld = 2, $idFrom = null, $idTo = null)
{
// Determine the expiration date:
$expireDate = date('Y-m-d H:i:s', time() - $daysOld * 24 * 60 * 60);
$callback = function ($select) use ($expireDate, $idFrom, $idTo) {
$where = $select->where->lessThan('created', $expireDate)
->equalTo('saved', 0);
if (null !== $idFrom) {
$where->and->greaterThanOrEqualTo('id', $idFrom);
}
if (null !== $idTo) {
$where->and->lessThanOrEqualTo('id', $idTo);
}
};
return $this->delete($callback);
}
/**
* Get the lowest id and highest id for expired searches.
*
* @param int $daysOld Age in days of an "expired" search.
*
* @return array|bool Array of lowest id and highest id or false if no expired
* records found
*/
public function getExpiredIdRange($daysOld = 2)
{
// Determine the expiration date:
$expireDate = date('Y-m-d H:i:s', time() - $daysOld * 24 * 60 * 60);
$callback = function ($select) use ($expireDate) {
$select->where->lessThan('created', $expireDate)->equalTo('saved', 0);
};
$select = $this->getSql()->select();
$select->columns(
[
'id' => new Expression('1'), // required for TableGateway
'minId' => new Expression('MIN(id)'),
'maxId' => new Expression('MAX(id)'),
]
);
$select->where($callback);
$result = $this->selectWith($select)->current();
if (null === $result->minId) {
return false;
}
return [$result->minId, $result->maxId];
}
/**
* Get a query representing expired searches (this can be passed
* to select() or delete() for further processing).
......
module/VuFind/src/VuFind/Db/Table/Session.php
+ 59
− 0
View file @ d7e6b4d3
......@@ -5,6 +5,7 @@
* PHP version 5
*
* Copyright (C) Villanova University 2010.
* Copyright (C) The National Library of Finland 2016.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2,
......@@ -22,11 +23,13 @@
* @category VuFind
* @package Db_Table
* @author Demian Katz <demian.katz@villanova.edu>
* @author Ere Maijala <ere.maijala@helsinki.fi>
* @license http://opensource.org/licenses/gpl-2.0.php GNU General Public License
* @link https://vufind.org Main Page
*/
namespace VuFind\Db\Table;
use VuFind\Exception\SessionExpired as SessionExpiredException;
use Zend\Db\Sql\Expression;
/**
* Table Definition for session
......@@ -34,6 +37,7 @@ use VuFind\Exception\SessionExpired as SessionExpiredException;
* @category VuFind
* @package Db_Table
* @author Demian Katz <demian.katz@villanova.edu>
* @author Ere Maijala <ere.maijala@helsinki.fi>
* @license http://opensource.org/licenses/gpl-2.0.php GNU General Public License
* @link https://vufind.org Main Site
*/
......@@ -139,6 +143,61 @@ class Session extends Gateway
$this->delete($callback);
}
/**
* Delete expired sessions. Allows setting of 'from' and 'to' ID's so that rows
* can be deleted in small batches.
*
* @param int $daysOld Age in days of an "expired" session.
* @param int $idFrom Lowest id of rows to delete.
* @param int $idTo Highest id of rows to delete.
*
* @return int Number of rows deleted
*/
public function deleteExpired($daysOld = 2, $idFrom = null, $idTo = null)
{
$expireDate = time() - $daysOld * 24 * 60 * 60;
$callback = function ($select) use ($expireDate, $idFrom, $idTo) {
$where = $select->where->lessThan('last_used', $expireDate);
if (null !== $idFrom) {
$where->and->greaterThanOrEqualTo('id', $idFrom);
}
if (null !== $idTo) {
$where->and->lessThanOrEqualTo('id', $idTo);
}
};
return $this->delete($callback);
}
/**
* Get the lowest id and highest id for expired sessions.
*
* @param int $daysOld Age in days of an "expired" session.
*
* @return array|bool Array of lowest id and highest id or false if no expired
* records found
*/
public function getExpiredIdRange($daysOld = 2)
{
$expireDate = time() - $daysOld * 24 * 60 * 60;
$callback = function ($select) use ($expireDate) {
$select->where->lessThan('last_used', $expireDate);
};
$select = $this->getSql()->select();
$select->columns(
[
'id' => new Expression('1'), // required for TableGateway
'minId' => new Expression('MIN(id)'),
'maxId' => new Expression('MAX(id)'),
]
);
$select->where($callback);
$result = $this->selectWith($select)->current();
if (null === $result->minId) {
return false;
}
return [$result->minId, $result->maxId];
}
/**
* Get a query representing expired sessions (this can be passed
* to select() or delete() for further processing).
......
module/VuFindConsole/src/VuFindConsole/Controller/UtilController.php
+ 87
− 37
View file @ d7e6b4d3
......@@ -453,6 +453,39 @@ class UtilController extends AbstractBase
return $this->getSuccessResponse();
}
/**
* Display help for the search or session expiration actions
*
* @param string $rows Plural name of records to delete
*
* @return \Zend\Console\Response
*/
protected function expirationHelp($rows)
{
Console::writeLine("Expire old $rows in the database.");
Console::writeLine('');
Console::writeLine(
'Optional parameters: [--batch=size] [--sleep=time] [age]'
);
Console::writeLine('');
Console::writeLine(
' batch: number of records to delete in a single batch'
. ' (default 1000)'
);
Console::writeLine(
' sleep: milliseconds to sleep between batches (default 100)'
);
Console::writeLine(
" age: the age (in days) of $rows to expire (default 2)"
);
Console::writeLine('');
Console::writeLine(
"By default, $rows more than 2 days old will be removed."
);
return $this->getFailureResponse();
}
/**
* Command-line tool to clear unwanted entries
* from search history database table.
......@@ -464,21 +497,13 @@ class UtilController extends AbstractBase
$this->consoleOpts->addRules(
[
'h|help' => 'Get help',
'batch=i' => 'Batch size',
'sleep=i' => 'Sleep interval between batches'
]
);
if ($this->consoleOpts->getOption('h')
|| $this->consoleOpts->getOption('help')
) {
Console::writeLine('Expire old searches in the database.');
Console::writeLine('');
Console::writeLine(
'Optional parameter: the age (in days) of searches to expire;'
);
Console::writeLine(
'by default, searches more than 2 days old will be removed.'
);
return $this->getFailureResponse();
if ($this->consoleOpts->getOption('h')) {
return $this->expirationHelp('searches');
}
return $this->expire(
......@@ -499,21 +524,13 @@ class UtilController extends AbstractBase
$this->consoleOpts->addRules(
[
'h|help' => 'Get help',
'batch=i' => 'Batch size',
'sleep=i' => 'Sleep interval between batches'
]
);
if ($this->consoleOpts->getOption('h')
|| $this->consoleOpts->getOption('help')
) {
Console::writeLine('Expire old sessions in the database.');
Console::writeLine('');
Console::writeLine(
'Optional parameter: the age (in days) of sessions to expire;'
);
Console::writeLine(
'by default, sessions more than 2 days old will be removed.'
);
return $this->getFailureResponse();
if ($this->consoleOpts->getOption('h')) {
return $this->expirationHelp('sessions');
}
return $this->expire(
......@@ -651,7 +668,7 @@ class UtilController extends AbstractBase
/**
* Abstract delete method.
*
* @param string $table Table to operate on.
* @param string $tableName Table to operate on.
* @param string $successString String for reporting success.
* @param string $failString String for reporting failure.
* @param int $minAge Minimum age allowed for expiration (also used
......@@ -659,7 +676,7 @@ class UtilController extends AbstractBase
*
* @return mixed
*/
protected function expire($table, $successString, $failString, $minAge = 2)
protected function expire($tableName, $successString, $failString, $minAge = 2)
{
// Get command-line arguments
$argv = $this->consoleOpts->getRemainingArgs();
......@@ -667,6 +684,11 @@ class UtilController extends AbstractBase
// Use command line value as expiration age, or default to $minAge.
$daysOld = isset($argv[0]) ? intval($argv[0]) : $minAge;
// Use command line values for batch size and sleep time if specified.
$options = $this->consoleOpts->getArguments();
$batchSize = isset($options['batch']) ? $options['batch'] : 1000;
$sleepTime = isset($options['sleep']) ? $options['sleep'] : 100;
// Abort if we have an invalid expiration age.
if ($daysOld < 2) {
Console::writeLine(
......@@ -678,23 +700,51 @@ class UtilController extends AbstractBase
return $this->getFailureResponse();
}
// Delete the expired searches--this cleans up any junk left in the database
// from old search histories that were not
// caught by the session garbage collector.
$search = $this->getTable($table);
if (!method_exists($search, 'getExpiredQuery')) {
throw new \Exception($table . ' does not support getExpiredQuery()');
// Delete the expired rows--this cleans up any junk left in the database
// e.g. from old searches or sessions that were not caught by the session
// garbage collector.
$table = $this->getTable($tableName);
if (!method_exists($table, 'getExpiredIdRange')) {
throw new \Exception("$tableName does not support getExpiredIdRange()");
}
$query = $search->getExpiredQuery($daysOld);
if (($count = count($search->select($query))) == 0) {
Console::writeLine($failString);
if (!method_exists($table, 'deleteExpired')) {
throw new \Exception("$tableName does not support deleteExpired()");
}
$idRange = $table->getExpiredIdRange($daysOld);
if (false === $idRange) {
$this->timestampedMessage($failString);
return $this->getSuccessResponse();
}
$search->delete($query);
Console::writeLine(str_replace('%%count%%', $count, $successString));
// Delete records in batches
for ($batch = $idRange[0]; $batch <= $idRange[1]; $batch += $batchSize) {
$count = $table->deleteExpired(
$daysOld, $batch, $batch + $batchSize - 1
);
$this->timestampedMessage(
str_replace('%%count%%', $count, $successString)
);
// Be nice to others and wait between batches
if ($batch + $batchSize <= $idRange[1]) {
usleep($sleepTime * 1000);
}
}
return $this->getSuccessResponse();
}
/**
* Print a message with a time stamp to the console
*
* @param string $msg Message
*
* @return void
*/
protected function timestampedMessage($msg)
{
Console::writeLine('[' . date('Y-m-d H:i:s') . '] ' . $msg);
}
/**
* Convert hash algorithms
* Expected parameters: oldmethod:oldkey (or none) newmethod:newkey
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment