Commit 1d7a5865 authored by Niels Erik G. Nielsen's avatar Niels Erik G. Nielsen
Browse files
parents 59f8e264 413baff5
......@@ -172,6 +172,11 @@
mysql_db: name=localindices state=import target=/vagrant/sql/v2.13/2020-04-15.sql
notify: Restart Tomcat
- name: Load 2.14 table alteration
become: yes
mysql_db: name=localindices state=import target=/vagrant/sql/v2.14/2020-04-14.sql
notify: Restart Tomcat
- name: Load gbv table data
become: yes
mysql_db: name=localindices state=import target=/vagrant/sql/load-gbv-data-to-v2.13.sql
......
......@@ -12,6 +12,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
......@@ -826,4 +827,20 @@ public class JobController {
return list;
}
Map<String,String> modesMap = new HashMap<String,String>();
public List<SelectItem> getFailedRecordsLoggingItems() {
modesMap.put("NO_STORE", "Don't save failed records");
modesMap.put("CLEAN_DIRECTORY", "Do save. Clean up directory first.");
modesMap.put("CREATE_OVERWRITE", "Do save. Overwrite existing files.");
modesMap.put("ADD_ALL", "Do save. Add numbered versions for existing files.");
List<SelectItem> list = new LinkedList<SelectItem>();
for (String mode : Arrays.asList("NO_STORE","CLEAN_DIRECTORY", "CREATE_OVERWRITE", "ADD_ALL")) {
SelectItem selectItem = new SelectItem();
selectItem.setLabel(modesMap.get(mode));
selectItem.setValue(mode);
list.add(selectItem);
}
return list;
}
}
......@@ -104,6 +104,20 @@ Harvest_Jobs.General_information.Connection/read_timeout_(seconds) = Specify a n
Harvest_Jobs.General_information.Log_level = Specify the logging level for the job with DEBUG being the most verbose. \
INFO is the recommended log level in most cases.
Harvest_Jobs.General_information.Saving_failed_records = Specify whether or not failed records should be saved as XML files in a \
designated log directory. Also specify retention policy for the directory, that is, whether \
to retain files that were saved in previous runs and, if so, whether to overwrite an existing \
file if the same record fails again or rather add a sequence number to the new file name in order \
not to overwrite.
Harvest_Jobs.General_information.Maximum_number_of_failed_records_saved_next_run = Sets a maximum number of files to save in \
the failed records directory per run. The job log will tell when the limit is reached.
Harvest_Jobs.General_information.Maximum_number_of_failed_records_saved_total = Sets a maximum number of files to be saved \
in the failed records directory at any given time - as the sum of previously saved \
records (that were not cleaned up before this run) plus any new records added during the run.\
The job log will tell when the limit is reached.
Harvest_Jobs.General_information.Notification_e-mail_address(es)_(separate_with_comma) = Specify comma separated list of e-mail addresses that \
should receive notification on job completion.
......
......@@ -181,6 +181,24 @@
</h:selectOneMenu>
<id:helplink field="[Harvest Jobs][General information][Log level]"/>
</h:panelGroup>
<h:outputText value="Saving failed records:" />
<h:panelGroup>
<h:selectOneMenu value="#{resourceController.resource.failedRecordsLogging}">
<f:selectItems
value="#{resourceController.failedRecordsLoggingItems}" />
</h:selectOneMenu>
<id:helplink field="[Harvest Jobs][General information][Saving failed records]"/>
</h:panelGroup>
<h:outputText value="Maximum number of failed records saved next run:" />
<h:panelGroup>
<h:inputText value="#{resourceController.resource.maxSavedFailedRecordsPerRun}" size="20" />
<id:helplink area="Harvest Jobs" section="General information" label="Maximum number of failed records saved next run"/>
</h:panelGroup>
<h:outputText value="Maximum number of failed records saved total:" />
<h:panelGroup>
<h:inputText value="#{resourceController.resource.maxSavedFailedRecordsTotal}" size="20" />
<id:helplink area="Harvest Jobs" section="General information" label="Maximum number of failed records saved total"/>
</h:panelGroup>
<h:outputText value="Notification e-mail address(es) (separate with comma): " />
<h:panelGroup>
<h:inputText value="#{resourceController.resource.mailAddress}" size="80" />
......
......@@ -117,9 +117,7 @@ public class OAIRecordHarvestJob extends AbstractRecordHarvestJob {
setStatus(HarvestStatus.valueOf(resource.getCurrentStatus()));
if (getStatus().equals(HarvestStatus.NEW) || getStatus().equals(HarvestStatus.ERROR))
this.initialRun = true;
if (resource.isStoreOriginal()) {
originalBuff = new ByteArrayOutputStream(ORIGINAL_BUFF_SIZE);
}
originalBuff = new ByteArrayOutputStream(ORIGINAL_BUFF_SIZE);
// this.resource.setMessage(null);
}
......@@ -444,12 +442,10 @@ public class OAIRecordHarvestJob extends AbstractRecordHarvestJob {
String id = HarvesterVerb.getSingleString(node, "./oai20:header/oai20:identifier/text()");
String isDeleted = HarvesterVerb.getSingleString(node, "./oai20:header/@status");
byte[] original = null;
if (resource.isStoreOriginal()) {
originalBuff.reset();
//TODO find oai subrecord
XmlUtils.serialize(node, originalBuff);
original = originalBuff.toByteArray();
}
originalBuff.reset();
//TODO find oai subrecord
XmlUtils.serialize(node, originalBuff);
original = originalBuff.toByteArray();
RecordDOMImpl record = new RecordDOMImpl(id, resource.getId().toString(), node, original);
if ("deleted".equalsIgnoreCase(isDeleted)) {
logger.log(Level.DEBUG, "OAI delete record found");
......
......@@ -28,9 +28,7 @@ public class RecordStorageConsumer implements MessageConsumer {
recordStorage = storage;
this.logger = logger;
this.storeOriginal = storeOriginal;
if (storeOriginal) {
originalBuff = new ByteArrayOutputStream(ORIGINAL_BUFF_SIZE);
}
originalBuff = new ByteArrayOutputStream(ORIGINAL_BUFF_SIZE);
}
@Override
......@@ -38,15 +36,13 @@ public class RecordStorageConsumer implements MessageConsumer {
long creationStart = System.currentTimeMillis();
logger.log(Level.TRACE, "Document in pipeline for storage: " + nodeAsString(xmlNode));
byte[] original = null;
if (this.storeOriginal) {
originalBuff.reset();
try {
XmlUtils.serialize(xmlNode, originalBuff);
} catch (TransformerException ex) {
logger.error("Failed to serialize original contents for storage, record num "+added);
}
original = originalBuff.toByteArray();
originalBuff.reset();
try {
XmlUtils.serialize(xmlNode, originalBuff);
} catch (TransformerException ex) {
logger.error("Failed to serialize original contents for storage, record num "+added);
}
original = originalBuff.toByteArray();
Record record = new RecordDOMImpl(null, null, xmlNode, original);
record.setCreationTiming(System.currentTimeMillis()-creationStart);
try {
......
......@@ -10,17 +10,17 @@ public class ExceptionRecordError implements RecordError {
public String stackTrace;
public String context;
public String typeOfEntity;
public String transaction;
public String entity;
public ExceptionRecordError(Exception e, String context, String typeOfEntity) {
public ExceptionRecordError(Exception e, String context, String typeOfEntity, String transaction, String entity) {
this.exceptionType = e.getClass().getSimpleName();
this.message = e.getLocalizedMessage();
this.stackTrace = stackTraceAsString(e);
this.context = context==null ? "" : context;
this.typeOfEntity = typeOfEntity;
}
public ExceptionRecordError(Exception e, String context) {
this(e, context, "unspecified");
this.transaction = transaction;
this.entity = entity;
}
private String stackTraceAsString (Exception e) {
......@@ -36,28 +36,43 @@ public class ExceptionRecordError implements RecordError {
}
@Override
public String getMessage() {
public String getMessageWithContext() {
return context + "; [" + exceptionType + "]; " + message;
}
public String getErrorContext() {
@Override
public String getAdditionalContext() {
return context;
}
public String getType() {
@Override
public String getErrorType() {
return exceptionType;
}
public String getBriefMessage() {
@Override
public String getServerMessage() {
return message;
}
public String getCountingKey () {
@Override
public String getShortMessageForCounting() {
return message;
}
public String getStorageEntity() {
@Override
public String getRecordType() {
return typeOfEntity;
}
@Override
public String getTransaction() {
return transaction;
}
@Override
public String getEntity() {
return entity;
}
}
\ No newline at end of file
......@@ -5,7 +5,11 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import com.indexdata.masterkey.localindices.entity.Harvestable;
import com.indexdata.masterkey.localindices.harvest.job.StorageJobLogger;
import org.apache.commons.io.FileUtils;
......@@ -17,7 +21,7 @@ import org.apache.commons.io.FileUtils;
*
*/
public class FailedRecordsController {
// Configuration - requires implementation in Harvestable config
// Configuration
protected int maxFailedRecordFilesThisRun = 100;
protected int maxFailedRecordFilesTotal = 450;
protected enum StoreMode {NO_STORE, CLEAN_DIRECTORY, CREATE_OVERWRITE, ADD_ALL};
......@@ -33,10 +37,16 @@ public class FailedRecordsController {
int initialNumberOfFiles = 0;
int calculatedNumberOfFiles = 0;
public FailedRecordsController(StorageJobLogger logger, Long jobId) {
this.jobId = jobId;
public FailedRecordsController(StorageJobLogger logger, Harvestable config) {
this.jobId = config.getId();
this.recordFailureCounters = new RecordFailureCounters();
this.logger = logger;
String retention = config.getFailedRecordsLogging();
this.mode = (retention == null ? StoreMode.CLEAN_DIRECTORY : StoreMode.valueOf(retention));
this.maxFailedRecordFilesThisRun = config.getMaxSavedFailedRecordsPerRun();
this.maxFailedRecordFilesTotal = config.getMaxSavedFailedRecordsTotal();
prepareFailedRecordsDirectory(logger, jobId);
}
public String getMode() {
......@@ -51,7 +61,6 @@ public class FailedRecordsController {
* @param logger
* @param jobId
*/
@SuppressWarnings("unused") // Requires implementation in Harvestable config
private void prepareFailedRecordsDirectory(StorageJobLogger logger, Long jobId) {
Path failedRecordsDirectory = Paths.get(HARVESTER_LOG_DIR, FAILED_RECORDS_DIR, jobId.toString());
try {
......@@ -68,8 +77,7 @@ public class FailedRecordsController {
}
initialNumberOfFiles = countFiles(failedRecordsDirectory);
calculatedNumberOfFiles = initialNumberOfFiles;
logger.info("Initialized failed-records controller (mode " + getMode() + ")");
logger.info("There are " + initialNumberOfFiles + " file(s) in the job's failed-records directory at the beginning of this run");
logger.info("Initialized failed-records controller (mode " + getMode() + "). There are " + initialNumberOfFiles + " saved file(s) in the failed-records directory.");
} catch (IOException e) {
directoryReady = false;
this.logger.error("Could not initialize directory for storing failed records. Will not save any failed records");
......@@ -108,8 +116,8 @@ public class FailedRecordsController {
* Writes statistics to job log, counting errors by type
*/
public void writeLog() {
for (String key : recordFailureCounters.errorsByErrorMessage.keySet()) {
logger.info(String.format("%d records failed with %s", recordFailureCounters.errorsByErrorMessage.get(key),key));
for (String key : recordFailureCounters.errorsByShortErrorMessage.keySet()) {
logger.info(String.format("%d records failed with %s", recordFailureCounters.errorsByShortErrorMessage.get(key),key));
}
}
......@@ -184,10 +192,22 @@ public class FailedRecordsController {
}
private String getFileName (RecordWithErrors failedRecord) {
String filename = ((String) failedRecord.getRecordIdentifier()) + ".xml";
String filename;
String recid = failedRecord.getRecordIdentifier();
if (recid==null) {
filename = "timestamp-" + timestamp() + ".xml";
} else {
filename = recid + ".xml";
}
return filename;
}
private String timestamp () {
LocalDateTime now = LocalDateTime.now();
String timestamp = now.format(DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS", Locale.getDefault()));
return timestamp;
}
private String getFileName (RecordWithErrors failedRecord, int version) {
return String.format("%s-%d.xml", failedRecord.getRecordIdentifier(), version);
}
......@@ -196,8 +216,8 @@ public class FailedRecordsController {
return this.recordFailureCounters;
}
public int getErrorsByErrorKey(String errorKey) {
return getCounters().errorsByErrorMessage.get(errorKey);
public int getErrorsByShortErrorMessage(String errorKey) {
return getCounters().errorsByShortErrorMessage.get(errorKey);
}
public int incrementErrorCount (RecordError error) {
......
......@@ -5,56 +5,71 @@ import org.apache.http.StatusLine;
public class HttpRecordError implements RecordError {
public int statusCode;
public String reason;
public String response;
public String context;
public String reasonPhrase;
public String serverMessage;
public String additionalContext;
public String recordType;
public String countingMessage;
public String transaction;
public String entity;
public String countingKey;
public HttpRecordError(int status, String reason, String response, String countingKey, String context, String entity) {
this.statusCode = status;
this.reason = reason;
this.response = response;
this.context = context;
public HttpRecordError(int statusCode, String reasonPhrase, String serverMessage, String countingMessage, String additionalContext, String recordType, String transaction, String entity) {
this.statusCode = statusCode;
this.reasonPhrase = reasonPhrase;
this.serverMessage = serverMessage;
this.additionalContext = additionalContext;
this.recordType = recordType;
this.countingMessage = countingMessage;
this.transaction = transaction;
this.entity = entity;
this.countingKey = countingKey;
}
public HttpRecordError(int status, String reason, String response, String countingKey, String context) {
this(status, reason, response, context, countingKey, "unspecified");
public HttpRecordError(StatusLine httpStatus, String serverMessage, String countingMessage, String additionalContext, String recordType, String transaction, String entity) {
this(httpStatus.getStatusCode(), httpStatus.getReasonPhrase(), serverMessage, countingMessage, additionalContext, recordType, transaction, entity);
}
public HttpRecordError(StatusLine httpStatus, String response, String countingKey, String context, String entity) {
this(httpStatus.getStatusCode(), httpStatus.getReasonPhrase(), response, countingKey, context, entity);
@Override
public String toString() {
return recordType + ": " + additionalContext + ". Status code ["+statusCode+"]." + reasonPhrase + "]." + serverMessage;
}
@Override
public String toString() {
return entity + ": " + context + ". Status code ["+statusCode+"]." + reason + "]." + response;
public String getMessageWithContext() {
return additionalContext + "; " + reasonPhrase + "; " + serverMessage;
}
@Override
public String getMessage() {
return context + "; " + reason + "; " + response;
public String getAdditionalContext() {
return additionalContext;
}
public String getErrorContext() {
return context;
@Override
public String getErrorType() {
return reasonPhrase;
}
public String getType() {
return reason;
@Override
public String getServerMessage() {
return serverMessage;
}
public String getBriefMessage() {
return response;
@Override
public String getShortMessageForCounting() {
return countingMessage;
}
public String getCountingKey() {
return countingKey;
@Override
public String getTransaction() {
return transaction;
}
public String getStorageEntity() {
return entity;
@Override
public String getEntity() {
return entity;
}
@Override
public String getRecordType() {
return recordType;
}
}
......@@ -88,7 +88,7 @@ public class InventoryUpdateContext {
timingsCreatingRecord.setLogLevelForIntervals(Level.DEBUG);
timingsTransformingRecord = new HourlyPerformanceStats("Transforming incoming record before storing", logger);
timingsTransformingRecord.setLogLevelForIntervals(Level.DEBUG);
failedRecordsController = new FailedRecordsController(logger, harvestable.getId());
failedRecordsController = new FailedRecordsController(logger, harvestable);
}
public void setClient (CloseableHttpClient inventoryClient) {
......
package com.indexdata.masterkey.localindices.harvest.storage.folioinventory;
public interface RecordError {
public String getMessage ();
public String getErrorContext();
public String getType();
public String getBriefMessage();
public String getStorageEntity();
public String getCountingKey();
public String getMessageWithContext();
public String getAdditionalContext();
public String getErrorType();
public String getServerMessage();
public String getRecordType();
public String getShortMessageForCounting();
public String getTransaction();
public String getEntity();
}
\ No newline at end of file
......@@ -4,14 +4,14 @@ import java.util.HashMap;
import java.util.Map;
public class RecordFailureCounters {
protected final Map<String,Integer> errorsByErrorMessage = new HashMap<String,Integer>();
protected final Map<String,Integer> errorsByShortErrorMessage = new HashMap<String,Integer>();
protected final Map<String,Integer> failedRecordsSavedByErrorMessage = new HashMap<String,Integer>();
protected int failedRecordsSaved = 0;
// TODO: maybe concatenate messages in case of multiple errors
public void countFailedRecordsSaved (RecordWithErrors record) {
failedRecordsSaved++;
String message = record.errors.get(0).getMessage();
String message = record.errors.get(0).getMessageWithContext();
if (failedRecordsSavedByErrorMessage.containsKey(message)) {
failedRecordsSavedByErrorMessage.put(message,failedRecordsSavedByErrorMessage.get(message)+1);
} else {
......@@ -20,12 +20,12 @@ public class RecordFailureCounters {
}
public int incrementErrorCount(RecordError error) {
String errorKey = error.getCountingKey();
if (errorsByErrorMessage.containsKey(errorKey)) {
errorsByErrorMessage.put(errorKey,errorsByErrorMessage.get(errorKey)+1);
String errorKey = error.getShortMessageForCounting();
if (errorsByShortErrorMessage.containsKey(errorKey)) {
errorsByShortErrorMessage.put(errorKey, errorsByShortErrorMessage.get(errorKey)+1);
} else {
errorsByErrorMessage.put(errorKey, 1);
errorsByShortErrorMessage.put(errorKey, 1);
}
return errorsByErrorMessage.get(errorKey);
return errorsByShortErrorMessage.get(errorKey);
}
}
\ No newline at end of file
......@@ -85,11 +85,11 @@ public class RecordWithErrors {
addError(error);
int count = failCtrl.incrementErrorCount(error);
if (count <= 10) {
logger.error(error.getMessage());
logger.error(error.getMessageWithContext());
} else if (count>10 && count < 100) {
logger.log(logLevel, error.getBriefMessage());
logger.log(logLevel, error.getServerMessage());
} else if (count % 100 == 0) {
logger.error(String.format("%d records failed with %s", failCtrl.getErrorsByErrorKey(error.getCountingKey()), error.getCountingKey()));
logger.error(String.format("%d records failed with %s", failCtrl.getErrorsByShortErrorMessage(error.getShortMessageForCounting()), error.getShortMessageForCounting()));
}
}
......@@ -106,19 +106,18 @@ public class RecordWithErrors {
* and a brief error message with a total count for every 100 records with only that error thereafter.
*
* @param logger
* @param counters
*/
void writeErrorsLog(StorageJobLogger logger) {
if (hasErrors()) {
int i=0;
for (RecordError error : errors) {
i++;
int occurrences = failCtrl.getErrorsByErrorKey(error.getCountingKey());
int occurrences = failCtrl.getErrorsByShortErrorMessage(error.getShortMessageForCounting());
if (occurrences < 10) {
if (i==1) logger.error("Error" + (numberOfErrors() > 1 ? "s" : "") + " updating Inventory with " + transformedRecord.getJson());
logger.error("#" + i + " " + error.getMessage());
logger.error("#" + i + " " + error.getMessageWithContext());
} else if (occurrences % 100 == 0) {
logger.error(occurrences + " records have failed with " + error.getCountingKey());
logger.error(occurrences + " records have failed with " + error.getShortMessageForCounting());
}
}
}
......@@ -136,6 +135,8 @@ public class RecordWithErrors {
* @return byte array of the original record XML
*/
private byte[] getOriginalRecord () {
return transformedRecord.getOriginalContent();
/*
if (transformedRecord.getOriginalXml() != null) {
return transformedRecord.getOriginalXml().getBytes();
} else {
......@@ -146,6 +147,7 @@ public class RecordWithErrors {
return null;
}