Commit db0533be authored by Niels Erik G. Nielsen's avatar Niels Erik G. Nielsen
Browse files

Merge branch 'master' into marcftp

# Conflicts:
#	harvester/src/main/java/com/indexdata/masterkey/localindices/harvest/storage/folioinventory/InventoryRecordUpdater.java
#	harvester/src/main/java/com/indexdata/masterkey/localindices/util/MarcXMLToJson.java
parents c4b0ed61 9295e2ae
......@@ -177,11 +177,6 @@
mysql_db: name=localindices state=import target=/vagrant/sql/load-gbv-data-to-v2.13.sql
notify: Restart Tomcat
- name: Load reshare table data
become: yes
mysql_db: name=localindices state=import target=/vagrant/sql/load-reshare-data-to-v2.13.sql
notify: Restart Tomcat
- name: Build Harvester and Harvester Admin web apps
shell: mvn install > mvn-install.log creates=/vagrant/mvn-install.log chdir=/vagrant/
notify: Restart Tomcat
......
......@@ -8,19 +8,19 @@ public class ExceptionRecordError implements RecordError {
public String exceptionType;
public String message;
public String stackTrace;
public String shortDescription;
public String context;
public String typeOfEntity;
public ExceptionRecordError(Exception e, String shortDescription, String typeOfEntity) {
public ExceptionRecordError(Exception e, String context, String typeOfEntity) {
this.exceptionType = e.getClass().getSimpleName();
this.message = e.getLocalizedMessage();
this.stackTrace = stackTraceAsString(e);
this.shortDescription = shortDescription==null ? "" : shortDescription;
this.context = context==null ? "" : context;
this.typeOfEntity = typeOfEntity;
}
public ExceptionRecordError(Exception e, String shortDescription) {
this(e, shortDescription, "unspecified");
public ExceptionRecordError(Exception e, String context) {
this(e, context, "unspecified");
}
private String stackTraceAsString (Exception e) {
......@@ -32,16 +32,16 @@ public class ExceptionRecordError implements RecordError {
@Override
public String toString() {
return shortDescription + ". Exception ["+exceptionType+"]. " + message + ". Stacktrace [" + stackTrace + "]. " + (typeOfEntity.isEmpty() ? "" : "For entity type [" + typeOfEntity + "]");
return context + ". Exception ["+exceptionType+"]. " + message + ". Stacktrace [" + stackTrace + "]. " + (typeOfEntity.isEmpty() ? "" : "For entity type [" + typeOfEntity + "]");
}
@Override
public String getMessage() {
return shortDescription + "; [" + exceptionType + "]; " + message;
return context + "; [" + exceptionType + "]; " + message;
}
public String getLabel() {
return shortDescription;
public String getErrorContext() {
return context;
}
public String getType() {
......@@ -52,6 +52,10 @@ public class ExceptionRecordError implements RecordError {
return message;
}
public String getCountingKey () {
return message;
}
public String getStorageEntity() {
return typeOfEntity;
}
......
......@@ -196,8 +196,8 @@ public class FailedRecordsController {
return this.recordFailureCounters;
}
public int getErrorsByErrorMessage(String message) {
return getCounters().errorsByErrorMessage.get(message);
public int getErrorsByErrorKey(String errorKey) {
return getCounters().errorsByErrorMessage.get(errorKey);
}
public int incrementErrorCount (RecordError error) {
......
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package com.indexdata.masterkey.localindices.harvest.storage.folioinventory;
import java.net.URI;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
/**
*
* @author ne
*/
public class HttpDeleteWithBody extends HttpEntityEnclosingRequestBase {
public static final String METHOD_NAME = "DELETE";
@Override
public String getMethod() {
return METHOD_NAME;
}
public HttpDeleteWithBody(final String uri) {
super();
setURI(URI.create(uri));
}
public HttpDeleteWithBody(final URI uri) {
super();
setURI(uri);
}
public HttpDeleteWithBody() {
super();
}
}
......@@ -7,37 +7,39 @@ public class HttpRecordError implements RecordError {
public int statusCode;
public String reason;
public String response;
public String shortDescription;
public String context;
public String entity;
public String countingKey;
public HttpRecordError(int status, String reason, String response, String shortDescription, String entity) {
public HttpRecordError(int status, String reason, String response, String countingKey, String context, String entity) {
this.statusCode = status;
this.reason = reason;
this.response = response;
this.shortDescription = shortDescription;
this.context = context;
this.entity = entity;
this.countingKey = countingKey;
}
public HttpRecordError(int status, String reason, String response, String shortDescription) {
this(status, reason, response, shortDescription, "unspecified");
public HttpRecordError(int status, String reason, String response, String countingKey, String context) {
this(status, reason, response, context, countingKey, "unspecified");
}
public HttpRecordError(StatusLine httpStatus, String response, String additionalDescription, String entity) {
this(httpStatus.getStatusCode(), httpStatus.getReasonPhrase(), response, additionalDescription, entity);
public HttpRecordError(StatusLine httpStatus, String response, String countingKey, String context, String entity) {
this(httpStatus.getStatusCode(), httpStatus.getReasonPhrase(), response, countingKey, context, entity);
}
@Override
public String toString() {
return entity + ": " + shortDescription + ". Status code ["+statusCode+"]." + reason + "]." + response;
return entity + ": " + context + ". Status code ["+statusCode+"]." + reason + "]." + response;
}
@Override
public String getMessage() {
return shortDescription + "; " + reason + "; " + response;
return context + "; " + reason + "; " + response;
}
public String getLabel() {
return shortDescription;
public String getErrorContext() {
return context;
}
public String getType() {
......@@ -48,6 +50,10 @@ public class HttpRecordError implements RecordError {
return response;
}
public String getCountingKey() {
return countingKey;
}
public String getStorageEntity() {
return entity;
}
......
......@@ -80,31 +80,28 @@ public class InventoryStorageController implements RecordStorage {
this.databaseProperties = properties;
ctxt = new InventoryUpdateContext(harvestable, logger);
logger.info("Request to start job [" + database + "]"
+ ", storage URL [" + ctxt.folioAddress + "]"
+ (properties != null ? ", with db properties " + properties : " (no db properties defined) "));
logger.info("Starting job [" + database + "]");
logger.info("Storage URL [" + ctxt.folioAddress + (ctxt.useInventoryUpsert ? ctxt.inventoryUpsertPath : ctxt.instanceStoragePath));
if (ctxt.folioAuthSkip) logger.info("Storage configured to skip FOLIO authentication!");
client = HttpClients.createDefault();
ctxt.setClient(client);
authenticateToInventory();
if (!ctxt.folioAuthSkip) {
authenticateToInventory();
}
ctxt.setLocationsToInstitutionsMap(getLocationsMap());
}
private void authenticateToInventory() throws StorageException {
if (ctxt.folioUsername != null && ctxt.folioPassword != null && ctxt.folioTenant != null && ctxt.folioAuthPath != null) {
String authToken = getAuthtoken(client,
ctxt.folioAddress,
ctxt.folioAuthPath,
ctxt.folioUsername,
ctxt.folioPassword,
ctxt.folioTenant);
ctxt.setAuthToken(authToken);
logger.info("Authenticated to FOLIO Inventory, tenant [" + ctxt.folioTenant + "]");
} else {
logger.warn("Did not authenticate to FOLIO Inventory for lack of one or more configuration parameters. This will only work for FOLIO install with no authentication required.");
}
String authToken = getAuthtoken(client,
ctxt.folioAddress,
ctxt.folioAuthPath,
ctxt.folioUsername,
ctxt.folioPassword,
ctxt.folioTenant);
ctxt.setAuthToken(authToken);
logger.info("Authenticated to FOLIO Inventory, tenant [" + ctxt.folioTenant + "]");
}
/**
......@@ -163,7 +160,8 @@ public class InventoryStorageController implements RecordStorage {
setHeaders(httpGet, "application/json");
CloseableHttpResponse response = client.execute(httpGet);
if(! Arrays.asList(200, 404).contains(response.getStatusLine().getStatusCode())) {
throw new IOException(String.format("Got error retrieving locations",
throw new IOException(String.format("Got error '" +
response.getStatusLine().getStatusCode() + ": " + response.getStatusLine().getReasonPhrase() + "' when retrieving locations",
EntityUtils.toString(response.getEntity())));
}
JSONObject jsonResponse;
......@@ -183,7 +181,7 @@ public class InventoryStorageController implements RecordStorage {
throw new StorageException("Failed to retrieve any locations from Inventory, found no 'locations' in response");
}
} catch (IOException | ParseException e) {
throw new StorageException ("Error occurred trying to build map of location to institutions from FOLIO Inventory: " + e.getMessage());
throw new StorageException ("Error occurred trying to build map of locations to institutions from FOLIO Inventory: " + e.getMessage());
}
}
......
......@@ -17,10 +17,11 @@ import org.json.simple.parser.ParseException;
public class InventoryUpdateContext {
public Harvestable harvestable;
protected static final String FOLIO_AUTH_PATH = "folioAuthPath";
protected static final String FOLIO_TENANT = "folioTenant";
protected static final String FOLIO_AUTH_PATH = "folioAuthPath";
protected static final String FOLIO_USERNAME = "folioUsername";
protected static final String FOLIO_PASSWORD = "folioPassword";
protected static final String FOLIO_AUTH_SKIP = "folioAuthSkip";
protected static final String INSTANCE_STORAGE_PATH = "instanceStoragePath";
protected static final String HOLDINGS_STORAGE_PATH = "holdingsStoragePath";
protected static final String ITEM_STORAGE_PATH = "itemStoragePath";
......@@ -29,15 +30,17 @@ public class InventoryUpdateContext {
public String folioAddress;
private JSONObject storageConfig;
public String folioAuthPath;
public String folioTenant;
public String folioAuthPath;
public String folioUsername;
public String folioPassword;
public boolean folioAuthSkip = false;
public String instanceStoragePath;
public String holdingsStoragePath;
public String itemStoragePath;
public String marcStoragePath;
public String inventoryUpsertPath;
public boolean useInventoryUpsert = false;
public String instanceStorageUrl;
public String holdingsStorageUrl;
......@@ -126,13 +129,17 @@ public class InventoryUpdateContext {
}
private void setStorageConfigs() {
folioAuthPath = getRequiredConfig(FOLIO_AUTH_PATH);
folioTenant = getRequiredConfig(FOLIO_TENANT);
folioUsername = getRequiredConfig(FOLIO_USERNAME);
folioPassword = getRequiredConfig(FOLIO_PASSWORD);
folioAuthSkip = getConfig(FOLIO_AUTH_SKIP, "false").equalsIgnoreCase("true");
if (!folioAuthSkip) {
folioAuthPath = getRequiredConfig(FOLIO_AUTH_PATH);
folioUsername = getRequiredConfig(FOLIO_USERNAME);
folioPassword = getRequiredConfig(FOLIO_PASSWORD);
}
if (getConfig(INVENTORY_UPSERT_PATH) != null) {
useInventoryUpsert = true;
inventoryUpsertPath = getConfig(INVENTORY_UPSERT_PATH);
inventoryUpsertUrl = (inventoryUpsertPath != null ? folioAddress + inventoryUpsertPath : null);
} else {
instanceStoragePath = getRequiredConfig(INSTANCE_STORAGE_PATH);
instanceStorageUrl = folioAddress + instanceStoragePath;
......@@ -162,4 +169,8 @@ public class InventoryUpdateContext {
return (String) storageConfig.get(key);
}
public String getConfig(String key, String defaultValue) {
return (storageConfig.get(key) != null ? (String) storageConfig.get(key) : "false" );
}
}
\ No newline at end of file
......@@ -2,8 +2,9 @@ package com.indexdata.masterkey.localindices.harvest.storage.folioinventory;
public interface RecordError {
public String getMessage ();
public String getLabel();
public String getErrorContext();
public String getType();
public String getBriefMessage();
public String getStorageEntity();
public String getCountingKey();
}
\ No newline at end of file
......@@ -20,12 +20,12 @@ public class RecordFailureCounters {
}
public int incrementErrorCount(RecordError error) {
String message = error.getMessage();
if (errorsByErrorMessage.containsKey(message)) {
errorsByErrorMessage.put(message,errorsByErrorMessage.get(message)+1);
String errorKey = error.getCountingKey();
if (errorsByErrorMessage.containsKey(errorKey)) {
errorsByErrorMessage.put(errorKey,errorsByErrorMessage.get(errorKey)+1);
} else {
errorsByErrorMessage.put(message, 1);
errorsByErrorMessage.put(errorKey, 1);
}
return errorsByErrorMessage.get(message);
return errorsByErrorMessage.get(errorKey);
}
}
\ No newline at end of file
......@@ -73,15 +73,7 @@ public class RecordWithErrors {
}
void reportAndThrowError(RecordError error, Level logLevel, Throwable exception) throws InventoryUpdateException {
addError(error);
int count = failCtrl.incrementErrorCount(error);
if (count <= 10) {
logger.log(logLevel, error.getMessage());
} else if (count>10 && count < 100) {
logger.log(logLevel, error.getMessage());
} else if (count % 100 == 0) {
logger.error(String.format("%d records failed with %s", failCtrl.getErrorsByErrorMessage(error.getMessage()), error.getMessage()));
}
reportError(error, logLevel);
if (exception != null) {
throw new InventoryUpdateException(error.toString(),exception);
} else {
......@@ -89,20 +81,16 @@ public class RecordWithErrors {
}
}
void addResponseError(int status, String reason, String response, String message, String entity) {
errors.add(new HttpRecordError(status, reason, response, message, entity));
}
void addResponseError(int status, String reason, String response, String message) {
errors.add(new HttpRecordError(status, reason, response, message));
}
void addExceptionError(Exception e, String message, String entity) {
errors.add(new ExceptionRecordError(e, message, entity));
}
void addExceptionError(Exception e, String message) {
errors.add(new ExceptionRecordError(e, message, ""));
void reportError (RecordError error, Level logLevel) {
addError(error);
int count = failCtrl.incrementErrorCount(error);
if (count <= 10) {
logger.error(error.getMessage());
} else if (count>10 && count < 100) {
logger.log(logLevel, error.getBriefMessage());
} else if (count % 100 == 0) {
logger.error(String.format("%d records failed with %s", failCtrl.getErrorsByErrorKey(error.getCountingKey()), error.getCountingKey()));
}
}
boolean hasErrors () {
......@@ -125,12 +113,12 @@ public class RecordWithErrors {
int i=0;
for (RecordError error : errors) {
i++;
int occurrences = failCtrl.getErrorsByErrorMessage(error.getMessage());
int occurrences = failCtrl.getErrorsByErrorKey(error.getCountingKey());
if (occurrences < 10) {
if (i==1) logger.error("Error" + (numberOfErrors() > 1 ? "s" : "") + " updating Inventory with " + transformedRecord.getJson());
logger.error("#" + i + " " + error.getMessage());
} else if (occurrences % 100 == 0) {
logger.error(occurrences + " records have failed with " + error.getMessage());
logger.error(occurrences + " records have failed with " + error.getCountingKey());
}
}
}
......@@ -238,7 +226,7 @@ public class RecordWithErrors {
for (RecordError error : errors) {
Element errorElement = failedRecord.createElement("error");
Element labelElement = failedRecord.createElement("label");
labelElement.setTextContent(error.getLabel());
labelElement.setTextContent(error.getErrorContext());
Element errorTypeElement = failedRecord.createElement("type");
errorTypeElement.setTextContent(error.getType());
Element messageElement = failedRecord.createElement("message");
......
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment