Skip to content
Snippets Groups Projects
Commit b82706cb authored by Leila Gonzales's avatar Leila Gonzales Committed by Demian Katz
Browse files

Geographic features - update error logging routines (#1087)

parent 5bace375
No related merge requests found
Hide whitespace changes
Inline Side-by-side
import/index_java/src/org/vufind/index/GeoTools.java
+ 152
− 52
View file @ b82706cb
...@@ -25,18 +25,32 @@ package org.vufind.index; ...@@ -25,18 +25,32 @@ package org.vufind.index;
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/ */
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.marc4j.marc.Record;
import org.marc4j.marc.VariableField;
import org.marc4j.marc.ControlField; import org.marc4j.marc.ControlField;
import org.marc4j.marc.DataField; import org.marc4j.marc.DataField;
import org.marc4j.marc.Record;
import org.marc4j.marc.Subfield; import org.marc4j.marc.Subfield;
import java.util.HashMap; import org.marc4j.marc.VariableField;
import java.util.List; import org.solrmarc.index.indexer.ValueIndexerFactory;
import java.util.regex.Matcher; import org.solrmarc.index.SolrIndexer;
import java.util.regex.Pattern; import org.solrmarc.tools.PropertyUtils;
import org.solrmarc.tools.SolrMarcIndexerException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/** /**
* Geographic indexing routines. * Geographic indexing routines.
...@@ -46,10 +60,26 @@ public class GeoTools ...@@ -46,10 +60,26 @@ public class GeoTools
private static final Pattern COORDINATES_PATTERN = Pattern.compile("^([eEwWnNsS])(\\d{3})(\\d{2})(\\d{2})"); private static final Pattern COORDINATES_PATTERN = Pattern.compile("^([eEwWnNsS])(\\d{3})(\\d{2})(\\d{2})");
private static final Pattern HDMSHDD_PATTERN = Pattern.compile("^([eEwWnNsS])(\\d+(\\.\\d+)?)"); private static final Pattern HDMSHDD_PATTERN = Pattern.compile("^([eEwWnNsS])(\\d+(\\.\\d+)?)");
private static final Pattern PMDD_PATTERN = Pattern.compile("^([-+]?\\d+(\\.\\d+)?)"); private static final Pattern PMDD_PATTERN = Pattern.compile("^([-+]?\\d+(\\.\\d+)?)");
static String datePrefix = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date());
static String vufindLocal = System.getenv("VUFIND_LOCAL_DIR");
static String vufindHome = System.getenv("VUFIND_HOME");
private static Properties vufindConfigs = null;
// Initialize logging category // Initialize logging category
static Logger logger = Logger.getLogger(GeoTools.class.getName()); static Logger logger = Logger.getLogger(GeoTools.class.getName());
/**
* Constructor
*/
public GeoTools()
{
try {
vufindConfigs = PropertyUtils.loadProperties(ValueIndexerFactory.instance().getHomeDirs(), "vufind.properties");
} catch (IllegalArgumentException e) {
// If the properties load failed, don't worry about it -- we'll use defaults.
}
}
/** /**
* Convert MARC coordinates into long_lat format. * Convert MARC coordinates into long_lat format.
* *
...@@ -62,10 +92,6 @@ public class GeoTools ...@@ -62,10 +92,6 @@ public class GeoTools
if (list034 != null) { if (list034 != null) {
for (VariableField vf : list034) { for (VariableField vf : list034) {
HashMap<Character, String> coords = getCoordinateValues(vf); HashMap<Character, String> coords = getCoordinateValues(vf);
//DEBUG output
//ControlField recID = (ControlField) record.getVariableField("001");
//String recNum = recID.getData();
//logger.info("Record ID: " + recNum.trim() + " ...Coordinates: [ {" + coords.get('d') + "} {" + coords.get('e') + "} {" + coords.get('f') + "} {" + coords.get('g') + "} ]");
// Check for null coordinates // Check for null coordinates
if (validateCoordinateValues(record, coords)) { if (validateCoordinateValues(record, coords)) {
...@@ -79,11 +105,7 @@ public class GeoTools ...@@ -79,11 +105,7 @@ public class GeoTools
// Note - storage in Solr follows the WENS order, but display is WSEN order // Note - storage in Solr follows the WENS order, but display is WSEN order
String result = String.format("ENVELOPE(%s,%s,%s,%s)", new Object[] { west, east, north, south }); String result = String.format("ENVELOPE(%s,%s,%s,%s)", new Object[] { west, east, north, south });
geo_coordinates.add(result); geo_coordinates.add(result);
} else { }
logger.error(".......... Not indexing INVALID coordinates: [ {" + coords.get('d') + "} {" + coords.get('e') + "} {" + coords.get('f') + "} {" + coords.get('g') + "} ]");
}
} else {
logger.error(".......... Not indexing INVALID coordinates: [ {" + coords.get('d') + "} {" + coords.get('e') + "} {" + coords.get('f') + "} {" + coords.get('g') + "} ]");
} }
} }
} }
...@@ -106,14 +128,83 @@ public class GeoTools ...@@ -106,14 +128,83 @@ public class GeoTools
if (validateCoordinateValues(record, coords)) { if (validateCoordinateValues(record, coords)) {
String result = String.format("%s %s %s %s", new Object[] { coords.get('d'), coords.get('e'), coords.get('f'), coords.get('g') }); String result = String.format("%s %s %s %s", new Object[] { coords.get('d'), coords.get('e'), coords.get('f'), coords.get('g') });
geo_coordinates.add(result); geo_coordinates.add(result);
} else {
logger.error(".......... Not indexing INVALID Display coordinates: [ {" + coords.get('d') + "} {" + coords.get('e') + "} {" + coords.get('f') + "} {" + coords.get('g') + "} ]");
} }
} }
} }
return geo_coordinates; return geo_coordinates;
} }
/**
* Log coordinate indexing errors to external log file.
*
* @param Record record
* @param HashMap coords
* @param String error message
*/
public static void logErrorMessage(Record record, HashMap coords, String message) {
// Initialize error logging variables
String msgError = message;
String recNum = "Not available";
ControlField recID = (ControlField) record.getVariableField("001");
if (recID != null) {
recNum = recID.getData().trim();
}
String coordinates = "Coordinates: {" + coords.get('d') + "} {" + coords.get('e') + "} {" + coords.get('f') + "} {" + coords.get('g') + "}";
String logPath = getLogPath();
String logFilename = datePrefix + "_CoordinateErrors.txt";
String outPath = logPath + "/" + logFilename;
// Output Error message
logger.error("Not indexing INVALID coordinates for Record ID: " + recNum);
logger.error("... " + msgError);
if (logPath != null) {
logger.error("... Check coordinate error log: " + outPath);
// Log ID and error message and coordinates in error file
try {
PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(outPath, true)));
out.println(logFilename + "\t" + recNum + "\t" + msgError + "\t" + coordinates);
out.close();
} catch (IOException e) {
System.out.println("io exception occured");
e.printStackTrace();
}
} else {
// output error that log file cannot be created
logger.error("..... No coordinate error log. Check vufind.properties settings...");
}
}
/**
* Get path for coordinate error log file.
*
* @return String logPath
*/
public static String getLogPath() {
// Get coordinate error log path setting
String coordLogPath = PropertyUtils.getProperty(vufindConfigs, "coordinate.log.path");
//If coordinate.log.path doesn't exist or is not set, try some other places
if (coordLogPath == null) {
if (vufindLocal != null) {
File dir = new File(vufindLocal + "/import");
if (dir.exists()) {
coordLogPath = vufindLocal + "/import";
}
} else if (vufindLocal == null && vufindHome != null) {
File dir = new File(vufindHome + "/import");
if (dir.exists()) {
coordLogPath = vufindHome + "/import";
}
} else {
coordLogPath = null;
}
}
return coordLogPath;
}
/** /**
* Get all coordinate values from list034 * Get all coordinate values from list034
* *
...@@ -164,9 +255,8 @@ public class GeoTools ...@@ -164,9 +255,8 @@ public class GeoTools
if (coords.containsKey('d') && coords.containsKey('e') && coords.containsKey('f') && coords.containsKey('g')) { if (coords.containsKey('d') && coords.containsKey('e') && coords.containsKey('f') && coords.containsKey('g')) {
return true; return true;
} }
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "Coordinate values contain null values.";
String recNum = recID.getData(); logErrorMessage(record, coords, msgError);
logger.error("Record ID: " + recNum.trim() + " - Coordinate values contain null values.");
return false; return false;
} }
...@@ -211,8 +301,7 @@ public class GeoTools ...@@ -211,8 +301,7 @@ public class GeoTools
coordinate = Double.parseDouble(PMDmatcher.group(1)); coordinate = Double.parseDouble(PMDmatcher.group(1));
return coordinate; return coordinate;
} else { } else {
logger.error("Decimal Degree Coordinate Conversion Error: Poorly formed coordinate: [" + coordinateStr + "] ... Returning null value ... "); return Double.NaN;
return null;
} }
} }
...@@ -235,7 +324,7 @@ public class GeoTools ...@@ -235,7 +324,7 @@ public class GeoTools
} }
return coordinate; return coordinate;
} }
return null; return Double.NaN;
} }
/** /**
...@@ -280,9 +369,9 @@ public class GeoTools ...@@ -280,9 +369,9 @@ public class GeoTools
*/ */
public boolean validateLines(Record record, Double west, Double east, Double north, Double south) { public boolean validateLines(Record record, Double west, Double east, Double north, Double south) {
if ((!west.equals(east) && north.equals(south)) && (north == 90 || south == -90)) { if ((!west.equals(east) && north.equals(south)) && (north == 90 || south == -90)) {
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "Coordinates form a line at the pole";
String recNum = recID.getData(); HashMap<Character, String> coords = buildCoordinateHashMap(west, east, north, south);
logger.error("Record ID: " + recNum.trim() + " - Coordinates form a line at the pole"); logErrorMessage(record, coords, msgError);
return false; return false;
} }
return true; return true;
...@@ -296,10 +385,10 @@ public class GeoTools ...@@ -296,10 +385,10 @@ public class GeoTools
* @return boolean * @return boolean
*/ */
public boolean validateValues(Record record, Double west, Double east, Double north, Double south) { public boolean validateValues(Record record, Double west, Double east, Double north, Double south) {
if (west == null || east == null || north == null || south == null) { if (west.isNaN() || east.isNaN() || north.isNaN() || south.isNaN()) {
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "Decimal Degree coordinates contain invalid values";
String recNum = recID.getData(); HashMap<Character, String> coords = buildCoordinateHashMap(west, east, north, south);
logger.error("Record ID: " + recNum.trim() + " - Decimal Degree coordinates contain null values: [ {" + west + "} {" + east + "} {" + north + "} {" + south + "} ]"); logErrorMessage(record, coords, msgError);
return false; return false;
} }
return true; return true;
...@@ -313,16 +402,12 @@ public class GeoTools ...@@ -313,16 +402,12 @@ public class GeoTools
* @return boolean * @return boolean
*/ */
public boolean validateExtent(Record record, Double west, Double east, Double north, Double south) { public boolean validateExtent(Record record, Double west, Double east, Double north, Double south) {
if (west > 180.0 || west < -180.0 || east > 180.0 || east < -180.0) { if (west > 180.0 || west < -180.0 || east > 180.0 || east < -180.0
ControlField recID = (ControlField) record.getVariableField("001"); || north > 90.0 || north < -90.0 || south > 90.0 || south < -90.0
String recNum = recID.getData(); ) {
logger.error("Record ID: " + recNum.trim() + " - Coordinates exceed map extent."); String msgError = "Coordinates exceed map extent.";
return false; HashMap<Character, String> coords = buildCoordinateHashMap(west, east, north, south);
} logErrorMessage(record, coords, msgError);
if (north > 90.0 || north < -90.0 || south > 90.0 || south < -90.0) {
ControlField recID = (ControlField) record.getVariableField("001");
String recNum = recID.getData();
logger.error("Record ID: " + recNum.trim() + " - Coordinates exceed map extent.");
return false; return false;
} }
return true; return true;
...@@ -337,9 +422,9 @@ public class GeoTools ...@@ -337,9 +422,9 @@ public class GeoTools
*/ */
public boolean validateNorthSouth(Record record, Double north, Double south) { public boolean validateNorthSouth(Record record, Double north, Double south) {
if (north < south) { if (north < south) {
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "North < South.";
String recNum = recID.getData(); HashMap<Character, String> coords = buildCoordinateHashMap(Double.NaN, Double.NaN, north, south);
logger.error("Record ID: " + recNum.trim() + " - North < South."); logErrorMessage(record, coords, msgError);
return false; return false;
} }
return true; return true;
...@@ -363,9 +448,9 @@ public class GeoTools ...@@ -363,9 +448,9 @@ public class GeoTools
} }
// Check again // Check again
if (east < west) { if (east < west) {
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "East < West.";
String recNum = recID.getData(); HashMap<Character, String> coords = buildCoordinateHashMap(west, east, Double.NaN, Double.NaN);
logger.error("Record ID: " + recNum.trim() + " - East < West."); logErrorMessage(record, coords, msgError);
return false; return false;
} }
} }
...@@ -386,19 +471,34 @@ public class GeoTools ...@@ -386,19 +471,34 @@ public class GeoTools
//Check for South Pole coordinate distance //Check for South Pole coordinate distance
if ((north == -90 || south == -90) && (distNS > 0 && distNS < 0.167)) { if ((north == -90 || south == -90) && (distNS > 0 && distNS < 0.167)) {
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "Coordinates < 0.167 degrees from South Pole. Coordinate Distance: "+distNS;
String recNum = recID.getData(); HashMap<Character, String> coords = buildCoordinateHashMap(west, east, north, south);
logger.error("Record ID: " + recNum.trim() + " - Coordinates < 0.167 degrees from South Pole. Coordinate Distance: "+distNS); logErrorMessage(record, coords, msgError);
return false; return false;
} }
//Check for East-West coordinate distance //Check for East-West coordinate distance
if ((west == 0 || east == 0) && (distEW > -2 && distEW <0)) { if ((west == 0 || east == 0) && (distEW > -2 && distEW <0)) {
ControlField recID = (ControlField) record.getVariableField("001"); String msgError = "Coordinates within 2 degrees of Prime Meridian. Coordinate Distance: "+distEW;
String recNum = recID.getData(); HashMap<Character, String> coords = buildCoordinateHashMap(west, east, north, south);
logger.error("Record ID: " + recNum.trim() + " - Coordinates within 2 degrees of Prime Meridian. Coordinate Distance: "+distEW); logErrorMessage(record, coords, msgError);
return false; return false;
} }
return true; return true;
} }
/**
* Build coordinate hash map for logging.
*
* @param Double west, east, north, south
* @return HashMap coords
*/
public HashMap buildCoordinateHashMap (Double west, Double east, Double north, Double south) {
HashMap<Character, String> coords = new HashMap();
coords.put('d', Double.toString(west));
coords.put('e', Double.toString(east));
coords.put('f', Double.toString(north));
coords.put('g', Double.toString(south));
return coords;
}
} }
import/vufind.properties
+ 5
− 0
View file @ b82706cb
...@@ -6,3 +6,8 @@ ...@@ -6,3 +6,8 @@
# look in the VuFind 2.x-style directory path, but you can uncomment the line # look in the VuFind 2.x-style directory path, but you can uncomment the line
# below for compatibility with VuFind 1.x. # below for compatibility with VuFind 1.x.
#vufind.config.relative_path = web/conf #vufind.config.relative_path = web/conf
# coordinate error log location - where to put coordinate indexing error logs
# if not set, the default path will be VUFIND_LOCAL_DIR/import, and failing that,
# VUFIND_HOME/import.
#coordinate.log.path = /usr/local/vufind/import
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment