Reduce debug logs. Some new logs. Exception guard in finally block.

harvester/src/main/java/com/indexdata/masterkey/localindices/client/XmlMarcClient.java

@@ -292,8 +292,13 @@ public class XmlMarcClient extends AbstractHarvestClient {
         //       FTPConnectionClosedException will be thrown here.
         //       This exception is used when deciding whether to attempt a reconnect.
         //       See download(URL url)
-        input.close();
-        logger.debug("StoreAny: Input stream closed.");
+        logger.debug("In storeAny - finally block. Attempting to close input");
+        try {
+          if (input != null) input.close();
+          logger.debug("StoreAny: Input stream closed.");
+        } catch (IOException ioe) {
+          logger.debug("IO exception when attempting to close input: " + ioe.getMessage());
+        }
     }
   }
 

harvester/src/main/java/com/indexdata/masterkey/localindices/harvest/job/BulkRecordHarvestJob.java

@@ -9,6 +9,7 @@ import java.io.File;
 import java.net.Proxy;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 
@@ -149,6 +150,7 @@ public class BulkRecordHarvestJob extends AbstractRecordHarvestJob {
   }
 
   private void downloadList(String[] list, boolean diskRun, DiskCache dc) throws Exception {
+    logger.debug("Downloading list: " + Arrays.asList(list).toString());
     Date lastDate = null;
     if (resource.getAllowCondReq()) {
       // conditonal request are enabled, manual override takes precedence and
@@ -164,10 +166,12 @@ public class BulkRecordHarvestJob extends AbstractRecordHarvestJob {
     for (String item : list) {
       try {
         int noErrors = 0;
-        if (!diskRun)
+        if (!diskRun) {
+          logger.debug("Downloading URL: " + item);
           noErrors = client.download(new URL(item));
-        else
+        } else {
           noErrors = client.download(new File(item));
+        }
         if (noErrors > 0) {
           setStatus(HarvestStatus.WARN, client.getErrors());
         }

harvester/src/main/java/com/indexdata/masterkey/localindices/harvest/messaging/InstanceXmlToInstanceJsonTransformerRouter.java

@@ -9,6 +9,7 @@ import java.util.NoSuchElementException;
 
 import javax.xml.transform.TransformerException;
 
+import org.apache.log4j.Level;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.w3c.dom.Node;
@@ -109,7 +110,7 @@ public class InstanceXmlToInstanceJsonTransformerRouter implements MessageRouter
 
   private void consume(Object documentIn) {
     if (documentIn instanceof Record) {
-      logger.debug("documentIn is of class " + documentIn.getClass().getName());
+      logger.log(Level.TRACE, "documentIn is of class " + documentIn.getClass().getName());
       Record recordIn = (Record) documentIn;
       RecordJSON recordOut = new RecordJSONImpl();
       try {
@@ -117,14 +118,14 @@ public class InstanceXmlToInstanceJsonTransformerRouter implements MessageRouter
           if (recordIn.getSubRecords().isEmpty()) {
             logger.debug(this.getClass().getSimpleName() + ": Empty collection came in from queue, skipping further processing of this document (split level could be set too high)");
           } else {
-            logger.debug(this.getClass().getSimpleName() + " has Record with " + recordIn.getSubRecords().size() + " sub record(s). Iterating:");
+            logger.log(Level.TRACE, this.getClass().getSimpleName() + " has Record with " + recordIn.getSubRecords().size() + " sub record(s). Iterating:");
             JSONObject jsonRecords = new JSONObject();
             Collection<Record> subrecords = recordIn.getSubRecords();
             jsonRecords.put("collection", new JSONArray());
             int i=0;
             for (Record rec : subrecords) {
               try {
-                logger.debug(this.getClass().getSimpleName() + ": Sub record " + ++i);
+                logger.log(Level.TRACE, this.getClass().getSimpleName() + ": Sub record " + ++i);
                 JSONObject json = makeInventoryJson(rec);
                 ((JSONArray)(jsonRecords.get("collection"))).add(json);
               } catch(Exception e) {

harvester/src/main/java/com/indexdata/masterkey/localindices/harvest/storage/TransformationRecordStorageProxy.java

@@ -72,7 +72,7 @@ public class TransformationRecordStorageProxy extends AbstractTransformationReco
     if (job.isKillSent()) {
       throw new RuntimeException("Job killed");
     }
-    logger.debug("TransformationRecordStorageProxy adding record of class "
+    logger.log(Level.TRACE, "TransformationRecordStorageProxy adding record of class "
         + record.getClass().getName() + ", target is of class " + getTarget().getClass().getName());
     RecordDOMImpl recordDOM = new RecordDOMImpl(record);
     while (true) {

harvester/src/main/java/com/indexdata/masterkey/localindices/harvest/storage/folioinventory/InventoryRecordUpdater.java

@@ -802,7 +802,7 @@ import com.indexdata.masterkey.localindices.util.MarcXMLToJson;
           logger.log(Level.TRACE,"Treating source record as ISO2079");
           marcJson = MarcToJson.convertMarcRecordsToJson(originalContentString).get(0);
         }
-        logger.debug(marcJson.toJSONString());
+        logger.log(Level.TRACE,marcJson.toJSONString());
       } catch (IOException | ParserConfigurationException | SAXException e) {
 
         updateCounters.sourceRecordsFailed++;
@@ -900,9 +900,9 @@ import com.indexdata.masterkey.localindices.util.MarcXMLToJson;
         JSONObject marcRecords = (JSONObject) (new JSONParser().parse(responseAsString));
         final Long count = (Long) (marcRecords.getOrDefault("totalRecords", 0));
         if (count == 0) {
-          logger.debug("No MARC source record found for instance [" + instanceId + "], institution [" + institutionId + "] and local identifier [" + localIdentifier + "]");
+          logger.log(Level.TRACE,"No MARC source record found for instance [" + instanceId + "], institution [" + institutionId + "] and local identifier [" + localIdentifier + "]");
         } else if (count == 1) {
-          logger.debug("Found existing MARC source record for instance [" + instanceId + "], institution [" + institutionId + "] and local identifier [" + localIdentifier + "]");
+          logger.log(Level.TRACE, "Found existing MARC source record for instance [" + instanceId + "], institution [" + institutionId + "] and local identifier [" + localIdentifier + "]");
           JSONArray records = (JSONArray) marcRecords.get("marcrecords");
           marcRecord = (JSONObject) records.get(0);
         } else {
@@ -914,7 +914,7 @@ import com.indexdata.masterkey.localindices.util.MarcXMLToJson;
       RecordError error = new ExceptionRecordError(e, "Error when checking for previously existing MARC record","MARC source","GET", "");
       recordWithErrors.reportAndThrowError(error, Level.DEBUG);
     }
-    logger.debug("Got existing MARC in: " + (System.currentTimeMillis()-startGetExistingMarc));
+    logger.log(Level.TRACE,"Got existing MARC in: " + (System.currentTimeMillis()-startGetExistingMarc));
     return marcRecord;
   }
 
@@ -940,7 +940,7 @@ import com.indexdata.masterkey.localindices.util.MarcXMLToJson;
         HttpEntityEnclosingRequestBase request;
         JSONObject marcRecord = getExistingMarcRecord(instanceId, institutionId, localIdentifier);
         if (marcRecord == null) {
-          logger.debug("This MARC record did not exist in storage; creating it.");
+          logger.log(Level.TRACE,"This MARC record did not exist in storage; creating it.");
           long startAddingMarc = System.currentTimeMillis();
           request = new HttpPost(ctxt.marcStorageUrl);
           request.setEntity(entity);
@@ -953,13 +953,13 @@ import com.indexdata.masterkey.localindices.util.MarcXMLToJson;
             RecordError error = new HttpRecordError(response.getStatusLine(), responseAsString, responseAsString, "Error adding MARC source record ", "MARC source", "POST", "{}");
             recordWithErrors.reportAndThrowError(error, Level.DEBUG);
           } else {
-            logger.debug("Status code: " + response.getStatusLine().getStatusCode() + " for POST of marc json " + marcPostJson.toJSONString());
+            logger.log(Level.TRACE, "Status code: " + response.getStatusLine().getStatusCode() + " for POST of marc json " + marcPostJson.toJSONString());
             updateCounters.sourceRecordsLoaded++;
           }
           response.close();
-          logger.debug("Added MARC in " + (System.currentTimeMillis()-startAddingMarc));
+          logger.log(Level.TRACE, "Added MARC in " + (System.currentTimeMillis()-startAddingMarc));
         } else {
-          logger.debug("This MARC record already existed in storage; updating it.");
+          logger.log(Level.TRACE, "This MARC record already existed in storage; updating it.");
           long startUpdatingMarc = System.currentTimeMillis();
           String id = (String) marcRecord.get("id");
           String url = ctxt.marcStorageUrl + "/" + id;
@@ -973,10 +973,10 @@ import com.indexdata.masterkey.localindices.util.MarcXMLToJson;
             RecordError error = new HttpRecordError(response.getStatusLine(), response.getEntity().toString(), response.getEntity().toString(), "Error updating existing MARC record", "MARC source", "PUT", "{}");
             recordWithErrors.reportAndThrowError(error, Level.DEBUG);
           } else {
-            logger.debug("Status code: " + response.getStatusLine().getStatusCode() + " for PUT of marc json " + marcPostJson.toJSONString());
+            logger.log(Level.TRACE,"Status code: " + response.getStatusLine().getStatusCode() + " for PUT of marc json " + marcPostJson.toJSONString());
             updateCounters.sourceRecordsLoaded++;
           }
-          logger.debug("Updated MARC in " + (System.currentTimeMillis()-startUpdatingMarc));
+          logger.log(Level.TRACE, "Updated MARC in " + (System.currentTimeMillis()-startUpdatingMarc));
         }
       } catch (IOException | org.apache.http.ParseException | UnsupportedCharsetException e) {
         updateCounters.sourceRecordsFailed++;