Commit 0f4d39cf authored by Steffen Köhler's avatar Steffen Köhler
Browse files

Merge branch 'master' into morelogs

parents 02dcaec9 8135a4b8
......@@ -10,6 +10,7 @@ import java.net.Proxy;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
......@@ -28,19 +29,22 @@ import com.indexdata.masterkey.localindices.harvest.storage.StorageException;
public class BulkRecordHarvestJob extends AbstractRecordHarvestJob {
@SuppressWarnings("unused")
private List<URL> urls = new ArrayList<URL>();
private List<URL> urls = new ArrayList<>();
private XmlBulkResource resource;
// private RecordStorage transformationStorage;
private Proxy proxy;
private String errors;
private HarvestStatus initialStatus;
private final Date previousHarvestStarted;
public BulkRecordHarvestJob(XmlBulkResource resource, Proxy proxy) {
this.proxy = proxy;
this.resource = resource;
this.resource.setMessage(null);
initialStatus = HarvestStatus.valueOf(resource.getCurrentStatus());
setStatus(initialStatus);
previousHarvestStarted = resource.getLastHarvestStarted();
setLogger((new FileStorageJobLogger(getClass(), resource)));
}
......@@ -102,7 +106,7 @@ public class BulkRecordHarvestJob extends AbstractRecordHarvestJob {
if (getStatus() == HarvestStatus.RUNNING)
setStatus(HarvestStatus.OK);
if (getStatus() == HarvestStatus.WARN || getStatus() == HarvestStatus.ERROR) {
subject = "Harvest status: " + getStatus().toString();
subject = "Harvest status: " + getStatus();
msg = getHarvestable().getMessage();
logError(subject, msg);
}
......@@ -152,22 +156,22 @@ public class BulkRecordHarvestJob extends AbstractRecordHarvestJob {
}
private void downloadList(String[] list, boolean diskRun, DiskCache dc) throws Exception {
logger.debug("BulkRecordHarvestJob downloading list: " + Arrays.asList(list).toString());
logger.debug("BulkRecordHarvestJob downloading list: " + Arrays.asList(list));
Date lastDate = null;
if (resource.getAllowCondReq()) {
// conditonal request are enabled, manual override takes precedence and
// gets cleaned-up
lastDate = resource.getFromDate();
resource.setFromDate(null);
// when no manual override, use the last completion date
// when no manual override, use the last started date
if (lastDate == null) {
lastDate = initialStatus == HarvestStatus.ERROR || initialStatus == HarvestStatus.WARN ? null : resource.getLastHarvestFinished();
lastDate = initialStatus == HarvestStatus.ERROR || initialStatus == HarvestStatus.WARN ? null : previousHarvestStarted;
}
}
XmlMarcClient client = new XmlMarcClient(resource, this, proxy, logger, dc, lastDate);
for (String item : list) {
try {
int noErrors = 0;
int noErrors;
if (!diskRun) {
logger.debug("BulkRecordHarvestJob downloading URL: " + item);
noErrors = client.download(new URL(item));
......
......@@ -69,11 +69,12 @@ public class TransformationStepAssociationsResource {
public TransformationStepAssociationsConverter get(
@QueryParam("start") @DefaultValue("0") int start,
@QueryParam("max") @DefaultValue("100") int max) {
@QueryParam("max") @DefaultValue("100") int max,
@QueryParam("query") @DefaultValue("") String query) {
List<TransformationStepAssociation> entities;
EntityQuery qry = new EntityQuery();
EntityQuery qry = new EntityQuery().withQuery( query );
if (max <= 0)
entities = new ArrayList();
entities = new ArrayList<>();
else
entities = dao.retrieve(start, max, qry);
return new TransformationStepAssociationsConverter(entities, context.getAbsolutePath(), start,
......
......@@ -27,11 +27,11 @@ public class EntityQuery {
private Query query;
private String filter = "";
private List<String> keywordAllFields = new ArrayList<String>();
private List<String> keywordAllFields = new ArrayList<>();
private String acl = "";
private String startsWithField = "";
private String startsWith = "";
private static Logger LOGGER = Logger.getLogger("com.indexdata.masterkey.harvester");
private static final Logger LOGGER = Logger.getLogger("com.indexdata.masterkey.harvester");
private Query getQuery () {
......@@ -51,14 +51,31 @@ public class EntityQuery {
String term = "";
String operator = "";
String value = "";
Pattern qry = Pattern.compile("\\(?(\\w+)[ ]*(\\!?=)[ ]*([\\*\\w]+)\\)?");
Pattern qry = Pattern.compile("\\(?([.\\w]+)[ ]*(!?=)[ ]*(.+)\\)?");
Query (String query) {
Matcher matcher = qry.matcher(query);
if (matcher.find()) {
term = matcher.group(1);
operator = matcher.group(2);
value = matcher.group(3);
this.query = query;
if (query != null && !query.isEmpty())
{
LOGGER.debug( "Constructing Query object from " + query );
if (query.startsWith("(") && query.endsWith(")"))
query = query.substring(1, query.length()-1);
Matcher matcher = qry.matcher( query );
if ( matcher.find() )
{
term = matcher.group( 1 );
operator = matcher.group( 2 );
value = matcher.group( 3 );
this.query = query;
LOGGER.debug( "term: " + term);
LOGGER.debug( "operator: " + operator );
LOGGER.debug( "value: " + value );
}
else
{
LOGGER.error( "No Query object constructed, couldn't parse query " + query );
}
} else {
LOGGER.debug("No query requested, constructing empty Query object.");
}
}
......@@ -82,15 +99,16 @@ public class EntityQuery {
}
String asWhereClause (String tableAlias) {
StringBuilder str = new StringBuilder("");
str.append(" (")
return new StringBuilder()
.append(" (")
.append(tableAlias)
.append(".")
.append(term)
.append(value.contains("*") ? (operator.equals("!=") ? " NOT " : "") + " LIKE '" + value.replaceAll("\\*","%") + "'" : "")
.append(value.contains("*") ?
(operator.equals("!=") ? " NOT " : "") + " LIKE '" + value.replaceAll("\\*","%") + "'"
: "")
.append(!value.contains("*") ? operator + "'" + value + "'" : "")
.append(" )");
return str.toString();
.append(" )").toString();
}
}
......@@ -188,13 +206,11 @@ public class EntityQuery {
/**
* Constructs where clause that would look for 'filter' in any of the provided columns.
* @param filterString
* @param columns
* @param tableAlias
* @param tableAlias The SQL query's alias for the table to create WHERE clause for
* @return " where concat( [alias].[cols-1], '||', [alias].[cols-2], ...) like '%[filter]%' "
*/
private String getFilteringWhereClause(String tableAlias, boolean withWHERE) {
StringBuilder expr = new StringBuilder("");
StringBuilder expr = new StringBuilder();
if (filter != null && filter.length()>0 && keywordAllFields != null && keywordAllFields.size()>0) {
int i = 0;
expr.append(withWHERE ? " WHERE " : "");
......@@ -225,7 +241,7 @@ public class EntityQuery {
}
private String getAclWhereClause (String tableAlias, boolean withWHERE ) {
StringBuilder expr = new StringBuilder("");
StringBuilder expr = new StringBuilder();
if (hasAcl()) {
expr.append(withWHERE ? " WHERE " : "")
.append(tableAlias)
......@@ -238,10 +254,10 @@ public class EntityQuery {
}
public String asWhereClause (String tableAlias) {
String whereClause = "";
StringBuilder whereClause = new StringBuilder();
if (hasSearchCriteria()) {
whereClause += " WHERE ";
List<String> whereClauses = new ArrayList<String>();
whereClause.append( " WHERE " );
List<String> whereClauses = new ArrayList<>();
if (hasFilter()) {
whereClauses.add(getFilteringWhereClause(tableAlias));
}
......@@ -256,11 +272,11 @@ public class EntityQuery {
}
Iterator<String> iter = whereClauses.iterator();
while (iter.hasNext()) {
whereClause += iter.next();
if (iter.hasNext()) whereClause += " AND ";
whereClause.append( iter.next());
if (iter.hasNext()) whereClause.append(" AND ");
}
}
return whereClause;
return whereClause.toString();
}
private static boolean isNotEmpty(String str) {
......@@ -269,7 +285,6 @@ public class EntityQuery {
@Override
public boolean equals(Object o) {
String.join(", ", keywordAllFields);
if (o instanceof EntityQuery) {
EntityQuery p = (EntityQuery)o;
return (p.asWhereClause("alias").equals(this.asWhereClause("alias")));
......@@ -299,10 +314,10 @@ public class EntityQuery {
query.setQuery("(usedBy=library)");
System.out.println(query.asUrlParameters());
System.out.println(query.asWhereClause("x"));
query = new EntityQuery();
query.setQuery("(usedBy=library)");
System.out.println(query.asUrlParameters());
System.out.println(query.asUrlParameters());
System.out.println(query.asWhereClause("b"));
query.setQuery("usedBy=*library");
System.out.println(query.asUrlParameters());
......
......@@ -31,15 +31,21 @@ import com.indexdata.utils.persistence.EntityUtil;
* @author jakub
*/
public class HarvestablesDAOJPA implements HarvestableDAO {
private static final Logger LOGGER = Logger.getLogger("com.indexdata.masterkey.harvester.dao");
private static final Logger logger = Logger.getLogger("com.indexdata.masterkey.harvester.dao");
public HarvestablesDAOJPA () {
logger.setLevel( Level.INFO );
}
@Override
public List<Harvestable> retrieve(int start, int max, EntityQuery query) {
debugLog( "JPA: retrieve(start,max,query) harvestables, query: " + (query != null ? query.asUrlParameters() : "none" ));
return retrieve(start, max, null, true, query);
}
@Override
public List<HarvestableBrief> retrieveBriefs(int start, int max, EntityQuery query) {
debugLog( "JPA: retrieve harvestables briefs(start,max,query), query: " + (query != null ? query.asUrlParameters() : "none") );
return retrieveBriefs(start, max, null, true, query);
}
public enum AllowedSortKey {
......@@ -48,8 +54,8 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
DATE_STARTED("lastHarvestStarted", "o.lastHarvestStarted"),
DATE_FINISHED("lastHarvestFinished", "o.lastHarvestFinished"),
DATE_STARTED_OR_FINISHED("lastHarvestStartedOrFinished", "coalesce(o.lastHarvestFinished, o.lastHarvestStarted)");
private String sortKey;
private String orderExpression;
private final String sortKey;
private final String orderExpression;
AllowedSortKey(String sortKey, String orderExpression) {
this.sortKey = sortKey;
this.orderExpression = orderExpression;
......@@ -63,7 +69,7 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
public static List<AllowedSortKey> fromString(String spec) {
if (spec != null) {
String[] fieldNames = spec.split(",");
List<AllowedSortKey> allowed = new ArrayList<AllowedSortKey>(fieldNames.length);
List<AllowedSortKey> allowed = new ArrayList<>(fieldNames.length);
for (String fieldName : fieldNames) {
for (AllowedSortKey em : values()) {
if (fieldName.equalsIgnoreCase(em.sortKey))
......@@ -89,11 +95,11 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
em.persist(harvestable);
tx.commit();
} catch (Exception ex) {
LOGGER.log(Level.DEBUG, ex);
logger.log(Level.DEBUG, ex);
try {
tx.rollback();
} catch (Exception e) {
LOGGER.log(Level.DEBUG, e);
logger.log(Level.DEBUG, e);
}
} finally {
em.close();
......@@ -118,11 +124,11 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
harvestable = em.merge(updHarvestable);
tx.commit();
} catch (Exception ex) {
LOGGER.log(Level.DEBUG, ex);
logger.log(Level.DEBUG, ex);
try {
tx.rollback();
} catch (Exception e) {
LOGGER.log(Level.DEBUG, e);
logger.log(Level.DEBUG, e);
}
} finally {
em.close();
......@@ -140,11 +146,11 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
em.remove(harvestable);
tx.commit();
} catch (Exception ex) {
LOGGER.log(Level.DEBUG, ex);
logger.log(Level.DEBUG, ex);
try {
tx.rollback();
} catch (Exception e) {
LOGGER.log(Level.DEBUG, e);
logger.log(Level.DEBUG, e);
}
} finally {
em.close();
......@@ -153,6 +159,8 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
@Override
public List<Harvestable> retrieve(int start, int max, String sortKey, boolean asc, EntityQuery query) {
debugLog("JPA: retrieve harvestables(start,max,sortKey,asc,query), sortKey: " +sortKey + ", query: " + (query != null ? query.asUrlParameters() : "none" ));
EntityManager em = getEntityManager();
EntityTransaction tx = em.getTransaction();
// Communication errors with the persistence Layer will now just look like 0 records exists.
......@@ -171,19 +179,21 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
}
}
orderBy += (asc ? " ASC" : " DESC");
String qs = "select object(o) from Harvestable as o " + query.asWhereClause("o") + " order by "
+ orderBy;
String qs = "select object(o) from Harvestable as o "
+ (query != null ? query.asWhereClause("o") : "")
+ " order by " + orderBy;
debugLog("JPA query is: " + qs);
Query q = em.createQuery(qs);
q.setMaxResults(max);
q.setFirstResult(start);
hables = q.getResultList();
tx.commit();
} catch (Exception ex) {
LOGGER.log(Level.ERROR, "Failed to select Harvestable(s)", ex);
logger.log(Level.ERROR, "Failed to select Harvestable(s)", ex);
try {
tx.rollback();
} catch (Exception e) {
LOGGER.log(Level.DEBUG, e);
logger.log(Level.DEBUG, e);
}
// Some sort of analysis on the exception is required.
// Temporary should either ignored or throw as Interrupted
......@@ -203,8 +213,7 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
public int getCount(EntityQuery query) {
EntityManager em = getEntityManager();
try {
int count = ((Long) em.createQuery("select count(o) from Harvestable as o " + query.asWhereClause("o")).getSingleResult()).intValue();
return count;
return ((Long) em.createQuery("select count(o) from Harvestable as o " + query.asWhereClause("o")).getSingleResult()).intValue();
} finally {
em.close();
}
......@@ -213,7 +222,9 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
@Override
public List<HarvestableBrief> retrieveBriefs(int start, int max, String sortKey, boolean asc, EntityQuery query) {
List<HarvestableBrief> hrefs = new ArrayList<HarvestableBrief>();
debugLog("JPA: retrieve brief harvestables(start,max,sortKey,asc,query), sortKey: " +sortKey + ", query: " + (query != null ? query.asUrlParameters() : "none") );
List<HarvestableBrief> hrefs = new ArrayList<>();
List<Harvestable> list = retrieve(start, max, sortKey, asc, query);
if (list == null)
return null;
......@@ -243,4 +254,11 @@ public class HarvestablesDAOJPA implements HarvestableDAO {
public void resetCache(long id) throws DAOException {
throw new UnsupportedOperationException("DiskCache removal must be performed through the web service.");
}
private void debugLog( String logStatement) {
if (!Thread.currentThread().getName().equals("SchedulerThread" ))
{
logger.debug( logStatement );
}
}
}
......@@ -34,14 +34,13 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
/**
* create (POST) entity to the Web Service
* @param TransformationStepAssociation
* @return
* @param entity the TSA to create
*/
@Override
public void create(TransformationStepAssociation entity) {
try {
ResourceConnector<TransformationStepAssociationsConverter> connector =
new ResourceConnector<TransformationStepAssociationsConverter>(
new ResourceConnector<>(
new URL(serviceBaseURL),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
......@@ -59,7 +58,7 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
TransformationStepAssociation entity = null;
try {
ResourceConnector<TransformationStepAssociationConverter> connector =
new ResourceConnector<TransformationStepAssociationConverter>(
new ResourceConnector<>(
new URL(serviceBaseURL + id + "/"),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
......@@ -72,13 +71,13 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
/**
* PUT Transformation to the Web Service
* @param Transformation entity to be put
* @param entity Transformation to be put
*/
@Override
public TransformationStepAssociation update(TransformationStepAssociation entity) {
try {
ResourceConnector<TransformationStepAssociationConverter> connector =
new ResourceConnector<TransformationStepAssociationConverter>(
new ResourceConnector<>(
new URL(serviceBaseURL + entity.getId() + "/"),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
......@@ -96,7 +95,7 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
public void delete(TransformationStepAssociation entity) {
try {
ResourceConnector<TransformationStepAssociationConverter> connector =
new ResourceConnector<TransformationStepAssociationConverter>(
new ResourceConnector<>(
new URL(serviceBaseURL + entity.getId() + "/"),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
......@@ -111,13 +110,13 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
String url = serviceBaseURL + "?start=" + start + "&max=" + max + query.asUrlParameters();
try {
ResourceConnector<TransformationStepAssociationsConverter> connector =
new ResourceConnector<TransformationStepAssociationsConverter>(
new ResourceConnector<>(
new URL(url),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
TransformationStepAssociationsConverter hc = connector.get();
// TransformationStepAssociationsConverter hc = connector.get();
// TODO implement convertion !!!
hc.getReferences();
// hc.getReferences();
} catch (Exception male) {
logger.log(Level.DEBUG, male);
}
......@@ -129,7 +128,7 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
String url = serviceBaseURL + "?start=" + start + "&max=" + max + query.asUrlParameters();
try {
ResourceConnector<TransformationStepAssociationsConverter> connector =
new ResourceConnector<TransformationStepAssociationsConverter>(
new ResourceConnector<>(
new URL(url),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
......@@ -152,7 +151,7 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
String url = serviceBaseURL + "?start=0&max=0" + query.asUrlParameters();
try {
ResourceConnector<TransformationStepAssociationsConverter> connector =
new ResourceConnector<TransformationStepAssociationsConverter>(
new ResourceConnector<>(
new URL(url),
"com.indexdata.masterkey.localindices.entity" +
":com.indexdata.masterkey.localindices.web.service.converter");
......@@ -196,7 +195,7 @@ public class TransformationStepAssociationDAOWS extends CommonDAOWS implements T
@Override
public List<TransformationStepAssociationBrief> retrieveBriefs(int start,
int max, String sortKey, boolean asc, EntityQuery query) {
return retrieveBriefs(start, max, sortKey, asc, query);
return retrieveBriefs(start, max, query);
}
......
......@@ -111,20 +111,22 @@ public class TransformationStepAssociationsDAOJPA implements TransformationStepA
EntityManager em = getEntityManager();
EntityTransaction tx = em.getTransaction();
// HACK: Hides database errors but does not crash the Scheduler
List<TransformationStepAssociation> hables = new LinkedList<TransformationStepAssociation>();
List<TransformationStepAssociation> hables = new LinkedList<>();
try {
tx.begin();
Query q = em.createQuery("select object(o) from TransformationStepAssociation as o" + query.asWhereClause("o"));
String queryString = "select object(o) from TransformationStepAssociation as o" + query.asWhereClause("o") + " order by o.transformation.id, o.position";
logger.info("Query is: " + queryString);
Query q = em.createQuery(queryString);
q.setMaxResults(max);
q.setFirstResult(start);
hables = q.getResultList();
tx.commit();
} catch (Exception ex) {
logger.log(Level.DEBUG, ex);
logger.log(Level.INFO, ex);
try {
tx.rollback();
} catch (Exception e) {
logger.log(Level.DEBUG, e);
logger.log(Level.INFO, e);
}
} finally {
em.close();
......@@ -188,8 +190,7 @@ public class TransformationStepAssociationsDAOJPA implements TransformationStepA
try {
Query query = em.createQuery("select count(o) from TransformationStepAssociation o where o.transformation.id = :id");
query.setParameter("id", id);
int count = ((Long) query.getSingleResult()).intValue();
return count;
return ((Long) query.getSingleResult()).intValue();
} finally {
em.close();
}
......@@ -201,8 +202,7 @@ public class TransformationStepAssociationsDAOJPA implements TransformationStepA
try {
Query query = em.createQuery("select count(o) from TransformationStepAssociation o where o.step.id = :id");
query.setParameter("id", id);
int count = ((Long) query.getSingleResult()).intValue();
return count;
return ((Long) query.getSingleResult()).intValue();
} finally {
em.close();
}
......@@ -210,7 +210,7 @@ public class TransformationStepAssociationsDAOJPA implements TransformationStepA
@Override
public List<TransformationStepAssociationBrief> retrieveBriefs(int start, int max, EntityQuery query) {
List<TransformationStepAssociationBrief> hrefs = new ArrayList<TransformationStepAssociationBrief>();
List<TransformationStepAssociationBrief> hrefs = new ArrayList<>();
for (TransformationStepAssociation hable : retrieve(start, max, query)) {
TransformationStepAssociationBrief href = new TransformationStepAssociationBrief(hable);
hrefs.add(href);
......@@ -228,9 +228,8 @@ public class TransformationStepAssociationsDAOJPA implements TransformationStepA
public int getCount(EntityQuery entityQuery) {
EntityManager em = getEntityManager();
try {
Query query = em.createQuery("select count(o) from TransformationStepAssociation o");
int count = ((Long) query.getSingleResult()).intValue();
return count;
Query query = em.createQuery("select count(o) from TransformationStepAssociation o " + entityQuery.asWhereClause("o"));
return ((Long) query.getSingleResult()).intValue();
} finally {
em.close();
}
......@@ -245,7 +244,7 @@ public class TransformationStepAssociationsDAOJPA implements TransformationStepA
@Override
public List<TransformationStepAssociationBrief> retrieveBriefs(int start,
int max, String sortKey, boolean asc, EntityQuery query) {
return retrieveBriefs(start, max, sortKey, asc, query);
return retrieveBriefs(start, max, query);
}
}
......@@ -45,7 +45,6 @@ LOCK TABLES `TRANSFORMATION_STEP` WRITE;
/*!40000 ALTER TABLE `TRANSFORMATION_STEP` DISABLE KEYS */;
INSERT INTO `TRANSFORMATION_STEP` (`ID`, `DTYPE`, `POSITION`, `TRANSFORMATION_ID`, `STEP_ID`) VALUES (10005,'TransformationStepAssociation',2,10004,10002);
INSERT INTO `TRANSFORMATION_STEP` (`ID`, `DTYPE`, `POSITION`, `TRANSFORMATION_ID`, `STEP_ID`) VALUES (10006,'TransformationStepAssociation',1,10004,10003);
INSERT INTO `TRANSFORMATION_STEP` (`ID`, `DTYPE`, `POSITION`, `TRANSFORMATION_ID`, `STEP_ID`) VALUES (10007,'TransformationStepAssociation',3,10004,4003);
/*!40000 ALTER TABLE `TRANSFORMATION_STEP` ENABLE KEYS */;
UNLOCK TABLES;