Commit e6853680 authored by Jens Kupferschmidt's avatar Jens Kupferschmidt
Browse files

Merge branch 'mymss-common_2021_06' of...

Merge branch 'mymss-common_2021_06' of git@git.sc.uni-leipzig.de:mycore_applications/orient/mymss-common.git into mymss-common_2021_06
parents 7d610826 cae91623
Pipeline #12122 passed with stage
in 2 minutes and 56 seconds
......@@ -222,4 +222,4 @@
</dependency>
</dependencies>
</project>
\ No newline at end of file
</project>
......@@ -40,7 +40,7 @@ public class MCRExportCommands {
}
/**
* Converts a list of MyCoRe objects represented by their Ids to CSV. Currently only manuscripts are supported.
* Converts a list of MyCoRe objects represented by their Ids to CSV. Currently, only manuscripts are supported.
*
* @param ids the Ids of the MyCoRe objects to convert
* @return the CSV string lines
......@@ -56,7 +56,7 @@ public class MCRExportCommands {
"Anzahl Bände;Sammelband;" +
"Sprache;Einband;Person;" +
"Beschreibstoff - Material;Beschreibstoff - Farbe;Beschreibstoff - Wasserzeichen;" +
"Beschreibstoff - Zustand" +
"Beschreibstoff - Zustand;" +
"Blattzahl;Format;Textspiegel;Zeilenzahl;Spaltenzahl;Kustoden;Schrift - Duktus;Schrift - Tinte;" +
"Schrift - Ausführung;Illumination;Illustrationen;Miniaturen;Datumsangaben;Ortsangaben;" +
"Besitzervermerke;Titel - in HS;Titel - wie in Ref.;Titel - Varianten;Vollständigkeit;Textanfang wie" +
......
......@@ -21,11 +21,18 @@
package org.mycore.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.ibm.icu.util.Calendar;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.jdom2.Document;
import org.jdom2.Element;
import org.mycore.common.MCRCalendar;
import org.mycore.common.MCRException;
import org.mycore.common.config.MCRConfiguration2;
import org.mycore.datamodel.classifications2.MCRCategory;
......@@ -34,8 +41,10 @@ import org.mycore.datamodel.classifications2.MCRCategoryID;
import org.mycore.datamodel.classifications2.utils.MCRCategoryTransformer;
import org.mycore.frontend.servlets.MCRServlet;
import org.mycore.frontend.servlets.MCRServletJob;
import org.mycore.services.fieldquery.MCRQuery;
import org.mycore.services.i18n.MCRTranslation;
import org.mycore.solr.MCRSolrClientFactory;
import org.mycore.solr.search.MCRSolrSearchUtils;
import java.io.IOException;
import java.util.ArrayList;
......@@ -58,13 +67,11 @@ public class SearchFormBuilderServlet extends MCRServlet {
private static final String PARAM_OBJ_TYPE = "objectType";
private static final String CONFIG_PREFIX_FIELDS = "MCR.SearchFormBuilder.Fields.";
private static final String I18N_CLASS_IS = "module.mymss.search.flexible.classification.is";
private static final String I18N_CLASS_NOT = "module.mymss.search.flexible.classification.not";
private static final String I18N_TEXT_CONTAINS = "module.mymss.search.flexible.text.contains";
private static final String I18N_TEXT_STARTS_WITH = "module.mymss.search.flexible.text.startswith";
private static final String I18N_TEXT_IS = "module.mymss.search.flexible.text.is";
private static final String CONFIG_CALENDARS = "MCR.SearchFormBuilder.Calendars";
private static final String I18N_RESULT_COUNT = "module.mymss.search.flexible.resultcount";
private static final String I18N_PREFIX_CALENDAR = "module.dptbase.common.editor.calendar.";
private static final Logger LOGGER = LogManager.getLogger();
@Override
protected void doGetPost(MCRServletJob job) throws Exception {
......@@ -96,6 +103,10 @@ public class SearchFormBuilderServlet extends MCRServlet {
job.getResponse().sendRedirect(url);
return;
}
case BuildCalendarOptions: {
result = jsonMapper.writeValueAsString(buildCalendarOptions());
break;
}
default: {
throw new RuntimeException("Unsupported Action " + action);
}
......@@ -125,17 +136,14 @@ public class SearchFormBuilderServlet extends MCRServlet {
operator = null;
}
final String value;
if (parameters.containsKey("value-" + id)) {
value = parameters.get("value-" + id)[0];
} else {
value = null;
}
final String[] values = parameters.getOrDefault("value-" + id, null);
if (StringUtils.isBlank(value) || StringUtils.isBlank(operator)) {
if (ArrayUtils.isEmpty(values) || StringUtils.isBlank(operator) ||
StringUtils.isBlank(String.join("", values))) {
return;
}
final Operator op = Operator.get(operator);
final QueryType queryType = QueryType.get(type);
final String solrParamName;
final String solrParamValue;
......@@ -143,39 +151,100 @@ public class SearchFormBuilderServlet extends MCRServlet {
case Classification: {
solrParamName = "fq";
if (StringUtils.equals(operator, "+")) {
solrParamValue = field + ":\"" + value + "\"";
solrParamValue = field + ":\"" + values[0] + "\"";
} else if (StringUtils.equals(operator, "-")) {
solrParamValue = "-" + field + ":\"" + value + "\"";
solrParamValue = "-" + field + ":\"" + values[0] + "\"";
} else {
throw new MCRException("Unsupported Operator " + operator);
}
solrParams.add(solrParamName, solrParamValue);
break;
}
case Text: {
solrParamName = "q";
final String parsedQueryString;
switch (op) {
case TextIs: {
parsedQueryString = values[0];
break;
}
case TextContains: {
parsedQueryString = "*" + values[0] + "*";
break;
}
case TextStartsWith: {
parsedQueryString = values[0] + "*";
break;
}
default: {
throw new UnsupportedOperationException("Operator " + op + " not supported!");
}
}
if (StringUtils.equals(operator, "is")) {
solrParamValue = field + ":" + value;
} else if (StringUtils.equals(operator, "startsWith")) {
solrParamValue = field + ":" + value + "*";
} else if (StringUtils.equals(operator, "contains")) {
solrParamValue = field + ":*" + value + "*";
// for text queries use MCRQuery and transform it to a Solr query
final Element conditionSearch = new Element("condition");
conditionSearch.setAttribute("field", field);
conditionSearch.setAttribute("operator", "contains");
conditionSearch.setAttribute("value", parsedQueryString);
final Element conditions = new Element("conditions");
conditions.addContent(conditionSearch);
final Element queryEl = new Element("query");
queryEl.addContent(conditions);
final MCRQuery query = MCRQuery.parseXML(new Document(queryEl));
final SolrQuery solrQuery = MCRSolrSearchUtils.getSolrQuery(query, query.buildXML(), null);
final String queryString = solrQuery.get("q");
// for q-queries we have to add them all to one param
// otherwise Solr does not perform the right search
solrParamName = "q";
final String current = solrParams.get(solrParamName);
if (StringUtils.isNotBlank(current)) {
solrParams.set(solrParamName, current + " " + queryString);
} else {
throw new MCRException("Unsupported operator " + operator);
solrParams.set(solrParamName, queryString);
}
break;
}
case Bool: {
solrParamName = "fq";
solrParamValue = field + ":true";
solrParams.add(solrParamName, solrParamValue);
break;
}
case DateRange: {
final String calType = parameters.get("caltype-" + id)[0];
final String[] fields = StringUtils.split(field, ",");
final String fromDateField = fields[0];
final String toDateField = fields[1];
final long fromDateNumber;
final long toDateNumber;
if (StringUtils.isNotBlank(values[0])) {
final String fromDate = values[0];
final Calendar fromDateBegin = MCRCalendar.getHistoryDateAsCalendar(fromDate, false, calType);
fromDateNumber = MCRCalendar.getJulianDayNumber(fromDateBegin);
} else {
fromDateNumber = MCRCalendar.MIN_JULIAN_DAY_NUMBER;
}
if (StringUtils.isNotBlank(values[1])) {
final String toDate = values[1];
final Calendar toDateEnd = MCRCalendar.getHistoryDateAsCalendar(toDate, true, calType);
toDateNumber = MCRCalendar.getJulianDayNumber(toDateEnd);
} else {
toDateNumber = MCRCalendar.MAX_JULIAN_DAY_NUMBER;
}
final String query = String.format("%s:[%s TO %s] AND %s:[%s TO %s]",
fromDateField, fromDateNumber, toDateNumber,
toDateField, fromDateNumber, toDateNumber);
solrParams.add("fq", query);
break;
}
default: {
throw new MCRException("Unsupported type " + type);
}
}
solrParams.add(solrParamName, solrParamValue);
});
// if we have only filter queries add the necessary *:* query
......@@ -196,14 +265,15 @@ public class SearchFormBuilderServlet extends MCRServlet {
switch (type) {
case Classification: {
return List.of(
new OperatorDTO(MCRTranslation.translate(I18N_CLASS_IS), "+"),
new OperatorDTO(MCRTranslation.translate(I18N_CLASS_NOT), "-"));
new OperatorDTO(Operator.ClassIs.getTranslation(), Operator.ClassIs.getOperator()),
new OperatorDTO(Operator.ClassIsNot.getTranslation(), Operator.ClassIsNot.getOperator()));
}
case Text: {
return List.of(
new OperatorDTO(MCRTranslation.translate(I18N_TEXT_STARTS_WITH), "startsWith"),
new OperatorDTO(MCRTranslation.translate(I18N_TEXT_CONTAINS), "contains"),
new OperatorDTO(MCRTranslation.translate(I18N_TEXT_IS), "is"));
new OperatorDTO(Operator.TextStartsWith.getTranslation(),
Operator.TextStartsWith.getOperator()),
new OperatorDTO(Operator.TextContains.getTranslation(), Operator.TextContains.getOperator()),
new OperatorDTO(Operator.TextIs.getTranslation(), Operator.TextIs.getOperator()));
}
case Bool: {
return Collections.emptyList();
......@@ -234,19 +304,15 @@ public class SearchFormBuilderServlet extends MCRServlet {
}
final String[] values = StringUtils.split(prop, ':');
final String field = StringUtils.replace(values[1], ";", ",");
final String type = values[0];
final String displayName;
final String field;
final String type;
final String classificationId;
if (3 == values.length) {
type = values[0];
field = values[1];
displayName = values[2];
classificationId = null;
} else if (4 == values.length) {
type = values[0];
field = values[1];
classificationId = values[2];
displayName = values[3];
} else {
......@@ -268,11 +334,12 @@ public class SearchFormBuilderServlet extends MCRServlet {
public List<CategoryDTO> buildClassificationOptions(String classId) {
final List<MCRCategory> categories =
MCRCategoryDAOFactory.getInstance().getChildren(MCRCategoryID.fromString(classId));
final List<CategoryDTO> categoryDTOS = new ArrayList<>(categories.size());
final List<CategoryDTO> categoryDTOS = new ArrayList<>(categories.size() + 1);
final MCRCategory parent = MCRCategoryDAOFactory.getInstance()
.getCategory(MCRCategoryID.fromString(classId), -1);
final Element items = MCRCategoryTransformer.getEditorItems(parent, true, false, true);
categoryDTOS.add(CategoryDTO.blank());
for (Element item : items.getChildren()) {
final String categoryId = item.getAttributeValue("value");
final String displayName =
......@@ -287,21 +354,39 @@ public class SearchFormBuilderServlet extends MCRServlet {
return categoryDTOS;
}
/**
* Returns a list of supported calendars which can be selected in date queries.
*
* @return a list of supported calendars which can be selected in date queries
*/
public List<CalendarDTO> buildCalendarOptions() {
final String[] calendars =
StringUtils.split(MCRConfiguration2.getString(CONFIG_CALENDARS).orElse(MCRCalendar.TAG_GREGORIAN), ",");
final List<CalendarDTO> result = new ArrayList<>(calendars.length);
for (String calendar : calendars) {
result.add(new CalendarDTO(MCRTranslation.translate(I18N_PREFIX_CALENDAR + calendar), calendar));
}
return result;
}
public String previewResultCount(Map<String, String[]> params) {
LOGGER.debug("Parsing incoming request params to Solr params: {}", params);
final ModifiableSolrParams solrParams = toSolrParams(params);
solrParams.set("rows", 0);
LOGGER.debug("Resulting Solr params: {}", solrParams);
long results = 0;
try {
final QueryResponse queryResponse = MCRSolrClientFactory.getMainSolrClient().query(solrParams);
LOGGER.debug("Solr response: {}", queryResponse.getResults());
results = queryResponse.getResults().getNumFound();
} catch (SolrServerException | IOException e) {
e.printStackTrace();
}
System.out.println("preview: " + results);
System.out.println("return: " + MCRTranslation.translate(I18N_RESULT_COUNT, results));
return MCRTranslation.translate(I18N_RESULT_COUNT, results);
}
......@@ -329,6 +414,18 @@ public class SearchFormBuilderServlet extends MCRServlet {
", categoryId='" + categoryId + '\'' +
'}';
}
/**
* Returns a blank category with the text please select.
*
* @return a blank category with the text please select
*/
public static CategoryDTO blank() {
final String displayName = MCRTranslation.translate("module.dptbase.common.editor.choose");
final String categoryId = "";
return new CategoryDTO(displayName, categoryId);
}
}
public static class FieldDTO {
......@@ -396,10 +493,29 @@ public class SearchFormBuilderServlet extends MCRServlet {
}
}
public static class CalendarDTO {
private final String displayName;
private final String value;
public CalendarDTO(String displayName, String value) {
this.displayName = displayName;
this.value = value;
}
public String getDisplayName() {
return displayName;
}
public String getValue() {
return value;
}
}
public enum Action {
BuildFieldList("buildFieldList"),
BuildClassificationOptions("buildClassificationOptions"),
BuildOperatorOptions("buildOperatorOptions"),
BuildCalendarOptions("buildCalendarOptions"),
Query("query"),
Preview("preview");
......@@ -423,7 +539,8 @@ public class SearchFormBuilderServlet extends MCRServlet {
public enum QueryType {
Classification("classification"),
Text("text"),
Bool("bool");
Bool("bool"),
DateRange("daterange");
private final String queryType;
......@@ -441,4 +558,35 @@ public class SearchFormBuilderServlet extends MCRServlet {
.findFirst().orElseThrow();
}
}
public enum Operator {
ClassIs("+", "module.mymss.search.flexible.classification.is"),
ClassIsNot("-", "module.mymss.search.flexible.classification.not"),
TextIs("is", "module.mymss.search.flexible.text.is"),
TextContains("contains", "module.mymss.search.flexible.text.contains"),
TextStartsWith("startsWith", "module.mymss.search.flexible.text.startswith"),
Between("between", null);
private final String operator;
private final String i18nKey;
Operator(String operator, String i18nKey) {
this.operator = operator;
this.i18nKey = i18nKey;
}
public String getOperator() {
return operator;
}
public String getTranslation() {
return MCRTranslation.translate(i18nKey);
}
public static Operator get(String value) {
return Arrays.stream(values())
.filter(operator -> operator.getOperator().equals(value))
.findFirst().orElseThrow();
}
}
}
......@@ -53,53 +53,13 @@ public class MyMssSearchServlet extends MCRQLSearchServletLpz {
public void doGetPost(MCRServletJob job) throws IOException {
final HttpServletRequest request = job.getRequest();
final HttpServletResponse response = job.getResponse();
final String objectType = getReqParameter(request, "objectType", "");
final String objectTypeAndField = getReqParameter(request, "objectType", "");
final String objectType = StringUtils.substringBefore(objectTypeAndField, ";");
final String fields = StringUtils.substringAfter(objectTypeAndField, ";");
final String search = getReqParameter(request, "search", "*");
final String additionalReturnFields = getReqParameter(request, "additionalReturnFields", "");
final Element conditions = new Element("conditions");
conditions.setAttribute("format", "xml");
final Element and = new Element("boolean");
and.setAttribute("operator", "and");
conditions.addContent(and);
// add the object type and suppressed status condition
if (StringUtils.isNotEmpty(objectType)) {
final Element conditionObjectType = new Element("condition");
conditionObjectType.setAttribute("field", "objectType");
conditionObjectType.setAttribute("operator", "=");
conditionObjectType.setAttribute("value", StringUtils.substringBefore(objectType, "_"));
and.addContent(conditionObjectType);
and.addContent(getStatusCondition(Collections.singleton(objectType)));
} else {
// if object type not set: iterate over all active object types and prohibit suppressed status
Map<String, String> props = MCRConfiguration2.getSubPropertiesMap("MCR.Metadata.Type.");
Map<String, String> activeTypes = props.entrySet().stream()
.filter(type -> type.getValue().equals("true"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
and.addContent(getStatusCondition(activeTypes.keySet()));
}
// add the query condition
if (StringUtils.equals(objectType, SEARCH_INVENT)) {
final Element conditionSearch = new Element("condition");
conditionSearch.setAttribute("field", "mymss_mss01,mymss_mss01text,mymss_mss01shelfmark");
conditionSearch.setAttribute("operator", "contains");
conditionSearch.setAttribute("value", search);
and.addContent(conditionSearch);
} else {
final Element conditionSearch = new Element("condition");
conditionSearch.setAttribute("field", "allMeta,mymss_allmeta_diacr");
conditionSearch.setAttribute("operator", "contains");
conditionSearch.setAttribute("value", search);
and.addContent(conditionSearch);
}
final Element returnFields = new Element("returnFields");
if (StringUtils.isNotBlank(additionalReturnFields)) {
returnFields.setText(DEFAULT_RETURN_FIELDS + "," + additionalReturnFields);
......@@ -107,24 +67,18 @@ public class MyMssSearchServlet extends MCRQLSearchServletLpz {
returnFields.setText(DEFAULT_RETURN_FIELDS);
}
final String searchObjectType;
if (StringUtils.equals(objectType, SEARCH_INVENT)) {
searchObjectType = "manuscript";
} else {
searchObjectType = objectType;
}
final Element sortType = new Element("field");
sortType.setAttribute("name", "objectType");
sortType.setAttribute("order", "ascending");
final Element sortContent = new Element("field");
sortContent.setAttribute("name", defaultSortField(searchObjectType, getSession(request).getCurrentLanguage()));
sortContent.setAttribute("name", defaultSortField(objectType, getSession(request).getCurrentLanguage()));
sortContent.setAttribute("order", "ascending");
final Element sort = new Element("sortBy");
sort.addContent(sortType);
sort.addContent(sortContent);
final Element conditions = createConditions(objectType, fields, search);
final Element query = new Element("query");
query.setAttribute("maxResults", "0");
query.setAttribute("numPerPage", "50");
......@@ -139,6 +93,50 @@ public class MyMssSearchServlet extends MCRQLSearchServletLpz {
sendRedirect(request, response, MCRQLSearchUtils.buildFormQuery(queryDoc.getRootElement()), queryDoc);
}
public Element createConditions(final String objectType, final String fields, final String search) {
final String searchFields;
if (StringUtils.isBlank(fields)) {
searchFields = "allMeta,mymss_allmeta_diacr";
} else {
searchFields = fields;
}
final Element conditions = new Element("conditions");
conditions.setAttribute("format", "xml");
final Element and = new Element("boolean");
and.setAttribute("operator", "and");
conditions.addContent(and);
// add the object type and suppressed status condition
if (StringUtils.isNotEmpty(objectType)) {
final Element conditionObjectType = new Element("condition");
conditionObjectType.setAttribute("field", "objectType");
conditionObjectType.setAttribute("operator", "=");
conditionObjectType.setAttribute("value", objectType);
and.addContent(conditionObjectType);
and.addContent(getStatusCondition(Collections.singleton(objectType)));
} else {
// if object type not set: iterate over all active object types and prohibit suppressed status
Map<String, String> props = MCRConfiguration2.getSubPropertiesMap("MCR.Metadata.Type.");
Map<String, String> activeTypes = props.entrySet().stream()
.filter(type -> type.getValue().equals("true"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
and.addContent(getStatusCondition(activeTypes.keySet()));
}
// add the query condition
final Element conditionSearch = new Element("condition");
conditionSearch.setAttribute("field", searchFields);
conditionSearch.setAttribute("operator", "contains");
conditionSearch.setAttribute("value", search);
and.addContent(conditionSearch);
return conditions;
}
/**
* Adds the status condition to prohibit returning objects with suppressed status.
*
......@@ -155,7 +153,7 @@ public class MyMssSearchServlet extends MCRQLSearchServletLpz {
*
* @param objectTypes the object types for which status conditions are to be set
*/
private Element getStatusCondition(Collection<String> objectTypes) {
public Element getStatusCondition(Collection<String> objectTypes) {
final Element not = new Element("boolean");
not.setAttribute("operator", "not");
final Element or = new Element("boolean");
......@@ -181,7 +179,7 @@ public class MyMssSearchServlet extends MCRQLSearchServletLpz {
return not;
}
private String defaultSortField(String objectType, String lang) {
public String defaultSortField(String objectType, String lang) {
String sortField = MCRConfiguration2.getString("MCR.IndexBrowser." + objectType + ".FieldsToSort")
.orElse("id");
......
......@@ -40,7 +40,7 @@
<servlet-name>SearchFormBuilderServlet</servlet-name>
<servlet-class>org.mycore.service.SearchFormBuilderServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>SearchFormBuilderServlet</servlet-name>
<url-pattern>/servlets/SearchFormBuilderServlet</url-pattern>
......