Remove BeanShell scripts and refactor custom indexing code. (#883)

import/index_java/src/org/vufind/index/ConfigManager.java

+package org.vufind.index;
+/**
+ * VuFind configuration manager
+ *
+ * Copyright (C) Villanova University 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+import java.io.File;
+import java.io.FileReader;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import org.solrmarc.index.indexer.ValueIndexerFactory;
+import org.solrmarc.tools.PropertyUtils;
+import org.solrmarc.tools.SolrMarcIndexerException;
+import org.ini4j.Ini;
+import org.apache.log4j.Logger;
+
+/**
+ * VuFind configuration manager
+ */
+public class ConfigManager
+{
+    // Initialize logging category
+    static Logger logger = Logger.getLogger(ConfigManager.class.getName());
+    private static ConcurrentHashMap<String, Ini> configCache = new ConcurrentHashMap<String, Ini>();
+    private Properties vuFindConfigs = null;
+    private static ThreadLocal<ConfigManager> managerCache = 
+        new ThreadLocal<ConfigManager>()
+        {
+            @Override
+            protected ConfigManager initialValue()
+            {
+                return new ConfigManager();
+            }
+        };
+
+    public ConfigManager()
+    {
+        try {
+            vuFindConfigs = PropertyUtils.loadProperties(ValueIndexerFactory.instance().getHomeDirs(), "vufind.properties");
+        } catch (IllegalArgumentException e) {
+            // If the properties load failed, don't worry about it -- we'll use defaults.
+        }
+    }
+
+    public static ConfigManager instance()
+    {
+        return managerCache.get();
+    }
+
+    /**
+     * Given the base name of a configuration file, locate the full path.
+     * @param filename base name of a configuration file
+     */
+    private File findConfigFile(String filename)
+    {
+        // Find VuFind's home directory in the environment; if it's not available,
+        // try using a relative path on the assumption that we are currently in
+        // VuFind's import subdirectory:
+        String vufindHome = System.getenv("VUFIND_HOME");
+        if (vufindHome == null) {
+            vufindHome = "..";
+        }
+
+        // Check for VuFind 2.0's local directory environment variable:
+        String vufindLocal = System.getenv("VUFIND_LOCAL_DIR");
+
+        // Get the relative VuFind path from the properties file, defaulting to
+        // the 2.0-style config/vufind if necessary.
+        String relativeConfigPath = PropertyUtils.getProperty(
+            vuFindConfigs, "vufind.config.relative_path", "config/vufind"
+        );
+
+        // Try several different locations for the file -- VuFind 2 local dir,
+        // VuFind 2 base dir, VuFind 1 base dir.
+        File file;
+        if (vufindLocal != null) {
+            file = new File(vufindLocal + "/" + relativeConfigPath + "/" + filename);
+            if (file.exists()) {
+                return file;
+            }
+        }
+        file = new File(vufindHome + "/" + relativeConfigPath + "/" + filename);
+        if (file.exists()) {
+            return file;
+        }
+        file = new File(vufindHome + "/web/conf/" + filename);
+        return file;
+    }
+
+    /**
+     * Sanitize a VuFind configuration setting.
+     * @param str configuration setting
+     */
+    private String sanitizeConfigSetting(String str)
+    {
+        // Drop comments if necessary:
+        int pos = str.indexOf(';');
+        if (pos >= 0) {
+            str = str.substring(0, pos).trim();
+        }
+
+        // Strip wrapping quotes if necessary (the ini reader won't do this for us):
+        if (str.startsWith("\"")) {
+            str = str.substring(1, str.length());
+        }
+        if (str.endsWith("\"")) {
+            str = str.substring(0, str.length() - 1);
+        }
+        return str;
+    }
+
+    /**
+     * Load an ini file.
+     * @param filename name of {@code .ini} file
+     */
+    public Ini loadConfigFile(String filename)
+    {
+        // Retrieve the file if it is not already cached.
+        if (!configCache.containsKey(filename)) {
+            Ini ini = new Ini();
+            try {
+                ini.load(new FileReader(findConfigFile(filename)));
+                configCache.putIfAbsent(filename, ini);
+            } catch (Throwable e) {
+                dieWithError("Unable to access " + filename);
+            }
+        }
+        return configCache.get(filename);
+    }
+
+    /**
+     * Get a section from a VuFind configuration file.
+     * @param filename configuration file name
+     * @param section section name within the file
+     */
+    public Map<String, String> getConfigSection(String filename, String section)
+    {
+        // Grab the ini file.
+        Ini ini = loadConfigFile(filename);
+        Map<String, String> retVal = ini.get(section);
+
+        String parent = ini.get("Parent_Config", "path");
+        while (parent != null) {
+            Ini parentIni = loadConfigFile(parent);
+            Map<String, String> parentSection = parentIni.get(section);
+            for (String key : parentSection.keySet()) {
+                if (!retVal.containsKey(key)) {
+                    retVal.put(key, parentSection.get(key));
+                }
+            }
+            parent = parentIni.get("Parent_Config", "path");
+        }
+
+        // Check to see if we need to worry about an override file:
+        String override = ini.get("Extra_Config", "local_overrides");
+        if (override != null) {
+            Map<String, String> overrideSection = loadConfigFile(override).get(section);
+            for (String key : overrideSection.keySet()) {
+                retVal.put(key, overrideSection.get(key));
+            }
+        }
+        return retVal;
+    }
+
+    /**
+     * Get a setting from a VuFind configuration file.
+     * @param filename configuration file name
+     * @param section section name within the file
+     * @param setting setting name within the section
+     */
+    public String getConfigSetting(String filename, String section, String setting)
+    {
+        String retVal = null;
+
+        // Grab the ini file.
+        Ini ini = loadConfigFile(filename);
+
+        // Check to see if we need to worry about an override file:
+        String override = ini.get("Extra_Config", "local_overrides");
+        if (override != null) {
+            Ini overrideIni = loadConfigFile(override);
+            retVal = overrideIni.get(section, setting);
+            if (retVal != null) {
+                return sanitizeConfigSetting(retVal);
+            }
+        }
+
+        // Try to find the requested setting:
+        retVal = ini.get(section, setting);
+
+        //  No setting?  Check for a parent configuration:
+        while (retVal == null) {
+            String parent = ini.get("Parent_Config", "path");
+            if (parent !=  null) {
+                try {
+                    ini.load(new FileReader(new File(parent)));
+                } catch (Throwable e) {
+                    dieWithError("Unable to access " + parent);
+                }
+                retVal = ini.get(section, setting);
+            } else {
+                break;
+            }
+        }
+
+        // Return the processed setting:
+        return retVal == null ? null : sanitizeConfigSetting(retVal);
+    }
+
+    /**
+     * Log an error message and throw a fatal exception.
+     * @param msg message to log
+     */
+    private void dieWithError(String msg)
+    {
+        logger.error(msg);
+        throw new SolrMarcIndexerException(SolrMarcIndexerException.EXIT, msg);
+    }
+}
\ No newline at end of file

import/index_java/src/org/vufind/index/DatabaseManager.java

+package org.vufind.index;
+/**
+ * Database manager.
+ *
+ * Copyright (C) Villanova University 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+import org.apache.log4j.Logger;
+import org.solrmarc.tools.SolrMarcIndexerException;
+import java.sql.*;
+
+/**
+ * Database manager.
+ */
+public class DatabaseManager
+{
+    // Initialize logging category
+    static Logger logger = Logger.getLogger(DatabaseManager.class.getName());
+
+    // Initialize VuFind database connection (null until explicitly activated)
+    private Connection vufindDatabase = null;
+
+    // Shutdown flag:
+    private boolean shuttingDown = false;
+
+    private static ThreadLocal<DatabaseManager> managerCache = 
+        new ThreadLocal<DatabaseManager>()
+        {
+            @Override
+            protected DatabaseManager initialValue()
+            {
+                return new DatabaseManager();
+            }
+        };
+
+    public static DatabaseManager instance()
+    {
+        return managerCache.get();
+    }
+
+    /**
+     * Log an error message and throw a fatal exception.
+     * @param msg message to log
+     */
+    private void dieWithError(String msg)
+    {
+        logger.error(msg);
+        throw new SolrMarcIndexerException(SolrMarcIndexerException.EXIT, msg);
+    }
+
+    /**
+     * Connect to the VuFind database if we do not already have a connection.
+     */
+    private void connectToDatabase()
+    {
+        // Already connected?  Do nothing further!
+        if (vufindDatabase != null) {
+            return;
+        }
+
+        String dsn = ConfigManager.instance().getConfigSetting("config.ini", "Database", "database");
+
+        try {
+            // Parse key settings from the PHP-style DSN:
+            String username = "";
+            String password = "";
+            String classname = "invalid";
+            String prefix = "invalid";
+            if (dsn.substring(0, 8).equals("mysql://")) {
+                classname = "com.mysql.jdbc.Driver";
+                prefix = "mysql";
+            } else if (dsn.substring(0, 8).equals("pgsql://")) {
+                classname = "org.postgresql.Driver";
+                prefix = "postgresql";
+            }
+
+            Class.forName(classname).newInstance();
+            String[] parts = dsn.split("://");
+            if (parts.length > 1) {
+                parts = parts[1].split("@");
+                if (parts.length > 1) {
+                    dsn = prefix + "://" + parts[1];
+                    parts = parts[0].split(":");
+                    username = parts[0];
+                    if (parts.length > 1) {
+                        password = parts[1];
+                    }
+                }
+            }
+
+            // Connect to the database:
+            vufindDatabase = DriverManager.getConnection("jdbc:" + dsn, username, password);
+        } catch (Throwable e) {
+            dieWithError("Unable to connect to VuFind database");
+        }
+
+        Runtime.getRuntime().addShutdownHook(new DatabaseManagerShutdownThread(this));
+    }
+
+    private void disconnectFromDatabase()
+    {
+        if (vufindDatabase != null) {
+            try {
+                vufindDatabase.close();
+            } catch (SQLException e) {
+                System.err.println("Unable to disconnect from VuFind database");
+                logger.error("Unable to disconnect from VuFind database");
+            }
+        }
+    }
+
+    public void shutdown()
+    {
+        disconnectFromDatabase();
+        shuttingDown = true;
+    }
+
+    public Connection getConnection()
+    {
+        connectToDatabase();
+        return vufindDatabase;
+    }
+
+    public boolean isShuttingDown()
+    {
+        return shuttingDown;
+    }
+
+    class DatabaseManagerShutdownThread extends Thread
+    {
+        private DatabaseManager manager;
+
+        public DatabaseManagerShutdownThread(DatabaseManager m)
+        {
+            manager = m;
+        }
+
+        public void run()
+        {
+            manager.shutdown();
+        }
+    }
+}
\ No newline at end of file

import/index_java/src/org/vufind/index/DateTools.java

+package org.vufind.index;
+/**
+ * Date indexing routines.
+ *
+ * Copyright (C) Villanova University 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+import org.marc4j.marc.Record;
+import org.marc4j.marc.VariableField;
+import org.marc4j.marc.DataField;
+import org.marc4j.marc.Subfield;
+import org.solrmarc.tools.DataUtil;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Date indexing routines.
+ */
+public class DateTools
+{
+    /**
+     * Get all available dates from the record.
+     *
+     * @param  record MARC record
+     * @return set of dates
+     */
+    public Set<String> getDates(final Record record) {
+        Set<String> dates = new LinkedHashSet<String>();
+
+        // First check old-style 260c date:
+        List<VariableField> list260 = record.getVariableFields("260");
+        for (VariableField vf : list260) {
+            DataField df = (DataField) vf;
+            List<Subfield> currentDates = df.getSubfields('c');
+            for (Subfield sf : currentDates) {
+                String currentDateStr = DataUtil.cleanDate(sf.getData());
+                if (currentDateStr != null) dates.add(currentDateStr);
+            }
+        }
+
+        // Now track down relevant RDA-style 264c dates; we only care about
+        // copyright and publication dates (and ignore copyright dates if
+        // publication dates are present).
+        Set<String> pubDates = new LinkedHashSet<String>();
+        Set<String> copyDates = new LinkedHashSet<String>();
+        List<VariableField> list264 = record.getVariableFields("264");
+        for (VariableField vf : list264) {
+            DataField df = (DataField) vf;
+            List<Subfield> currentDates = df.getSubfields('c');
+            for (Subfield sf : currentDates) {
+                String currentDateStr = DataUtil.cleanDate(sf.getData());
+                char ind2 = df.getIndicator2();
+                switch (ind2)
+                {
+                    case '1':
+                        if (currentDateStr != null) pubDates.add(currentDateStr);
+                        break;
+                    case '4':
+                        if (currentDateStr != null) copyDates.add(currentDateStr);
+                        break;
+                }
+            }
+        }
+        if (pubDates.size() > 0) {
+            dates.addAll(pubDates);
+        } else if (copyDates.size() > 0) {
+            dates.addAll(copyDates);
+        }
+
+        return dates;
+    }
+
+    /**
+     * Get the earliest publication date from the record.
+     *
+     * @param  record MARC record
+     * @return earliest date
+     */
+    public String getFirstDate(final Record record) {
+        String result = null;
+        Set<String> dates = getDates(record);
+        for(String current: dates) {
+            if (result == null || Integer.parseInt(current) < Integer.parseInt(result)) {
+                result = current;
+            }
+        }
+        return result;
+    }
+}
\ No newline at end of file

import/index_java/src/org/vufind/index/FormatCalculator.java

+package org.vufind.index;
+/**
+ * Format determination logic.
+ *
+ * Copyright (C) Villanova University 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+import org.marc4j.marc.Record;
+import org.marc4j.marc.ControlField;
+import org.marc4j.marc.DataField;
+import org.marc4j.marc.VariableField;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Format determination logic.
+ */
+public class FormatCalculator
+{
+    /**
+     * Determine Record Format(s)
+     *
+     * @param  record MARC record
+     * @return set of record formats
+     */
+    public Set<String> getFormat(final Record record){
+        Set<String> result = new LinkedHashSet<String>();
+        String leader = record.getLeader().toString();
+        char leaderBit;
+        ControlField fixedField = (ControlField) record.getVariableField("008");
+        DataField title = (DataField) record.getVariableField("245");
+        String formatString;
+        char formatCode = ' ';
+        char formatCode2 = ' ';
+        char formatCode4 = ' ';
+
+        // check if there's an h in the 245
+        if (title != null) {
+            if (title.getSubfield('h') != null){
+                if (title.getSubfield('h').getData().toLowerCase().contains("[electronic resource]")) {
+                    result.add("Electronic");
+                    return result;
+                }
+            }
+        }
+
+        // check the 007 - this is a repeating field
+        List<VariableField> fields = record.getVariableFields("007");
+        Iterator<VariableField> fieldsIter = fields.iterator();
+        if (fields != null) {
+            // TODO: update loop to for(:) syntax, but problem with type casting.
+            ControlField formatField;
+            while(fieldsIter.hasNext()) {
+                formatField = (ControlField) fieldsIter.next();
+                formatString = formatField.getData().toUpperCase();
+                formatCode = formatString.length() > 0 ? formatString.charAt(0) : ' ';
+                formatCode2 = formatString.length() > 1 ? formatString.charAt(1) : ' ';
+                formatCode4 = formatString.length() > 4 ? formatString.charAt(4) : ' ';
+                switch (formatCode) {
+                    case 'A':
+                        switch(formatCode2) {
+                            case 'D':
+                                result.add("Atlas");
+                                break;
+                            default:
+                                result.add("Map");
+                                break;
+                        }
+                        break;
+                    case 'C':
+                        switch(formatCode2) {
+                            case 'A':
+                                result.add("TapeCartridge");
+                                break;
+                            case 'B':
+                                result.add("ChipCartridge");
+                                break;
+                            case 'C':
+                                result.add("DiscCartridge");
+                                break;
+                            case 'F':
+                                result.add("TapeCassette");
+                                break;
+                            case 'H':
+                                result.add("TapeReel");
+                                break;
+                            case 'J':
+                                result.add("FloppyDisk");
+                                break;
+                            case 'M':
+                            case 'O':
+                                result.add("CDROM");
+                                break;
+                            case 'R':
+                                // Do not return - this will cause anything with an
+                                // 856 field to be labeled as "Electronic"
+                                break;
+                            default:
+                                result.add("Software");
+                                break;
+                        }
+                        break;
+                    case 'D':
+                        result.add("Globe");
+                        break;
+                    case 'F':
+                        result.add("Braille");
+                        break;
+                    case 'G':
+                        switch(formatCode2) {
+                            case 'C':
+                            case 'D':
+                                result.add("Filmstrip");
+                                break;
+                            case 'T':
+                                result.add("Transparency");
+                                break;
+                            default:
+                                result.add("Slide");
+                                break;
+                        }
+                        break;
+                    case 'H':
+                        result.add("Microfilm");
+                        break;
+                    case 'K':
+                        switch(formatCode2) {
+                            case 'C':
+                                result.add("Collage");
+                                break;
+                            case 'D':
+                                result.add("Drawing");
+                                break;
+                            case 'E':
+                                result.add("Painting");
+                                break;
+                            case 'F':
+                                result.add("Print");
+                                break;
+                            case 'G':
+                                result.add("Photonegative");
+                                break;
+                            case 'J':
+                                result.add("Print");
+                                break;
+                            case 'L':
+                                result.add("Drawing");
+                                break;
+                            case 'O':
+                                result.add("FlashCard");
+                                break;
+                            case 'N':
+                                result.add("Chart");
+                                break;
+                            default:
+                                result.add("Photo");
+                                break;
+                        }
+                        break;
+                    case 'M':
+                        switch(formatCode2) {
+                            case 'F':
+                                result.add("VideoCassette");
+                                break;
+                            case 'R':
+                                result.add("Filmstrip");
+                                break;
+                            default:
+                                result.add("MotionPicture");
+                                break;
+                        }
+                        break;
+                    case 'O':
+                        result.add("Kit");
+                        break;
+                    case 'Q':
+                        result.add("MusicalScore");
+                        break;
+                    case 'R':
+                        result.add("SensorImage");
+                        break;
+                    case 'S':
+                        switch(formatCode2) {
+                            case 'D':
+                                result.add("SoundDisc");
+                                break;
+                            case 'S':
+                                result.add("SoundCassette");
+                                break;
+                            default:
+                                result.add("SoundRecording");
+                                break;
+                        }
+                        break;
+                    case 'V':
+                        switch(formatCode2) {
+                            case 'C':
+                                result.add("VideoCartridge");
+                                break;
+                            case 'D':
+                                switch(formatCode4) {
+                                    case 'S':
+                                        result.add("BRDisc");
+                                        break;
+                                    case 'V':
+                                    default:
+                                        result.add("VideoDisc");
+                                        break;
+                                }
+                                break;
+                            case 'F':
+                                result.add("VideoCassette");
+                                break;
+                            case 'R':
+                                result.add("VideoReel");
+                                break;
+                            default:
+                                result.add("Video");
+                                break;
+                        }
+                        break;
+                }
+            }
+            if (!result.isEmpty()) {
+                return result;
+            }
+        }
+
+        // check the Leader at position 6
+        leaderBit = leader.charAt(6);
+        switch (Character.toUpperCase(leaderBit)) {
+            case 'C':
+            case 'D':
+                result.add("MusicalScore");
+                break;
+            case 'E':
+            case 'F':
+                result.add("Map");
+                break;
+            case 'G':
+                result.add("Slide");
+                break;
+            case 'I':
+                result.add("SoundRecording");
+                break;
+            case 'J':
+                result.add("MusicRecording");
+                break;
+            case 'K':
+                result.add("Photo");
+                break;
+            case 'M':
+                result.add("Electronic");
+                break;
+            case 'O':
+            case 'P':
+                result.add("Kit");
+                break;
+            case 'R':
+                result.add("PhysicalObject");
+                break;
+            case 'T':
+                result.add("Manuscript");
+                break;
+        }
+        if (!result.isEmpty()) {
+            return result;
+        }
+
+        // check the Leader at position 7
+        leaderBit = leader.charAt(7);
+        switch (Character.toUpperCase(leaderBit)) {
+            // Monograph
+            case 'M':
+                if (formatCode == 'C') {
+                    result.add("eBook");
+                } else {
+                    result.add("Book");
+                }
+                break;
+            // Component parts
+            case 'A':
+                result.add("BookComponentPart");
+                break;
+            case 'B':
+                result.add("SerialComponentPart");
+                break;
+            // Serial
+            case 'S':
+                // Look in 008 to determine what type of Continuing Resource
+                formatCode = fixedField.getData().toUpperCase().charAt(21);
+                switch (formatCode) {
+                    case 'N':
+                        result.add("Newspaper");
+                        break;
+                    case 'P':
+                        result.add("Journal");
+                        break;
+                    default:
+                        result.add("Serial");
+                        break;
+                }
+        }
+
+        // Nothing worked!
+        if (result.isEmpty()) {
+            result.add("Unknown");
+        }
+
+        return result;
+    }
+}
\ No newline at end of file

import/index_java/src/org/vufind/index/IllustrationTools.java

+package org.vufind.index;
+/**
+ * Illustration indexing routines.
+ *
+ * Copyright (C) Villanova University 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+import java.util.Iterator;
+import org.marc4j.marc.Record;
+import org.marc4j.marc.VariableField;
+import org.marc4j.marc.ControlField;
+import org.marc4j.marc.DataField;
+import org.marc4j.marc.Subfield;
+import java.util.List;
+
+/**
+ * Illustration indexing routines.
+ */
+public class IllustrationTools
+{
+    /**
+     * Determine if a record is illustrated.
+     *
+     * @param record  current MARC record
+     * @return "Illustrated" or "Not Illustrated"
+     */
+    public String isIllustrated(Record record) {
+        String leader = record.getLeader().toString();
+
+        // Does the leader indicate this is a "language material" that might have extra
+        // illustration details in the fixed fields?
+        if (leader.charAt(6) == 'a') {
+            String currentCode = "";         // for use in loops below
+
+            // List of 008/18-21 codes that indicate illustrations:
+            String illusCodes = "abcdefghijklmop";
+
+            // Check the illustration characters of the 008:
+            ControlField fixedField = (ControlField) record.getVariableField("008");
+            if (fixedField != null) {
+                String fixedFieldText = fixedField.getData().toLowerCase();
+                for (int i = 18; i <= 21; i++) {
+                    if (i < fixedFieldText.length()) {
+                        currentCode = fixedFieldText.substring(i, i + 1);
+                        if (illusCodes.contains(currentCode)) {
+                            return "Illustrated";
+                        }
+                    }
+                }
+            }
+
+            // Now check if any 006 fields apply:
+            List<VariableField> fields = record.getVariableFields("006");
+            Iterator<VariableField> fieldsIter = fields.iterator();
+            if (fields != null) {
+                while(fieldsIter.hasNext()) {
+                    fixedField = (ControlField) fieldsIter.next();
+                    String fixedFieldText = fixedField.getData().toLowerCase();
+                    for (int i = 1; i <= 4; i++) {
+                         if (i < fixedFieldText.length()) {
+                            currentCode = fixedFieldText.substring(i, i + 1);
+                            if (illusCodes.contains(currentCode)) {
+                                return "Illustrated";
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // Now check for interesting strings in 300 subfield b:
+        List<VariableField> fields = record.getVariableFields("300");
+        Iterator<VariableField> fieldsIter = fields.iterator();
+        if (fields != null) {
+            DataField physical;
+            while(fieldsIter.hasNext()) {
+                physical = (DataField) fieldsIter.next();
+                List<Subfield> subfields = physical.getSubfields('b');
+                for (Subfield sf: subfields) {
+                    String desc = sf.getData().toLowerCase();
+                    if (desc.contains("ill.") || desc.contains("illus.")) {
+                        return "Illustrated";
+                    }
+                }
+            }
+        }
+
+        // If we made it this far, we found no sign of illustrations:
+        return "Not Illustrated";
+    }
+}
\ No newline at end of file

import/index_java/src/org/vufind/index/PunctuationTools.java

+package org.vufind.index;
+/**
+ * Punctuation indexing routines.
+ *
+ * Copyright (C) Villanova University 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+import org.marc4j.marc.Record;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import org.solrmarc.index.SolrIndexer;
+
+/**
+ * Punctuation indexing routines.
+ */
+public class PunctuationTools
+{
+    /**
+     * Normalize trailing punctuation. This mimics the functionality built into VuFind's
+     * textFacet field type, so that you can get equivalent values when indexing into
+     * a string field. (Useful for docValues support).
+     *
+     * Can return null
+     *
+     * @param record current MARC record
+     * @param fieldSpec which MARC fields / subfields need to be analyzed
+     * @return Set containing normalized values
+     */
+    public Set<String> normalizeTrailingPunctuation(Record record, String fieldSpec) {
+        // Initialize our return value:
+        Set<String> result = new LinkedHashSet<String>();
+
+        // Loop through the specified MARC fields:
+        Set<String> input = SolrIndexer.instance().getFieldList(record, fieldSpec);
+        Pattern pattern = Pattern.compile("(?<!\b[A-Z])[.\\s]*$");
+        for (String current: input) {
+            result.add(pattern.matcher(current).replaceAll(""));
+        }
+
+        // If we found no matches, return null; otherwise, return our results:
+        return result.isEmpty() ? null : result;
+    }
+}
\ No newline at end of file