diff --git a/harvest/batch-import-marc-auth.bat b/harvest/batch-import-marc-auth.bat index 1297a145848121cf1539b852bd9fe5b315f70cf1..6686185d6b7ca03b39b9495d7265922d0bfc3526 100644 --- a/harvest/batch-import-marc-auth.bat +++ b/harvest/batch-import-marc-auth.bat @@ -32,6 +32,7 @@ set HARVEST_DIR=%VUFIND_HOME%\harvest :harvestpathfound set BASEPATH_UNDER_HARVEST=1 +set LOGGING=1 set MOVE_DATA=1 rem Save script name for message below (otherwise it may get shifted away) @@ -40,7 +41,9 @@ set SCRIPT_NAME=%0 rem Process switches :switchloop if "%1"=="-d" goto dswitch +if "%1"=="-h" goto helpmessage if "%1"=="-m" goto mswitch +if "%1"=="-z" goto zswitch goto switchloopend :dswitch set BASEPATH_UNDER_HARVEST=0 @@ -50,13 +53,18 @@ goto switchloop set MOVE_DATA=0 shift goto switchloop +:zswitch +set LOGGING=0 +shift +goto switchloop :switchloopend rem Make sure command line parameter was included: if not "!%2!"=="!!" goto paramsokay +:helpmessage echo This script processes a batch of harvested authority records. echo. -echo Usage: %0 [harvest subdirectory] [SolrMarc properties file] +echo Usage: %0 [-dhmz] [harvest subdirectory] [SolrMarc properties file] echo. echo [harvest subdirectory] is a directory name created by the OAI-PMH harvester. echo This script will search the harvest subdirectories of the directories defined @@ -67,7 +75,9 @@ echo. echo Options: echo -d: Use the directory path as-is, do not append it to %HARVEST_DIR%. echo Useful for non-OAI batch loading. +echo -h: Print this message echo -m: Do not move the data files after importing. +echo -z: No logging. goto end :paramsokay @@ -83,6 +93,7 @@ goto end rem Create log/processed directories as needed: if exist %BASEPATH%\log goto logfound +if "%LOGGING%"=="0" goto logfound md %BASEPATH%\log :logfound if exist %BASEPATH%\processed goto processedfound @@ -92,9 +103,14 @@ md %BASEPATH%\processed rem Process all the files in the target directory: for %%a in (%BASEPATH%\*.xml %BASEPATH%\*.mrc) do ( rem Capture solrmarc output to log - call %VUFIND_HOME%\import-marc-auth.bat %%a %2 2> %BASEPATH%\log\%%~nxa.log + if "%LOGGING%"=="0" ( + call %VUFIND_HOME%\import-marc-auth.bat %%a %2 + ) + if "%LOGGING%"=="1" ( + call %VUFIND_HOME%\import-marc-auth.bat %%a %2 2> %BASEPATH%\log\%%~nxa.log + ) if "%MOVE_DATA%"=="1" ( - move %%a %BASEPATH%\processed\ > nul + move %%a %BASEPATH%\processed\ > nul ) ) diff --git a/harvest/batch-import-marc-auth.sh b/harvest/batch-import-marc-auth.sh index 3fa503803ffecde745830d494d1ad0db42a9e8ca..ab2d392b8629ad1cfd089537bb6b963fee6dc169 100755 --- a/harvest/batch-import-marc-auth.sh +++ b/harvest/batch-import-marc-auth.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Make sure VUFIND_HOME is set: if [ -z "$VUFIND_HOME" ] @@ -15,13 +15,38 @@ then fi BASEPATH_UNDER_HARVEST=true +LOGGING=true MOVE_DATA=true -while getopts ":dm" OPT +function usage { +cat <<EOF +This script processes a batch of harvested authority records. + +Usage: $(basename $0) [-dhmz] _harvest_subdirectory_ _SolrMarc_properties_file_ + +_harvest_subdirectory_ is a directory name created by the OAI-PMH harvester. +This script will search the harvest subdirectories of the directories defined +by the VUFIND_LOCAL_DIR or VUFIND_HOME environment variables. + +Example: $(basename $0) lcnaf marc_lcnaf.properties + +Options: +-d: Use the directory path as-is, do not append it to $HARVEST_DIR. + Useful for non-OAI batch loading. +-h: Print this message +-m: Do not move the data files after importing. +-z: No logging. +EOF +} + +while getopts ":dhmz" OPT do case $OPT in d) BASEPATH_UNDER_HARVEST=false;; + h) usage; + exit 0;; m) MOVE_DATA=false;; + z) LOGGING=false;; :) echo "argument to '-$OPTARG' is missing" >&2 exit -1;; @@ -34,20 +59,7 @@ shift $(($OPTIND - 1)) # Make sure command line parameter was included: if [ -z "$2" ] then - echo "This script processes a batch of harvested authority records." - echo "" - echo "Usage: `basename $0` [-d] [-m] [harvest subdirectory] [SolrMarc properties file]" - echo "" - echo "[harvest subdirectory] is a directory name created by the OAI-PMH harvester." - echo "This script will search the harvest subdirectories of the directories defined" - echo "by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables." - echo "" - echo "Example: `basename $0` lcnaf marc_lcnaf.properties" - echo "" - echo "Options:" - echo "-d: Use the directory path as-is, do not append it to $HARVEST_DIR." - echo " Useful for non-OAI batch loading." - echo "-m: Do not move the data files after importing." + usage exit 1 fi @@ -65,22 +77,40 @@ then fi # Create log/processed directories as needed: -if [ ! -d $BASEPATH/log ] +if [ $LOGGING == true ] then - mkdir $BASEPATH/log + if [ ! -d $BASEPATH/log ] + then + mkdir $BASEPATH/log + fi fi if [ ! -d $BASEPATH/processed ] then mkdir $BASEPATH/processed fi +# The log() function can be redefined to suit a variety of logging needs +# Positional parameters must be consistent: +# $1 = name of the file being imported +if [ $LOGGING == false ] +then + function log { + cat - > /dev/null + } +else + function log { + local FILE=$1 + cat -u - > $BASEPATH/log/`basename $FILE`.log + } +fi + # Process all the files in the target directory: for file in $BASEPATH/*.xml $BASEPATH/*.mrc do if [ -f $file ] then # Capture solrmarc output to log - $VUFIND_HOME/import-marc-auth.sh $file $2 2> $BASEPATH/log/`basename $file`.log + $VUFIND_HOME/import-marc-auth.sh $file $2 2> >(log $file) if [ $MOVE_DATA == true ] then mv $file $BASEPATH/processed/`basename $file` diff --git a/harvest/batch-import-marc.bat b/harvest/batch-import-marc.bat index 5603346d13f78d828399c2126a87c2cb553687c6..957eab5cbfe4cc748e2fd13a41e53788213aa3b8 100644 --- a/harvest/batch-import-marc.bat +++ b/harvest/batch-import-marc.bat @@ -32,6 +32,7 @@ set HARVEST_DIR=%VUFIND_HOME%\harvest :harvestpathfound set BASEPATH_UNDER_HARVEST=1 +set LOGGING=1 set MOVE_DATA=1 rem Save script name for message below (otherwise it may get shifted away) @@ -40,7 +41,9 @@ set SCRIPT_NAME=%0 rem Process switches :switchloop if "%1"=="-d" goto dswitch +if "%1"=="-h" goto helpmessage if "%1"=="-m" goto mswitch +if "%1"=="-z" goto zswitch goto switchloopend :dswitch set BASEPATH_UNDER_HARVEST=0 @@ -50,13 +53,18 @@ goto switchloop set MOVE_DATA=0 shift goto switchloop +:zswitch +set LOGGING=0 +shift +goto switchloop :switchloopend rem Make sure command line parameter was included: if not "!%1!"=="!!" goto paramsokay +:helpmessage echo This script processes a batch of harvested MARC records. echo. -echo Usage: %SCRIPT_NAME% [-d] [-m] [harvest subdirectory] +echo Usage: %SCRIPT_NAME% [-dhmz] [harvest subdirectory] echo. echo [harvest subdirectory] is a directory name created by the OAI-PMH harvester. echo This script will search the harvest subdirectories of the directories defined @@ -67,7 +75,9 @@ echo. echo Options: echo -d: Use the directory path as-is, do not append it to %HARVEST_DIR%. echo Useful for non-OAI batch loading. +echo -h: Print this message echo -m: Do not move the data files after importing. +echo -z: No logging. goto end :paramsokay @@ -83,6 +93,7 @@ goto end rem Create log/processed directories as needed: if exist %BASEPATH%\log goto logfound +if "%LOGGING%"=="0" goto logfound md %BASEPATH%\log :logfound if exist %BASEPATH%\processed goto processedfound @@ -92,7 +103,12 @@ md %BASEPATH%\processed rem Process all the files in the target directory: for %%a in (%BASEPATH%\*.xml %BASEPATH%\*.mrc) do ( rem Capture solrmarc output to log - call %VUFIND_HOME%\import-marc.bat %%a 2> %BASEPATH%\log\%%~nxa.log + if "%LOGGING%"=="0" ( + call %VUFIND_HOME%\import-marc.bat %%a + ) + if "%LOGGING%"=="1" ( + call %VUFIND_HOME%\import-marc.bat %%a 2> %BASEPATH%\log\%%~nxa.log + ) if "%MOVE_DATA%"=="1" ( move %%a %BASEPATH%\processed\ > nul ) diff --git a/harvest/batch-import-marc.sh b/harvest/batch-import-marc.sh index 17002f3bf905e916d396beac85a42a8fed333295..ceb53df46ba37e2d078c5365dfcf9c49f86198a8 100755 --- a/harvest/batch-import-marc.sh +++ b/harvest/batch-import-marc.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Make sure VUFIND_HOME is set: if [ -z "$VUFIND_HOME" ] @@ -15,13 +15,38 @@ then fi BASEPATH_UNDER_HARVEST=true +LOGGING=true MOVE_DATA=true -while getopts ":dm" OPT +function usage { +cat <<EOF +This script processes a batch of harvested MARC records. + +Usage: $(basename $0) [-dhmz] _harvest_subdirectory_ + +_harvest_subdirectory_ is a directory name created by the OAI-PMH harvester. +This script will search the harvest subdirectories of the directories defined +by the VUFIND_LOCAL_DIR or VUFIND_HOME environment variables. + +Example: $(basename $0) oai_source + +Options: +-d: Use the directory path as-is, do not append it to $HARVEST_DIR. + Useful for non-OAI batch loading. +-h: Print this message +-m: Do not move the data files after importing. +-z: No logging. +EOF +} + +while getopts ":dhmz" OPT do case $OPT in d) BASEPATH_UNDER_HARVEST=false;; + h) usage; + exit 0;; m) MOVE_DATA=false;; + z) LOGGING=false;; :) echo "argument to '-$OPTARG' is missing" >&2 exit -1;; @@ -34,20 +59,7 @@ shift $(($OPTIND - 1)) # Make sure command line parameter was included: if [ -z "$1" ] then - echo "This script processes a batch of harvested MARC records." - echo "" - echo "Usage: `basename $0` [-d] [-m] [harvest subdirectory]" - echo "" - echo "[harvest subdirectory] is a directory name created by the OAI-PMH harvester." - echo "This script will search the harvest subdirectories of the directories defined" - echo "by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables." - echo "" - echo "Example: `basename $0` oai_source" - echo "" - echo "Options:" - echo "-d: Use the directory path as-is, do not append it to $HARVEST_DIR." - echo " Useful for non-OAI batch loading." - echo "-m: Do not move the data files after importing." + usage exit 1 fi @@ -65,22 +77,40 @@ then fi # Create log/processed directories as needed: -if [ ! -d $BASEPATH/log ] +if [ $LOGGING == true ] then - mkdir $BASEPATH/log + if [ ! -d $BASEPATH/log ] + then + mkdir $BASEPATH/log + fi fi if [ ! -d $BASEPATH/processed ] then mkdir $BASEPATH/processed fi +# The log() function can be redefined to suit a variety of logging needs +# Positional parameters must be consistent: +# $1 = name of the file being imported +if [ $LOGGING == false ] +then + function log { + cat - > /dev/null + } +else + function log { + local FILE=$1 + cat -u - > $BASEPATH/log/`basename $FILE`.log + } +fi + # Process all the files in the target directory: for file in $BASEPATH/*.xml $BASEPATH/*.mrc do if [ -f $file ] then # Capture solrmarc output to log - $VUFIND_HOME/import-marc.sh $file 2> $BASEPATH/log/`basename $file`.log + $VUFIND_HOME/import-marc.sh $file 2> >(log $file) if [ $MOVE_DATA == true ] then mv $file $BASEPATH/processed/`basename $file`