diff --git a/harvest/batch-import-marc-auth.bat b/harvest/batch-import-marc-auth.bat index 6532044cf35ce46dff706df5a8919b26272dbea1..a6f566dce87ad0e3b727e3abc92ff375b688704f 100644 --- a/harvest/batch-import-marc-auth.bat +++ b/harvest/batch-import-marc-auth.bat @@ -25,6 +25,33 @@ echo You need to set the VUFIND_HOME environmental variable before running this goto end :vufindhomefound +rem Find harvest directory for future use: +set HARVEST_DIR=%VUFIND_LOCAL_DIR%\harvest +if exist %HARVEST_DIR% goto harvestpathfound +set HARVEST_DIR=%VUFIND_HOME%\harvest +:harvestpathfound + +set BASEPATH_UNDER_HARVEST=1 +set MOVE_DATA=1 + +rem Save script name for message below (otherwise it may get shifted away) +set SCRIPT_NAME=%0 + +rem Process switches +:switchloop +if "%1"=="-d" goto dswitch +if "%1"=="-m" goto mswitch +goto switchloopend +:dswitch +set BASEPATH_UNDER_HARVEST=0 +shift +goto switchloop +:mswitch +set MOVE_DATA=0 +shift +goto switchloop +:switchloopend + rem Make sure command line parameter was included: if not "!%2!"=="!!" goto paramsokay echo This script processes a batch of harvested authority records. @@ -36,13 +63,19 @@ echo This script will search the harvest subdirectories of the directories defin echo by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables. echo. echo Example: %0 lcnaf marc_lcnaf.properties +echo. +echo Options: +echo -d: Use the directory path as-is, do not append it to %HARVEST_DIR%. +echo Useful for non-OAI batch loading. +echo -m: Do not move the data files after importing. goto end :paramsokay rem Check if the path is valid: -set BASEPATH="%VUFIND_LOCAL_DIR%\harvest\%1" -if exist %BASEPATH% goto basepathfound -set BASEPATH="%VUFIND_HOME%\harvest\%1" +set BASEPATH="%HARVEST_DIR%\%1" +if "%BASEPATH_UNDER_HARVEST%"=="1" goto checkbasepath +set BASEPATH="%1" +:checkbasepath if exist %BASEPATH% goto basepathfound echo Directory %BASEPATH% does not exist! goto end @@ -58,9 +91,11 @@ md %BASEPATH%\processed rem Process all the files in the target directory: for %%a in (%BASEPATH%\*.xml %BASEPATH%\*.mrc) do ( - echo Processing %%a... - call %VUFIND_HOME%\import-marc-auth.bat %%a %2 > %BASEPATH%\log\%%~nxa.log + rem Capture solrmarc output to log + call %VUFIND_HOME%\import-marc-auth.bat %%a %2 2> %BASEPATH%\log\%%~nxa.log + if "%MOVE_DATA%"=="1" ( move %%a %BASEPATH%\processed\ > nul + ) ) :end \ No newline at end of file diff --git a/harvest/batch-import-marc-auth.sh b/harvest/batch-import-marc-auth.sh index 3537a146eb097b790d1e6d7338a059aa5e4cb818..3fa503803ffecde745830d494d1ad0db42a9e8ca 100755 --- a/harvest/batch-import-marc-auth.sh +++ b/harvest/batch-import-marc-auth.sh @@ -7,26 +7,56 @@ then exit 1 fi +# Find harvest directory for future use +HARVEST_DIR="$VUFIND_LOCAL_DIR/harvest" +if [ ! -d $HARVEST_DIR ] +then + HARVEST_DIR="$VUFIND_HOME/harvest" +fi + +BASEPATH_UNDER_HARVEST=true +MOVE_DATA=true + +while getopts ":dm" OPT +do + case $OPT in + d) BASEPATH_UNDER_HARVEST=false;; + m) MOVE_DATA=false;; + :) + echo "argument to '-$OPTARG' is missing" >&2 + exit -1;; + \?) echo "Unrecognized option '-$OPTARG'" >&2;; + esac +done +#Decrement the argument pointer so it points to next argument +shift $(($OPTIND - 1)) + # Make sure command line parameter was included: if [ -z "$2" ] then echo "This script processes a batch of harvested authority records." echo "" - echo "Usage: `basename $0` [harvest subdirectory] [SolrMarc properties file]" + echo "Usage: `basename $0` [-d] [-m] [harvest subdirectory] [SolrMarc properties file]" echo "" echo "[harvest subdirectory] is a directory name created by the OAI-PMH harvester." echo "This script will search the harvest subdirectories of the directories defined" echo "by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables." echo "" echo "Example: `basename $0` lcnaf marc_lcnaf.properties" + echo "" + echo "Options:" + echo "-d: Use the directory path as-is, do not append it to $HARVEST_DIR." + echo " Useful for non-OAI batch loading." + echo "-m: Do not move the data files after importing." exit 1 fi -# Check if the path is valid: -BASEPATH="$VUFIND_LOCAL_DIR/harvest/$1" -if [ ! -d $BASEPATH ] +# Set up BASEPATH and check if the path is valid: +if [ $BASEPATH_UNDER_HARVEST == false ] then - BASEPATH="$VUFIND_HOME/harvest/$1" + BASEPATH=$1 +else + BASEPATH="$HARVEST_DIR/$1" fi if [ ! -d $BASEPATH ] then @@ -49,8 +79,11 @@ for file in $BASEPATH/*.xml $BASEPATH/*.mrc do if [ -f $file ] then - echo "Processing $file ..." - $VUFIND_HOME/import-marc-auth.sh $file $2 > $BASEPATH/log/`basename $file`.log - mv $file $BASEPATH/processed/`basename $file` + # Capture solrmarc output to log + $VUFIND_HOME/import-marc-auth.sh $file $2 2> $BASEPATH/log/`basename $file`.log + if [ $MOVE_DATA == true ] + then + mv $file $BASEPATH/processed/`basename $file` + fi fi done diff --git a/harvest/batch-import-marc.bat b/harvest/batch-import-marc.bat index 6dda9378e7d7634947769d9f2b81a575004dd7d5..a920cce23e79e6a02a0bd8c0b072f438fb3bb69c 100644 --- a/harvest/batch-import-marc.bat +++ b/harvest/batch-import-marc.bat @@ -25,24 +25,57 @@ echo You need to set the VUFIND_HOME environmental variable before running this goto end :vufindhomefound +rem Find harvest directory for future use: +set HARVEST_DIR=%VUFIND_LOCAL_DIR%\harvest +if exist %HARVEST_DIR% goto harvestpathfound +set HARVEST_DIR=%VUFIND_HOME%\harvest +:harvestpathfound + +set BASEPATH_UNDER_HARVEST=1 +set MOVE_DATA=1 + +rem Save script name for message below (otherwise it may get shifted away) +set SCRIPT_NAME=%0 + +rem Process switches +:switchloop +if "%1"=="-d" goto dswitch +if "%1"=="-m" goto mswitch +goto switchloopend +:dswitch +set BASEPATH_UNDER_HARVEST=0 +shift +goto switchloop +:mswitch +set MOVE_DATA=0 +shift +goto switchloop +:switchloopend + rem Make sure command line parameter was included: if not "!%1!"=="!!" goto paramsokay echo This script processes a batch of harvested MARC records. echo. -echo Usage: %0 [harvest subdirectory] +echo Usage: %SCRIPT_NAME% [-d] [-m] [harvest subdirectory] echo. echo [harvest subdirectory] is a directory name created by the OAI-PMH harvester. echo This script will search the harvest subdirectories of the directories defined echo by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables. echo. echo Example: %0 oai_source +echo. +echo Options: +echo -d: Use the directory path as-is, do not append it to %HARVEST_DIR%. +echo Useful for non-OAI batch loading. +echo -m: Do not move the data files after importing. goto end :paramsokay rem Check if the path is valid: -set BASEPATH="%VUFIND_LOCAL_DIR%\harvest\%1" -if exist %BASEPATH% goto basepathfound -set BASEPATH="%VUFIND_HOME%\harvest\%1" +set BASEPATH="%HARVEST_DIR%\%1" +if "%BASEPATH_UNDER_HARVEST%"=="1" goto checkbasepath +set BASEPATH="%1" +:checkbasepath if exist %BASEPATH% goto basepathfound echo Directory %BASEPATH% does not exist! goto end @@ -58,9 +91,11 @@ md %BASEPATH%\processed rem Process all the files in the target directory: for %%a in (%BASEPATH%\*.xml %BASEPATH%\*.mrc) do ( - echo Processing %%a... - call %VUFIND_HOME%\import-marc.bat %%a > %BASEPATH%\log\%%~nxa.log - move %%a %BASEPATH%\processed\ > nul + rem Capture solrmarc output to log + call %VUFIND_HOME%\import-marc.bat %%a 2> %BASEPATH%\log\%%~nxa.log + if "%MOVE_DATA%"=="1" ( + move %%a %BASEPATH%\processed\ > nul + ) ) :end \ No newline at end of file diff --git a/harvest/batch-import-marc.sh b/harvest/batch-import-marc.sh index ed8299f63314962a982200bb195ccdb6e3171acb..17002f3bf905e916d396beac85a42a8fed333295 100755 --- a/harvest/batch-import-marc.sh +++ b/harvest/batch-import-marc.sh @@ -7,26 +7,56 @@ then exit 1 fi +# Find harvest directory for future use +HARVEST_DIR="$VUFIND_LOCAL_DIR/harvest" +if [ ! -d $HARVEST_DIR ] +then + HARVEST_DIR="$VUFIND_HOME/harvest" +fi + +BASEPATH_UNDER_HARVEST=true +MOVE_DATA=true + +while getopts ":dm" OPT +do + case $OPT in + d) BASEPATH_UNDER_HARVEST=false;; + m) MOVE_DATA=false;; + :) + echo "argument to '-$OPTARG' is missing" >&2 + exit -1;; + \?) echo "Unrecognized option '-$OPTARG'" >&2;; + esac +done +#Decrement the argument pointer so it points to next argument +shift $(($OPTIND - 1)) + # Make sure command line parameter was included: if [ -z "$1" ] then echo "This script processes a batch of harvested MARC records." echo "" - echo "Usage: `basename $0` [harvest subdirectory]" + echo "Usage: `basename $0` [-d] [-m] [harvest subdirectory]" echo "" echo "[harvest subdirectory] is a directory name created by the OAI-PMH harvester." echo "This script will search the harvest subdirectories of the directories defined" echo "by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables." echo "" echo "Example: `basename $0` oai_source" + echo "" + echo "Options:" + echo "-d: Use the directory path as-is, do not append it to $HARVEST_DIR." + echo " Useful for non-OAI batch loading." + echo "-m: Do not move the data files after importing." exit 1 fi -# Check if the path is valid: -BASEPATH="$VUFIND_LOCAL_DIR/harvest/$1" -if [ ! -d $BASEPATH ] +# Set up BASEPATH and check if the path is valid: +if [ $BASEPATH_UNDER_HARVEST == false ] then - BASEPATH="$VUFIND_HOME/harvest/$1" + BASEPATH=$1 +else + BASEPATH="$HARVEST_DIR/$1" fi if [ ! -d $BASEPATH ] then @@ -49,8 +79,11 @@ for file in $BASEPATH/*.xml $BASEPATH/*.mrc do if [ -f $file ] then - echo "Processing $file ..." - $VUFIND_HOME/import-marc.sh $file > $BASEPATH/log/`basename $file`.log - mv $file $BASEPATH/processed/`basename $file` + # Capture solrmarc output to log + $VUFIND_HOME/import-marc.sh $file 2> $BASEPATH/log/`basename $file`.log + if [ $MOVE_DATA == true ] + then + mv $file $BASEPATH/processed/`basename $file` + fi fi done diff --git a/import-marc.sh b/import-marc.sh index c2faffdbbe159e244af330db703413cc5ea8908c..9f2b3a8eb35d5a6f0612d2044bbdae7f1a43d437 100755 --- a/import-marc.sh +++ b/import-marc.sh @@ -138,7 +138,8 @@ MARC_FILE=`basename $1` RUN_CMD="$JAVA $INDEX_OPTIONS -Dsolr.core.name=$SOLRCORE $EXTRA_SOLRMARC_SETTINGS -jar $JAR_FILE $PROPERTIES_FILE $MARC_PATH/$MARC_FILE" echo "Now Importing $1 ..." -echo $RUN_CMD +# solrmarc writes log messages to stderr, write RUN_CMD to the same place +echo "`date '+%h %d, %H:%M:%S'` $RUN_CMD" >&2 exec $RUN_CMD exit 0