Skip to content
Snippets Groups Projects
Commit 3a6a3715 authored by Demian Katz's avatar Demian Katz
Browse files

Progress on VUFIND-729 (Additional options for batch-import-marc.sh and batch-import-marc-auth.sh).

parent 55b9f45d
No related merge requests found
Hide whitespace changes
Inline Side-by-side
harvest/batch-import-marc-auth.bat
+ 40
− 5
View file @ 3a6a3715
......@@ -25,6 +25,33 @@ echo You need to set the VUFIND_HOME environmental variable before running this
goto end
:vufindhomefound
rem Find harvest directory for future use:
set HARVEST_DIR=%VUFIND_LOCAL_DIR%\harvest
if exist %HARVEST_DIR% goto harvestpathfound
set HARVEST_DIR=%VUFIND_HOME%\harvest
:harvestpathfound
set BASEPATH_UNDER_HARVEST=1
set MOVE_DATA=1
rem Save script name for message below (otherwise it may get shifted away)
set SCRIPT_NAME=%0
rem Process switches
:switchloop
if "%1"=="-d" goto dswitch
if "%1"=="-m" goto mswitch
goto switchloopend
:dswitch
set BASEPATH_UNDER_HARVEST=0
shift
goto switchloop
:mswitch
set MOVE_DATA=0
shift
goto switchloop
:switchloopend
rem Make sure command line parameter was included:
if not "!%2!"=="!!" goto paramsokay
echo This script processes a batch of harvested authority records.
......@@ -36,13 +63,19 @@ echo This script will search the harvest subdirectories of the directories defin
echo by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables.
echo.
echo Example: %0 lcnaf marc_lcnaf.properties
echo.
echo Options:
echo -d: Use the directory path as-is, do not append it to %HARVEST_DIR%.
echo Useful for non-OAI batch loading.
echo -m: Do not move the data files after importing.
goto end
:paramsokay
rem Check if the path is valid:
set BASEPATH="%VUFIND_LOCAL_DIR%\harvest\%1"
if exist %BASEPATH% goto basepathfound
set BASEPATH="%VUFIND_HOME%\harvest\%1"
set BASEPATH="%HARVEST_DIR%\%1"
if "%BASEPATH_UNDER_HARVEST%"=="1" goto checkbasepath
set BASEPATH="%1"
:checkbasepath
if exist %BASEPATH% goto basepathfound
echo Directory %BASEPATH% does not exist!
goto end
......@@ -58,9 +91,11 @@ md %BASEPATH%\processed
rem Process all the files in the target directory:
for %%a in (%BASEPATH%\*.xml %BASEPATH%\*.mrc) do (
echo Processing %%a...
call %VUFIND_HOME%\import-marc-auth.bat %%a %2 > %BASEPATH%\log\%%~nxa.log
rem Capture solrmarc output to log
call %VUFIND_HOME%\import-marc-auth.bat %%a %2 2> %BASEPATH%\log\%%~nxa.log
if "%MOVE_DATA%"=="1" (
move %%a %BASEPATH%\processed\ > nul
)
)
:end
\ No newline at end of file
harvest/batch-import-marc-auth.sh
+ 41
− 8
View file @ 3a6a3715
......@@ -7,26 +7,56 @@ then
exit 1
fi
# Find harvest directory for future use
HARVEST_DIR="$VUFIND_LOCAL_DIR/harvest"
if [ ! -d $HARVEST_DIR ]
then
HARVEST_DIR="$VUFIND_HOME/harvest"
fi
BASEPATH_UNDER_HARVEST=true
MOVE_DATA=true
while getopts ":dm" OPT
do
case $OPT in
d) BASEPATH_UNDER_HARVEST=false;;
m) MOVE_DATA=false;;
:)
echo "argument to '-$OPTARG' is missing" >&2
exit -1;;
\?) echo "Unrecognized option '-$OPTARG'" >&2;;
esac
done
#Decrement the argument pointer so it points to next argument
shift $(($OPTIND - 1))
# Make sure command line parameter was included:
if [ -z "$2" ]
then
echo "This script processes a batch of harvested authority records."
echo ""
echo "Usage: `basename $0` [harvest subdirectory] [SolrMarc properties file]"
echo "Usage: `basename $0` [-d] [-m] [harvest subdirectory] [SolrMarc properties file]"
echo ""
echo "[harvest subdirectory] is a directory name created by the OAI-PMH harvester."
echo "This script will search the harvest subdirectories of the directories defined"
echo "by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables."
echo ""
echo "Example: `basename $0` lcnaf marc_lcnaf.properties"
echo ""
echo "Options:"
echo "-d: Use the directory path as-is, do not append it to $HARVEST_DIR."
echo " Useful for non-OAI batch loading."
echo "-m: Do not move the data files after importing."
exit 1
fi
# Check if the path is valid:
BASEPATH="$VUFIND_LOCAL_DIR/harvest/$1"
if [ ! -d $BASEPATH ]
# Set up BASEPATH and check if the path is valid:
if [ $BASEPATH_UNDER_HARVEST == false ]
then
BASEPATH="$VUFIND_HOME/harvest/$1"
BASEPATH=$1
else
BASEPATH="$HARVEST_DIR/$1"
fi
if [ ! -d $BASEPATH ]
then
......@@ -49,8 +79,11 @@ for file in $BASEPATH/*.xml $BASEPATH/*.mrc
do
if [ -f $file ]
then
echo "Processing $file ..."
$VUFIND_HOME/import-marc-auth.sh $file $2 > $BASEPATH/log/`basename $file`.log
mv $file $BASEPATH/processed/`basename $file`
# Capture solrmarc output to log
$VUFIND_HOME/import-marc-auth.sh $file $2 2> $BASEPATH/log/`basename $file`.log
if [ $MOVE_DATA == true ]
then
mv $file $BASEPATH/processed/`basename $file`
fi
fi
done
harvest/batch-import-marc.bat
+ 42
− 7
View file @ 3a6a3715
......@@ -25,24 +25,57 @@ echo You need to set the VUFIND_HOME environmental variable before running this
goto end
:vufindhomefound
rem Find harvest directory for future use:
set HARVEST_DIR=%VUFIND_LOCAL_DIR%\harvest
if exist %HARVEST_DIR% goto harvestpathfound
set HARVEST_DIR=%VUFIND_HOME%\harvest
:harvestpathfound
set BASEPATH_UNDER_HARVEST=1
set MOVE_DATA=1
rem Save script name for message below (otherwise it may get shifted away)
set SCRIPT_NAME=%0
rem Process switches
:switchloop
if "%1"=="-d" goto dswitch
if "%1"=="-m" goto mswitch
goto switchloopend
:dswitch
set BASEPATH_UNDER_HARVEST=0
shift
goto switchloop
:mswitch
set MOVE_DATA=0
shift
goto switchloop
:switchloopend
rem Make sure command line parameter was included:
if not "!%1!"=="!!" goto paramsokay
echo This script processes a batch of harvested MARC records.
echo.
echo Usage: %0 [harvest subdirectory]
echo Usage: %SCRIPT_NAME% [-d] [-m] [harvest subdirectory]
echo.
echo [harvest subdirectory] is a directory name created by the OAI-PMH harvester.
echo This script will search the harvest subdirectories of the directories defined
echo by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables.
echo.
echo Example: %0 oai_source
echo.
echo Options:
echo -d: Use the directory path as-is, do not append it to %HARVEST_DIR%.
echo Useful for non-OAI batch loading.
echo -m: Do not move the data files after importing.
goto end
:paramsokay
rem Check if the path is valid:
set BASEPATH="%VUFIND_LOCAL_DIR%\harvest\%1"
if exist %BASEPATH% goto basepathfound
set BASEPATH="%VUFIND_HOME%\harvest\%1"
set BASEPATH="%HARVEST_DIR%\%1"
if "%BASEPATH_UNDER_HARVEST%"=="1" goto checkbasepath
set BASEPATH="%1"
:checkbasepath
if exist %BASEPATH% goto basepathfound
echo Directory %BASEPATH% does not exist!
goto end
......@@ -58,9 +91,11 @@ md %BASEPATH%\processed
rem Process all the files in the target directory:
for %%a in (%BASEPATH%\*.xml %BASEPATH%\*.mrc) do (
echo Processing %%a...
call %VUFIND_HOME%\import-marc.bat %%a > %BASEPATH%\log\%%~nxa.log
move %%a %BASEPATH%\processed\ > nul
rem Capture solrmarc output to log
call %VUFIND_HOME%\import-marc.bat %%a 2> %BASEPATH%\log\%%~nxa.log
if "%MOVE_DATA%"=="1" (
move %%a %BASEPATH%\processed\ > nul
)
)
:end
\ No newline at end of file
harvest/batch-import-marc.sh
+ 41
− 8
View file @ 3a6a3715
......@@ -7,26 +7,56 @@ then
exit 1
fi
# Find harvest directory for future use
HARVEST_DIR="$VUFIND_LOCAL_DIR/harvest"
if [ ! -d $HARVEST_DIR ]
then
HARVEST_DIR="$VUFIND_HOME/harvest"
fi
BASEPATH_UNDER_HARVEST=true
MOVE_DATA=true
while getopts ":dm" OPT
do
case $OPT in
d) BASEPATH_UNDER_HARVEST=false;;
m) MOVE_DATA=false;;
:)
echo "argument to '-$OPTARG' is missing" >&2
exit -1;;
\?) echo "Unrecognized option '-$OPTARG'" >&2;;
esac
done
#Decrement the argument pointer so it points to next argument
shift $(($OPTIND - 1))
# Make sure command line parameter was included:
if [ -z "$1" ]
then
echo "This script processes a batch of harvested MARC records."
echo ""
echo "Usage: `basename $0` [harvest subdirectory]"
echo "Usage: `basename $0` [-d] [-m] [harvest subdirectory]"
echo ""
echo "[harvest subdirectory] is a directory name created by the OAI-PMH harvester."
echo "This script will search the harvest subdirectories of the directories defined"
echo "by the VUFIND_LOCAL_DIR and VUFIND_HOME environment variables."
echo ""
echo "Example: `basename $0` oai_source"
echo ""
echo "Options:"
echo "-d: Use the directory path as-is, do not append it to $HARVEST_DIR."
echo " Useful for non-OAI batch loading."
echo "-m: Do not move the data files after importing."
exit 1
fi
# Check if the path is valid:
BASEPATH="$VUFIND_LOCAL_DIR/harvest/$1"
if [ ! -d $BASEPATH ]
# Set up BASEPATH and check if the path is valid:
if [ $BASEPATH_UNDER_HARVEST == false ]
then
BASEPATH="$VUFIND_HOME/harvest/$1"
BASEPATH=$1
else
BASEPATH="$HARVEST_DIR/$1"
fi
if [ ! -d $BASEPATH ]
then
......@@ -49,8 +79,11 @@ for file in $BASEPATH/*.xml $BASEPATH/*.mrc
do
if [ -f $file ]
then
echo "Processing $file ..."
$VUFIND_HOME/import-marc.sh $file > $BASEPATH/log/`basename $file`.log
mv $file $BASEPATH/processed/`basename $file`
# Capture solrmarc output to log
$VUFIND_HOME/import-marc.sh $file 2> $BASEPATH/log/`basename $file`.log
if [ $MOVE_DATA == true ]
then
mv $file $BASEPATH/processed/`basename $file`
fi
fi
done
import-marc.sh
+ 2
− 1
View file @ 3a6a3715
......@@ -138,7 +138,8 @@ MARC_FILE=`basename $1`
RUN_CMD="$JAVA $INDEX_OPTIONS -Dsolr.core.name=$SOLRCORE $EXTRA_SOLRMARC_SETTINGS -jar $JAR_FILE $PROPERTIES_FILE $MARC_PATH/$MARC_FILE"
echo "Now Importing $1 ..."
echo $RUN_CMD
# solrmarc writes log messages to stderr, write RUN_CMD to the same place
echo "`date '+%h %d, %H:%M:%S'` $RUN_CMD" >&2
exec $RUN_CMD
exit 0
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment