forked from ISTI-ansible-roles/ansible-roles
222 lines
11 KiB
Django/Jinja
222 lines
11 KiB
Django/Jinja
#!/bin/bash
|
|
|
|
set -e
|
|
set -o pipefail
|
|
|
|
PROCNUM=$$
|
|
INFRA_REFERENCE={{ dataminer_infra_reference }}
|
|
ADD_ALGORITHM_DIR={{ smartgears_user_home }}/algorithmInstaller
|
|
ADD_ALGORITHM_PATH={{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh
|
|
ALGORITHMS_FILE={{ dataminer_wps_algorithms_dest }}/${INFRA_REFERENCE}/algorithms
|
|
OUT_DIR=$( mktemp -d -t algorithms-updater.XXXXXXXXXX )
|
|
ALGORITHMS_TEMP_SCRIPT=$OUT_DIR/add_algorithms
|
|
LOG_DIR={{ smartgears_user_home }}/wps_algorithms_install_log
|
|
LOG_FILE=${LOG_DIR}/algorithms_updater.log
|
|
ALGORITHMS_INSTALLED_FILE=${LOG_DIR}/already_installed_algorithms.txt
|
|
ALGORITHMS_INSTALLED_SORTED_FILE=${LOG_DIR}/already_installed_algorithms_sorted.txt
|
|
LOCK_FILE=${LOG_DIR}/.algorithms_updater.lock
|
|
ALGO_BASE_DIR={{ dataminer_wps_algorithms_dest }}
|
|
ALGO_DIR=${ALGO_BASE_DIR}/${INFRA_REFERENCE}
|
|
SVN_ALGORITHMS_URL={{ dataminer_wps_algorithms_svn }}
|
|
SVN_UPDATE_STATUS=
|
|
# In seconds. 60*60*6=21600s (6h)
|
|
UPDATER_PROCESS_MAX_RUNTIME=21600
|
|
OLDPROC=
|
|
OLDPROC_RUNNING=
|
|
|
|
trap "logger 'algorithms-updater: trap intercepted, exiting.' ; cleanup 1" SIGHUP SIGINT SIGTERM
|
|
|
|
cleanup() {
|
|
rm -fr $OUT_DIR
|
|
rm -f $LOCK_FILE
|
|
logger 'algorithms-updater: Exiting'
|
|
exit $1
|
|
}
|
|
|
|
create_log_dir() {
|
|
if [ ! -d $LOG_DIR ] ; then
|
|
mkdir -p $LOG_DIR
|
|
fi
|
|
}
|
|
|
|
check_lock_file() {
|
|
# Create the lock file
|
|
if [ -f "$LOCK_FILE" ] ; then
|
|
set +o pipefail
|
|
set +e
|
|
OLDPROC=$( cat "$LOCK_FILE" )
|
|
OLDPROC_RUNNING=$( ps auwwx | grep -v grep | awk '{ print $2 }' | grep "$OLDPROC" )
|
|
if [ ! -z "$OLDPROC_RUNNING" ] ; then
|
|
logger "algorithms_updater: pid of the already running process: $OLDPROC_RUNNING"
|
|
OLDPROC_RUNNING_TIME=$( ps -o etimes= -p "${OLDPROC_RUNNING}" )
|
|
if [ "$OLDPROC_RUNNING_TIME" -gt $UPDATER_PROCESS_MAX_RUNTIME ] ; then
|
|
logger "algorithms_updater: process $OLDPROC_RUNNING was running for $OLDPROC_RUNNING_TIME seconds. Got stuck, killing it"
|
|
kill -9 "$OLDPROC_RUNNING"
|
|
rm -f "$LOCK_FILE"
|
|
else
|
|
logger "algorithms_updater: another process is running, exiting."
|
|
rm -fr "$OUT_DIR"
|
|
exit 0
|
|
fi
|
|
else
|
|
logger "algorithms_updater: lock file exist but the process not. Continuing."
|
|
rm -f "$LOCK_FILE"
|
|
fi
|
|
else
|
|
logger 'algorithms-updater: no other jobs running, proceeding.'
|
|
fi
|
|
set -o pipefail
|
|
set -e
|
|
echo "$PROCNUM" > "$LOCK_FILE"
|
|
}
|
|
|
|
update_svn_repo() {
|
|
if [ -d $ALGO_DIR ]; then
|
|
logger 'algorithms-updater: update the SVN repo'
|
|
cd $ALGO_DIR
|
|
# Do a svn cleanup to be on the safe side
|
|
set +o pipefail
|
|
set +e
|
|
SVN_CLEANUP_OP=$( svn cleanup )
|
|
SVN_UPDATE_OP=$( svn update | tail -1 | grep Updated >$LOG_FILE 2>&1 )
|
|
SVN_UPDATE_STATUS=$?
|
|
set -o pipefail
|
|
set -e
|
|
else
|
|
cd $ALGO_BASE_DIR
|
|
logger "algorithms-updater: First subversion checkout"
|
|
svn co $SVN_ALGORITHMS_URL >> $LOG_FILE 2>&1
|
|
fi
|
|
}
|
|
|
|
sort_installed_algo_file() {
|
|
if [ -f $ALGORITHMS_INSTALLED_FILE ] ; then
|
|
sort -u $ALGORITHMS_INSTALLED_FILE > $ALGORITHMS_INSTALLED_SORTED_FILE
|
|
mv $ALGORITHMS_INSTALLED_SORTED_FILE $ALGORITHMS_INSTALLED_FILE
|
|
fi
|
|
}
|
|
|
|
algorithms_updater() {
|
|
logger 'algorithms-updater: scan the algorithms list and build the algorithms script. Reference infra is {{ dataminer_infra_reference }}'
|
|
echo "#!/bin/bash" > $ALGORITHMS_TEMP_SCRIPT.head
|
|
echo "cd $ADD_ALGORITHM_DIR" >> $ALGORITHMS_TEMP_SCRIPT.head
|
|
echo "" >> $ALGORITHMS_TEMP_SCRIPT.head
|
|
awk -F \| '{ print $2 "|" $6 "|" $8 }' $ALGORITHMS_FILE > $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
sed -i -e 's/<notextile>//g' $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
sed -i -e 's/<\/notextile>"//g' $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
sed -i -e 's/<\/notextile>//g' $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
sed -i -e 's/\ N\ /\ Y\ /g' $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
sed -i -e 's/\[/\ /g' $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
sed -i -e 's/\]/\ /g' $ALGORITHMS_TEMP_SCRIPT.algorithmslist
|
|
if [ ! -f $ALGORITHMS_INSTALLED_FILE ] ; then
|
|
touch $ALGORITHMS_INSTALLED_FILE
|
|
fi
|
|
set +o pipefail
|
|
set +e
|
|
while read algo ; do
|
|
ALGO_PRESENT=
|
|
ALGO_NAME=$( echo "$algo" | awk -F \| '{ print $1 }' )
|
|
ALGO_BODY_PRE=$( echo "$algo" | awk -F \| '{ print $2 }' )
|
|
ALGO_BODY="${ALGO_BODY_PRE} ${INFRA_REFERENCE}/software"
|
|
ALGO_BODY=$( sed "s/'/ /g" <<< ${ALGO_BODY} )
|
|
ALGO_DATE=$( echo "$algo" | awk -F \| '{ print $3 }' )
|
|
ALGO_TIMESTAMP=$( date +%s -d "$ALGO_DATE" 2>/dev/null )
|
|
if [ $? -ne 0 ] ; then
|
|
echo "the date for algorithm $ALGO_NAME is wrong"
|
|
else
|
|
ALGO_LINE=$( egrep ^"$ALGO_NAME " $ALGORITHMS_INSTALLED_FILE )
|
|
ALGO_PRESENT=$?
|
|
if [ $ALGO_PRESENT -ne 0 ] ; then
|
|
echo "logger 'algorithms-updater: New algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "logger 'algorithms-updater: running the add command of algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "logger 'algorithms-updater: the add command string is ${ALGO_BODY}'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "RETVAL=" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo '-------------------------------------------'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo 'Adding algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo 'With command ${ALGO_BODY}'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "cd ${ADD_ALGORITHM_DIR}" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "${ALGO_BODY}" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo 'RETVAL=$?' >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo 'Done.'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo 'if [ $RETVAL -ne 0 ] ; then' >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " logger 'algorithms-updater: the adding of algorithm $ALGO_NAME failed'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "else" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " egrep -v ^\"$ALGO_NAME \" $ALGORITHMS_INSTALLED_FILE > ${ALGORITHMS_INSTALLED_FILE}_" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " mv ${ALGORITHMS_INSTALLED_FILE}_ $ALGORITHMS_INSTALLED_FILE" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " echo '$ALGO_NAME | $ALGO_DATE' >> $ALGORITHMS_INSTALLED_FILE" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " logger 'algorithms-updater: the adding of algorithm $ALGO_NAME succeeded'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "fi" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
else
|
|
ALGO_LOCAL_TIMESTAMP=0
|
|
ALGO_LOCAL_DATE=$( egrep ^"$ALGO_NAME " $ALGORITHMS_INSTALLED_FILE | tail -1 | awk -F \| '{ print $2 }' | grep -v \' )
|
|
ALGO_LOCAL_TIMESTAMP=$( date +%s -d "$ALGO_LOCAL_DATE" 2>/dev/null )
|
|
if [ $? -ne 0 ] || [ $ALGO_LOCAL_TIMESTAMP -ne $ALGO_TIMESTAMP ]; then
|
|
echo "logger 'algorithms-updater: algorithm $ALGO_NAME is already present but a newer version exists'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "logger 'algorithms-updater: running the add command of algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "logger 'algorithms-updater: the add command string is ${ALGO_BODY}'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "RETVAL=" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo '-------------------------------------------'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo 'Adding algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo 'With command ${ALGO_BODY}'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "cd ${ADD_ALGORITHM_DIR}" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "${ALGO_BODY}" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo 'RETVAL=$?' >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo 'Done.'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "echo ''" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo 'if [ $RETVAL -ne 0 ] ; then' >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " logger 'algorithms-updater: the adding of algorithm $ALGO_NAME failed'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "else" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " egrep -v \"^$ALGO_NAME \" $ALGORITHMS_INSTALLED_FILE > ${ALGORITHMS_INSTALLED_FILE}_" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " mv ${ALGORITHMS_INSTALLED_FILE}_ $ALGORITHMS_INSTALLED_FILE" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " echo '$ALGO_NAME | $ALGO_DATE' >> $ALGORITHMS_INSTALLED_FILE" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo " logger 'algorithms-updater: the adding of algorithm $ALGO_NAME succeeded'" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
echo "fi" >> $ALGORITHMS_TEMP_SCRIPT.body_
|
|
fi
|
|
fi
|
|
fi
|
|
done < ${ALGORITHMS_TEMP_SCRIPT}.algorithmslist
|
|
set -o pipefail
|
|
set -e
|
|
if [ -f $ALGORITHMS_TEMP_SCRIPT.body_ ] ; then
|
|
mv $ALGORITHMS_TEMP_SCRIPT.body_ $ALGORITHMS_TEMP_SCRIPT.body
|
|
cat $ALGORITHMS_TEMP_SCRIPT.head $ALGORITHMS_TEMP_SCRIPT.body > $ALGORITHMS_TEMP_SCRIPT
|
|
chmod 755 $ALGORITHMS_TEMP_SCRIPT
|
|
if [ -x $ADD_ALGORITHM_PATH ] ; then
|
|
logger 'algorithms-updater: add the algorithms configurations. Reference infra is {{ dataminer_infra_reference }}'
|
|
$ALGORITHMS_TEMP_SCRIPT >> $LOG_FILE 2>&1
|
|
else
|
|
logger 'algorithms-updater: {{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh is not an executable, aborting'
|
|
cleanup 1
|
|
fi
|
|
else
|
|
cleanup 0
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Main
|
|
#
|
|
create_log_dir
|
|
check_lock_file
|
|
update_svn_repo
|
|
if [ $SVN_UPDATE_STATUS -ne 0 ] ; then
|
|
if [ -d "${LOG_DIR}" ] ; then
|
|
if [ ! -f "$ALGORITHMS_INSTALLED_FILE" ] ; then
|
|
logger "algorithms-updater: nothing new to from SVN but the file that lists the installed algorithms does not exist. We are going to reinstall all the algorithms."
|
|
else
|
|
logger "algorithms-updater: nothing new to from SVN, exiting."
|
|
cleanup 0
|
|
fi
|
|
fi
|
|
fi
|
|
sort_installed_algo_file
|
|
algorithms_updater
|
|
cleanup 0
|