dataminer: Fix the algorithms installer script, in some situations it created duplicated lines in the file that lists the already installed algorithms.

dataminer: Download the distribution files into /home/gcube/downloads

smartgears/dataminer_app/tasks/dataminer-app.yml

@@ -1,4 +1,5 @@
 ---
+# This is only needed to force an upgrade
 - block:
     - name: Remove the old dataminer installation
       file: path={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=absent
@@ -10,6 +11,7 @@
     - dataminer_app_upgrade
   tags: [ 'tomcat', 'dataminer', 'wps' ]
 
+# This is only needed to force an upgrade
 - block:
     - name: Remove the old dataminer algorithm installer
       file: path={{ smartgears_user_home }}/algorithmInstaller state=absent
@@ -23,7 +25,7 @@
     
 - block: 
     - name: Get and unpack the dataminer application
-      maven_artifact: artifact_id={{ dataminer_wps_name }} version={{ dataminer_wps_version | default(omit) }} group_id={{ dataminer_wps_group_id }} extension={{ dataminer_wps_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_user_home }}/{{ dataminer_wps_file }}
+      maven_artifact: artifact_id={{ dataminer_wps_name }} version={{ dataminer_wps_version | default(omit) }} group_id={{ dataminer_wps_group_id }} extension={{ dataminer_wps_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_wps_file }}
       register: dataminer_app_downloaded
 
     - name: Remove the old dataminer installation
@@ -34,7 +36,7 @@
       file: path={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=directory owner={{ smartgears_user }} group={{ smartgears_user }}
   
     - name: Unarchive the dataminer_wps war
-      unarchive: copy=no src={{ smartgears_user_home }}/{{ dataminer_wps_file }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} creates='{{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/WEB-INF/lib'
+      unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_wps_file }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} creates='{{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/WEB-INF/lib'
       notify: Restart smartgears
 
     - name: Install the dataminer web.xml
@@ -45,7 +47,7 @@
       file: path={{ smartgears_user_home }}/wps_algorithms state=directory
 
     - name: Get and unpack the algorithms installer
-      maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_user_home }}/{{ dataminer_algorithms_file }}
+      maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }}
       register: algorithms_installer_downloaded
       tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
 
@@ -55,7 +57,7 @@
       tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
 
     - name: Unarchive the algorithms installer
-      unarchive: copy=no src={{ smartgears_user_home }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh'
+      unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh'
       tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
 
     - name: Download the WPS algorithms from subversion

smartgears/dataminer_app/templates/algorithms-updater.j2

@@ -12,6 +12,7 @@ ALGORITHMS_TEMP_SCRIPT=$OUT_DIR/add_algorithms
 LOG_DIR={{ smartgears_user_home }}/wps_algorithms_install_log
 LOG_FILE=${LOG_DIR}/algorithms_updater.log
 ALGORITHMS_INSTALLED_FILE=${LOG_DIR}/already_installed_algorithms.txt
+ALGORITHMS_INSTALLED_SORTED_FILE=${LOG_DIR}/already_installed_algorithms_sorted.txt
 LOCK_FILE=${LOG_DIR}/.algorithms_updater.lock
 ALGO_DIR={{ dataminer_wps_algorithms_dest }}
 RUNNING_JOB=
@@ -60,6 +61,13 @@ function update_svn_repo() {
     svn update > $LOG_FILE 2>&1
 }
 
+function sort_installed_algo_file() {
+    if [ -f $ALGORITHMS_INSTALLED_FILE ] ; then
+        sort -u $ALGORITHMS_INSTALLED_FILE > $ALGORITHMS_INSTALLED_SORTED_FILE
+        mv $ALGORITHMS_INSTALLED_SORTED_FILE $ALGORITHMS_INSTALLED_FILE
+    fi
+}
+
 function algorithms_updater() {
     logger 'algorithms-updater: scan the algorithms list and build the algorithms script. Reference infra is {{ dataminer_infra_reference }}'
     echo "#!/bin/bash" > $ALGORITHMS_TEMP_SCRIPT.head
@@ -80,6 +88,9 @@ function algorithms_updater() {
         ALGO_BODY=$( echo "$algo" | awk -F \| '{ print $2 }' )
         ALGO_DATE=$( echo "$algo" | awk -F \| '{ print $3 }' )
         ALGO_TIMESTAMP=$( date +%s -d "$ALGO_DATE" )
+        if [ $? -ne 0 ] ; then
+            echo "the date for algorithm $ALGO_NAME is wrong"
+        fi
         ALGO_LINE=$( egrep ^"$ALGO_NAME" $ALGORITHMS_INSTALLED_FILE )
         ALGO_PRESENT=$?
         if [ $ALGO_PRESENT -ne 0 ] ; then
@@ -106,6 +117,9 @@ function algorithms_updater() {
             ALGO_LOCAL_TIMESTAMP=0
             ALGO_LOCAL_DATE=$( egrep ^"$ALGO_NAME" $ALGORITHMS_INSTALLED_FILE | awk -F \| '{ print $2 }' | grep -v \' )
             ALGO_LOCAL_TIMESTAMP=$(  date +%s -d "$ALGO_LOCAL_DATE" )
+            if [ $? -ne 0 ] ; then
+                echo "the locally saved date for algorithm $ALGO_NAME is wrong"
+            fi            
             if [ $ALGO_LOCAL_TIMESTAMP -ne $ALGO_TIMESTAMP ] ; then
                 echo "logger 'algorithms-updater: running the add command of algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_
                 echo "logger 'algorithms-updater: the add command string is $ALGO_BODY'" >> $ALGORITHMS_TEMP_SCRIPT.body_
@@ -155,5 +169,6 @@ function algorithms_updater() {
 create_log_dir
 check_lock_file
 update_svn_repo
+sort_installed_algo_file
 algorithms_updater
 cleanup 0