dataminer: more modularity, add smartgears as dependency.

smartgears/dataminer_app/defaults/main.yml

@@ -1,13 +1,13 @@
 ---
 dataminer_app_install: False
 dataminer_app_upgrade: False
+dataminer_algorithms_installer: True
 
 # ZIP file
 dataminer_product: prod
 dataminer_zip_url: 'http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/webapp/{{ dataminer_product }}/wps.zip'
 
 # Install from WAR
-dataminer_war_install: True
 dataminer_wps_version: latest
 dataminer_wps_name: wps
 dataminer_wps_group_id: org.gcube.data-analysis
@@ -32,10 +32,12 @@ dataminer_algorithms_updater_cron: present
 dataminer_algorithms_updater_minute_frequency: "0"
 dataminer_algorithms_updater_hour_frequency: "*"
 
+dataminer_download_gebco_file: True
 dataminer_data_file_from_thredds: 'http://thredds.d4science.org/thredds/fileServer/public/netcdf/gebco_08_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc'
 
-dataminer_conda_install: True
+dataminer_conda_install: False
 dataminer_hdf5: False
+dataminer_install_nscd: False
 
 # dev, prod, preprod
 dataminer_infra_reference: dev

smartgears/dataminer_app/meta/main.yml

@@ -1,4 +1,5 @@
 ---
 dependencies:
+  - { role: ../../library/roles/smartgears/smartgears-service }
   - { role: ../../library/roles/conda, when: dataminer_conda_install }
   - { role: ../../library/roles/hdf5, when: dataminer_hdf5 }

smartgears/dataminer_app/tasks/dataminer-algorithms-installer.yml

@@ -0,0 +1,77 @@
+---
+# This is only needed to force an upgrade of the algorithm importer script 
+- block:
+  - name: Remove the old dataminer algorithm installer to force an upgrade
+    file: path={{ smartgears_user_home }}/algorithmInstaller state=absent
+
+  - name: Remove the old dataminer algorithm installer distributions from the downloads directory
+    shell: rm -fr {{ smartgears_user_home }}/downloads/dataminer-algorithms-importer*.tar.gz
+
+  become: True
+  become_user: '{{ smartgears_user }}'
+  when:
+    - dataminer_app_install
+    - dataminer_algorithms_upgrade
+  tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
+  
+
+- block:
+
+  - name: Verify if we have the old algorithms setup
+    stat: path={{ smartgears_user_home }}/wps_algorithms/algorithms/AAPS_NASO_PUBLISHER.jar
+    register: old_algo_setup
+
+  - name: Wipe away the algoriths directory if it comes from the old setup
+    file: dest={{ smartgears_user_home }}/wps_algorithms state=absent
+    when: old_algo_setup.stat.exists
+
+  - name: Create a directory where to install the algorithms directory
+    file: path={{ smartgears_user_home }}/wps_algorithms/algorithms state=directory
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ]
+
+  - name: Install the subversion packages 
+    become_user: root 
+    apt: pkg=subversion state=present update_cache=yes cache_valid_time=1800
+
+  - name: Download the WPS algorithms from subversion
+    subversion: repo={{ dataminer_wps_algorithms_svn }} dest={{ smartgears_user_home }}/wps_algorithms/algorithms/{{ dataminer_infra_reference }} checkout=yes force=yes update=yes
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ]
+
+  - name: Get the algorithms installer via maven_artifact
+    maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }}
+    register: algorithms_installer_downloaded
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
+
+  - name: Remove the old dataminer algorithm installer
+    file: path={{ smartgears_user_home }}/algorithmInstaller state=absent
+    when: algorithms_installer_downloaded is changed
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
+
+  - name: Unarchive the algorithms installer
+    unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh'
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
+
+  - name: Create a directory where to install the algorithms updater logs and data
+    file: path={{ smartgears_user_home }}/wps_algorithms_install_log state=directory
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ]
+
+  - name: Install a script that updates the algorithms repository and adds the missing algorithms configurations
+    become_user: "{{ d4science_ansible_become_user | default('root') }}"
+    template: src=algorithms-updater.j2 dest=/usr/local/bin/algorithms-updater mode=0755
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ]
+
+  - name: Remove the list of the installed algorithms if we are upgrading from the old algorithms setup
+    file: dest={{ smartgears_user_home }}/wps_algorithms_install_log/already_installed_algorithms.txt state=absent
+    when: old_algo_setup.stat.exists
+    
+  - name: Cron job that updates the algorithms repository and adds the missing algorithms configurations
+    cron: name="SVN update the algorithms repository" minute="{{ dataminer_algorithms_updater_minute_frequency }}" hour="{{ dataminer_algorithms_updater_hour_frequency }}" job="/usr/local/bin/algorithms-updater > {{ smartgears_user_home }}/wps_algorithms_install_log/algorithms_updater_cron.log 2>&1" user='{{ smartgears_user }}' state={{ dataminer_algorithms_updater_cron }}
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_cron' ]
+
+  become: True
+  become_user: '{{ smartgears_user }}'
+  when:
+    - dataminer_app_install
+    - dataminer_algorithms_upgrade
+  tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
+  

smartgears/dataminer_app/tasks/dataminer-app.yml

@@ -1,31 +1,4 @@
 ---
-# This is only needed to force an upgrade
-- block:
-    - name: Remove the old dataminer installation
-      file: path={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=absent
-
-  become: True
-  become_user: '{{ smartgears_user }}'
-  when:
-    - dataminer_app_install
-    - dataminer_app_upgrade
-  tags: [ 'tomcat', 'dataminer', 'wps' ]
-
-# This is only needed to force an upgrade of the algorithm importer script 
-- block:
-    - name: Remove the old dataminer algorithm installer to force an upgrade
-      file: path={{ smartgears_user_home }}/algorithmInstaller state=absent
-
-    - name: Remove the old dataminer algorithm installer distributions from the downloads directory
-      shell: rm -fr {{ smartgears_user_home }}/downloads/dataminer-algorithms-importer*.tar.gz
-
-  become: True
-  become_user: '{{ smartgears_user }}'
-  when:
-    - dataminer_app_install
-    - dataminer_algorithms_upgrade
-  tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
-    
 - block: 
     - name: Get and unpack the dataminer application
       maven_artifact: artifact_id={{ dataminer_wps_name }} version={{ dataminer_wps_version | default(omit) }} group_id={{ dataminer_wps_group_id }} extension={{ dataminer_wps_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_wps_file }}
@@ -47,67 +20,6 @@
       template: src=web.xml.j2 dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/WEB-INF/web.xml
       notify: Restart smartgears
 
-    - name: Verify if we have the old algorithms setup
-      stat: path={{ smartgears_user_home }}/wps_algorithms/algorithms/AAPS_NASO_PUBLISHER.jar
-      register: old_algo_setup
-
-    - name: Wipe away the algoriths directory if it comes from the old setup
-      file: dest={{ smartgears_user_home }}/wps_algorithms state=absent
-      when: old_algo_setup.stat.exists
-
-    - name: Create a directory where to install the algorithms directory
-      file: path={{ smartgears_user_home }}/wps_algorithms/algorithms state=directory
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ]
-
-    - name: Install the subversion packages 
-      become_user: root 
-      apt: pkg=subversion state=present update_cache=yes cache_valid_time=1800
-
-    - name: Download the WPS algorithms from subversion
-      subversion: repo={{ dataminer_wps_algorithms_svn }} dest={{ smartgears_user_home }}/wps_algorithms/algorithms/{{ dataminer_infra_reference }} checkout=yes force=yes update=yes
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ]
-
-    - name: Get the algorithms installer via maven_artifact
-      maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }}
-      register: algorithms_installer_downloaded
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
-
-    - name: Remove the old dataminer algorithm installer
-      file: path={{ smartgears_user_home }}/algorithmInstaller state=absent
-      when: algorithms_installer_downloaded is changed
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
-
-    - name: Unarchive the algorithms installer
-      unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh'
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ]
-
-    - name: Create a directory where to install the algorithms updater logs and data
-      file: path={{ smartgears_user_home }}/wps_algorithms_install_log state=directory
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ]
-
-    - name: Install a script that updates the algorithms repository and adds the missing algorithms configurations
-      become_user: "{{ d4science_ansible_become_user | default('root') }}"
-      template: src=algorithms-updater.j2 dest=/usr/local/bin/algorithms-updater mode=0755
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ]
-
-    - name: Remove the list of the installed algorithms if we are upgrading from the old algorithms setup
-      file: dest={{ smartgears_user_home }}/wps_algorithms_install_log/already_installed_algorithms.txt state=absent
-      when: old_algo_setup.stat.exists
-      
-    - name: Cron job that updates the algorithms repository and adds the missing algorithms configurations
-      cron: name="SVN update the algorithms repository" minute="{{ dataminer_algorithms_updater_minute_frequency }}" hour="{{ dataminer_algorithms_updater_hour_frequency }}" job="/usr/local/bin/algorithms-updater > {{ smartgears_user_home }}/wps_algorithms_install_log/algorithms_updater_cron.log 2>&1" user='{{ smartgears_user }}' state={{ dataminer_algorithms_updater_cron }}
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_cron' ]
-
-    - name: Create a directory where to install the gebco_08.nc data file
-      file: path={{ smartgears_user_home }}/data state=directory
-
-    - name: Install the gebco_08.nc file from thredds. It is big, mostly 2GB
-      get_url: url={{ dataminer_data_file_from_thredds }} dest={{ smartgears_user_home }}/data/gebco_08.nc timeout=2400
-
-    - name: Create a symlink to the gebco_08.nc file
-      file: src={{ smartgears_user_home }}/data/gebco_08.nc dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/ecocfg/gebco_08.nc state=link
-      notify: Restart smartgears
-
     - name: Install the old properties files
       copy: src={{ item }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/ecocfg/{{ item }} force=yes
       with_items: '{{ dataminer_properties_files }}'
@@ -125,21 +37,30 @@
       file: dest={{ smartgears_user_home }}/wps_algorithms_install_log/already_installed_algorithms.txt state=absent 
       when: dataminer_updated is changed 
 
-    - name: Install the nscd service to work around dns timeouts
-      become_user: root 
-      apt: pkg=nscd state=present update_cache=yes cache_valid_time=1800
-      when: dataminer_install_nscd is defined and dataminer_install_nscd
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ]
-
-    - name: Ensure thad the nscd service is enabled and running
-      become_user: root 
-      service: name=nscd state=restarted enabled=yes
-      when: dataminer_install_nscd is defined and dataminer_install_nscd
-      tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ]
-
-
   become: True
   become_user: '{{ smartgears_user }}'
   when: dataminer_app_install
   tags: [ 'tomcat', 'dataminer', 'wps' ]
 
+- block:
+
+  - name: Install the nscd service to work around dns timeouts
+    become_user: root 
+    apt: pkg=nscd state=present update_cache=yes cache_valid_time=1800
+    when: dataminer_install_nscd is defined and dataminer_install_nscd
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ]
+
+  - name: Ensure thad the nscd service is enabled and running
+    become_user: root 
+    service: name=nscd state=restarted enabled=yes
+    when: dataminer_install_nscd is defined and dataminer_install_nscd
+    tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ]
+
+
+  become: True
+  become_user: '{{ smartgears_user }}'
+  when:
+    - dataminer_app_install
+    - dataminer_install_nscd
+  tags: [ 'tomcat', 'dataminer', 'wps' ]
+

smartgears/dataminer_app/tasks/dataminer-external-files.yml

@@ -0,0 +1,20 @@
+---
+- block:
+
+  - name: Create a directory where to install the gebco_08.nc data file
+    file: path={{ smartgears_user_home }}/data state=directory
+
+  - name: Install the gebco_08.nc file from thredds. It is big, mostly 2GB
+    get_url: url={{ dataminer_data_file_from_thredds }} dest={{ smartgears_user_home }}/data/gebco_08.nc timeout=2400
+
+  - name: Create a symlink to the gebco_08.nc file
+    file: src={{ smartgears_user_home }}/data/gebco_08.nc dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/ecocfg/gebco_08.nc state=link
+    notify: Restart smartgears
+
+  become: True
+  become_user: '{{ smartgears_user }}'
+  when:
+    - dataminer_app_install
+    - dataminer_download_gebco_file
+  tags: [ 'tomcat', 'dataminer', 'wps' ]
+  

smartgears/dataminer_app/tasks/dataminer-upgrade.yml

@@ -6,5 +6,7 @@
     - name: If it is an upgrade, brutally remove the wps installation
       file: dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=absent
 
-  when: dataminer_app_upgrade
+  when:
-  tags: [ 'tomcat', 'dataminer', 'dataminer_upgrade', 'wps' ]
+    - dataminer_app_install
+    - dataminer_app_upgrade
+  tags: [ dataminer', 'dataminer_upgrade', 'wps' ]

smartgears/dataminer_app/tasks/main.yml

@@ -2,5 +2,9 @@
 - import_tasks: dataminer-upgrade.yml
   when: dataminer_app_upgrade
 - import_tasks: dataminer-app.yml
-  when: dataminer_war_install
+  when: dataminer_app_install
+- import_tasks: dataminer-external-files.yml
+  when: dataminer_app_install
 - import_tasks: install-gcube-key.yml
+- import_tasks: dataminer-algorithms-installer.yml
+  when: dataminer_algorithms_installer