diff --git a/smartgears/dataminer_app/defaults/main.yml b/smartgears/dataminer_app/defaults/main.yml index 59985936..19f9b112 100644 --- a/smartgears/dataminer_app/defaults/main.yml +++ b/smartgears/dataminer_app/defaults/main.yml @@ -1,13 +1,13 @@ --- dataminer_app_install: False dataminer_app_upgrade: False +dataminer_algorithms_installer: True # ZIP file dataminer_product: prod dataminer_zip_url: 'http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/webapp/{{ dataminer_product }}/wps.zip' # Install from WAR -dataminer_war_install: True dataminer_wps_version: latest dataminer_wps_name: wps dataminer_wps_group_id: org.gcube.data-analysis @@ -32,10 +32,12 @@ dataminer_algorithms_updater_cron: present dataminer_algorithms_updater_minute_frequency: "0" dataminer_algorithms_updater_hour_frequency: "*" +dataminer_download_gebco_file: True dataminer_data_file_from_thredds: 'http://thredds.d4science.org/thredds/fileServer/public/netcdf/gebco_08_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc' -dataminer_conda_install: True +dataminer_conda_install: False dataminer_hdf5: False +dataminer_install_nscd: False # dev, prod, preprod dataminer_infra_reference: dev diff --git a/smartgears/dataminer_app/meta/main.yml b/smartgears/dataminer_app/meta/main.yml index a1e4dc58..b34d31fe 100644 --- a/smartgears/dataminer_app/meta/main.yml +++ b/smartgears/dataminer_app/meta/main.yml @@ -1,4 +1,5 @@ --- dependencies: + - { role: ../../library/roles/smartgears/smartgears-service } - { role: ../../library/roles/conda, when: dataminer_conda_install } - { role: ../../library/roles/hdf5, when: dataminer_hdf5 } diff --git a/smartgears/dataminer_app/tasks/dataminer-algorithms-installer.yml b/smartgears/dataminer_app/tasks/dataminer-algorithms-installer.yml new file mode 100644 index 00000000..f806e40f --- /dev/null +++ b/smartgears/dataminer_app/tasks/dataminer-algorithms-installer.yml @@ -0,0 +1,77 @@ +--- +# This is only needed to force an upgrade of the algorithm importer script +- block: + - name: Remove the old dataminer algorithm installer to force an upgrade + file: path={{ smartgears_user_home }}/algorithmInstaller state=absent + + - name: Remove the old dataminer algorithm installer distributions from the downloads directory + shell: rm -fr {{ smartgears_user_home }}/downloads/dataminer-algorithms-importer*.tar.gz + + become: True + become_user: '{{ smartgears_user }}' + when: + - dataminer_app_install + - dataminer_algorithms_upgrade + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] + + +- block: + + - name: Verify if we have the old algorithms setup + stat: path={{ smartgears_user_home }}/wps_algorithms/algorithms/AAPS_NASO_PUBLISHER.jar + register: old_algo_setup + + - name: Wipe away the algoriths directory if it comes from the old setup + file: dest={{ smartgears_user_home }}/wps_algorithms state=absent + when: old_algo_setup.stat.exists + + - name: Create a directory where to install the algorithms directory + file: path={{ smartgears_user_home }}/wps_algorithms/algorithms state=directory + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ] + + - name: Install the subversion packages + become_user: root + apt: pkg=subversion state=present update_cache=yes cache_valid_time=1800 + + - name: Download the WPS algorithms from subversion + subversion: repo={{ dataminer_wps_algorithms_svn }} dest={{ smartgears_user_home }}/wps_algorithms/algorithms/{{ dataminer_infra_reference }} checkout=yes force=yes update=yes + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ] + + - name: Get the algorithms installer via maven_artifact + maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} + register: algorithms_installer_downloaded + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] + + - name: Remove the old dataminer algorithm installer + file: path={{ smartgears_user_home }}/algorithmInstaller state=absent + when: algorithms_installer_downloaded is changed + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] + + - name: Unarchive the algorithms installer + unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh' + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] + + - name: Create a directory where to install the algorithms updater logs and data + file: path={{ smartgears_user_home }}/wps_algorithms_install_log state=directory + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ] + + - name: Install a script that updates the algorithms repository and adds the missing algorithms configurations + become_user: "{{ d4science_ansible_become_user | default('root') }}" + template: src=algorithms-updater.j2 dest=/usr/local/bin/algorithms-updater mode=0755 + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ] + + - name: Remove the list of the installed algorithms if we are upgrading from the old algorithms setup + file: dest={{ smartgears_user_home }}/wps_algorithms_install_log/already_installed_algorithms.txt state=absent + when: old_algo_setup.stat.exists + + - name: Cron job that updates the algorithms repository and adds the missing algorithms configurations + cron: name="SVN update the algorithms repository" minute="{{ dataminer_algorithms_updater_minute_frequency }}" hour="{{ dataminer_algorithms_updater_hour_frequency }}" job="/usr/local/bin/algorithms-updater > {{ smartgears_user_home }}/wps_algorithms_install_log/algorithms_updater_cron.log 2>&1" user='{{ smartgears_user }}' state={{ dataminer_algorithms_updater_cron }} + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_cron' ] + + become: True + become_user: '{{ smartgears_user }}' + when: + - dataminer_app_install + - dataminer_algorithms_upgrade + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] + diff --git a/smartgears/dataminer_app/tasks/dataminer-app.yml b/smartgears/dataminer_app/tasks/dataminer-app.yml index 767448cf..97aca129 100644 --- a/smartgears/dataminer_app/tasks/dataminer-app.yml +++ b/smartgears/dataminer_app/tasks/dataminer-app.yml @@ -1,31 +1,4 @@ --- -# This is only needed to force an upgrade -- block: - - name: Remove the old dataminer installation - file: path={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=absent - - become: True - become_user: '{{ smartgears_user }}' - when: - - dataminer_app_install - - dataminer_app_upgrade - tags: [ 'tomcat', 'dataminer', 'wps' ] - -# This is only needed to force an upgrade of the algorithm importer script -- block: - - name: Remove the old dataminer algorithm installer to force an upgrade - file: path={{ smartgears_user_home }}/algorithmInstaller state=absent - - - name: Remove the old dataminer algorithm installer distributions from the downloads directory - shell: rm -fr {{ smartgears_user_home }}/downloads/dataminer-algorithms-importer*.tar.gz - - become: True - become_user: '{{ smartgears_user }}' - when: - - dataminer_app_install - - dataminer_algorithms_upgrade - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] - - block: - name: Get and unpack the dataminer application maven_artifact: artifact_id={{ dataminer_wps_name }} version={{ dataminer_wps_version | default(omit) }} group_id={{ dataminer_wps_group_id }} extension={{ dataminer_wps_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_wps_file }} @@ -47,67 +20,6 @@ template: src=web.xml.j2 dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/WEB-INF/web.xml notify: Restart smartgears - - name: Verify if we have the old algorithms setup - stat: path={{ smartgears_user_home }}/wps_algorithms/algorithms/AAPS_NASO_PUBLISHER.jar - register: old_algo_setup - - - name: Wipe away the algoriths directory if it comes from the old setup - file: dest={{ smartgears_user_home }}/wps_algorithms state=absent - when: old_algo_setup.stat.exists - - - name: Create a directory where to install the algorithms directory - file: path={{ smartgears_user_home }}/wps_algorithms/algorithms state=directory - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ] - - - name: Install the subversion packages - become_user: root - apt: pkg=subversion state=present update_cache=yes cache_valid_time=1800 - - - name: Download the WPS algorithms from subversion - subversion: repo={{ dataminer_wps_algorithms_svn }} dest={{ smartgears_user_home }}/wps_algorithms/algorithms/{{ dataminer_infra_reference }} checkout=yes force=yes update=yes - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_svn' ] - - - name: Get the algorithms installer via maven_artifact - maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} - register: algorithms_installer_downloaded - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] - - - name: Remove the old dataminer algorithm installer - file: path={{ smartgears_user_home }}/algorithmInstaller state=absent - when: algorithms_installer_downloaded is changed - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] - - - name: Unarchive the algorithms installer - unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh' - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] - - - name: Create a directory where to install the algorithms updater logs and data - file: path={{ smartgears_user_home }}/wps_algorithms_install_log state=directory - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ] - - - name: Install a script that updates the algorithms repository and adds the missing algorithms configurations - become_user: "{{ d4science_ansible_become_user | default('root') }}" - template: src=algorithms-updater.j2 dest=/usr/local/bin/algorithms-updater mode=0755 - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ] - - - name: Remove the list of the installed algorithms if we are upgrading from the old algorithms setup - file: dest={{ smartgears_user_home }}/wps_algorithms_install_log/already_installed_algorithms.txt state=absent - when: old_algo_setup.stat.exists - - - name: Cron job that updates the algorithms repository and adds the missing algorithms configurations - cron: name="SVN update the algorithms repository" minute="{{ dataminer_algorithms_updater_minute_frequency }}" hour="{{ dataminer_algorithms_updater_hour_frequency }}" job="/usr/local/bin/algorithms-updater > {{ smartgears_user_home }}/wps_algorithms_install_log/algorithms_updater_cron.log 2>&1" user='{{ smartgears_user }}' state={{ dataminer_algorithms_updater_cron }} - tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_cron' ] - - - name: Create a directory where to install the gebco_08.nc data file - file: path={{ smartgears_user_home }}/data state=directory - - - name: Install the gebco_08.nc file from thredds. It is big, mostly 2GB - get_url: url={{ dataminer_data_file_from_thredds }} dest={{ smartgears_user_home }}/data/gebco_08.nc timeout=2400 - - - name: Create a symlink to the gebco_08.nc file - file: src={{ smartgears_user_home }}/data/gebco_08.nc dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/ecocfg/gebco_08.nc state=link - notify: Restart smartgears - - name: Install the old properties files copy: src={{ item }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/ecocfg/{{ item }} force=yes with_items: '{{ dataminer_properties_files }}' @@ -125,21 +37,30 @@ file: dest={{ smartgears_user_home }}/wps_algorithms_install_log/already_installed_algorithms.txt state=absent when: dataminer_updated is changed - - name: Install the nscd service to work around dns timeouts - become_user: root - apt: pkg=nscd state=present update_cache=yes cache_valid_time=1800 - when: dataminer_install_nscd is defined and dataminer_install_nscd - tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ] - - - name: Ensure thad the nscd service is enabled and running - become_user: root - service: name=nscd state=restarted enabled=yes - when: dataminer_install_nscd is defined and dataminer_install_nscd - tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ] - - become: True become_user: '{{ smartgears_user }}' when: dataminer_app_install tags: [ 'tomcat', 'dataminer', 'wps' ] +- block: + + - name: Install the nscd service to work around dns timeouts + become_user: root + apt: pkg=nscd state=present update_cache=yes cache_valid_time=1800 + when: dataminer_install_nscd is defined and dataminer_install_nscd + tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ] + + - name: Ensure thad the nscd service is enabled and running + become_user: root + service: name=nscd state=restarted enabled=yes + when: dataminer_install_nscd is defined and dataminer_install_nscd + tags: [ 'tomcat', 'dataminer', 'wps', 'dm_nscd' ] + + + become: True + become_user: '{{ smartgears_user }}' + when: + - dataminer_app_install + - dataminer_install_nscd + tags: [ 'tomcat', 'dataminer', 'wps' ] + diff --git a/smartgears/dataminer_app/tasks/dataminer-external-files.yml b/smartgears/dataminer_app/tasks/dataminer-external-files.yml new file mode 100644 index 00000000..d8be5a3a --- /dev/null +++ b/smartgears/dataminer_app/tasks/dataminer-external-files.yml @@ -0,0 +1,20 @@ +--- +- block: + + - name: Create a directory where to install the gebco_08.nc data file + file: path={{ smartgears_user_home }}/data state=directory + + - name: Install the gebco_08.nc file from thredds. It is big, mostly 2GB + get_url: url={{ dataminer_data_file_from_thredds }} dest={{ smartgears_user_home }}/data/gebco_08.nc timeout=2400 + + - name: Create a symlink to the gebco_08.nc file + file: src={{ smartgears_user_home }}/data/gebco_08.nc dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/ecocfg/gebco_08.nc state=link + notify: Restart smartgears + + become: True + become_user: '{{ smartgears_user }}' + when: + - dataminer_app_install + - dataminer_download_gebco_file + tags: [ 'tomcat', 'dataminer', 'wps' ] + \ No newline at end of file diff --git a/smartgears/dataminer_app/tasks/dataminer-upgrade.yml b/smartgears/dataminer_app/tasks/dataminer-upgrade.yml index 1db229e1..aded6302 100644 --- a/smartgears/dataminer_app/tasks/dataminer-upgrade.yml +++ b/smartgears/dataminer_app/tasks/dataminer-upgrade.yml @@ -6,5 +6,7 @@ - name: If it is an upgrade, brutally remove the wps installation file: dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=absent - when: dataminer_app_upgrade - tags: [ 'tomcat', 'dataminer', 'dataminer_upgrade', 'wps' ] + when: + - dataminer_app_install + - dataminer_app_upgrade + tags: [ dataminer', 'dataminer_upgrade', 'wps' ] diff --git a/smartgears/dataminer_app/tasks/main.yml b/smartgears/dataminer_app/tasks/main.yml index d044658b..08a82969 100644 --- a/smartgears/dataminer_app/tasks/main.yml +++ b/smartgears/dataminer_app/tasks/main.yml @@ -2,5 +2,9 @@ - import_tasks: dataminer-upgrade.yml when: dataminer_app_upgrade - import_tasks: dataminer-app.yml - when: dataminer_war_install + when: dataminer_app_install +- import_tasks: dataminer-external-files.yml + when: dataminer_app_install - import_tasks: install-gcube-key.yml +- import_tasks: dataminer-algorithms-installer.yml + when: dataminer_algorithms_installer