From 24e5f76098b3d0157e9cfc8324558e44ece3aac4 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Mon, 13 Jun 2016 16:24:30 +0200 Subject: [PATCH] library/roles/ghn-gcore, library/roles/smartgears: Move the ghn and smartgears roles from d4science-ghn-cluster to the shared library --- ghn-gcore/gcore-upgrade/tasks/main.yml | 15 + ghn-gcore/ghn-gcoreboot/tasks/main.yml | 19 ++ ghn-gcore/ghn-gcoreboot/templates/gcore.j2 | 33 +++ ghn-gcore/ghn-update/tasks/main.yml | 27 ++ ghn-gcore/provide-gcube-key/handlers/main.yml | 8 + ghn-gcore/provide-gcube-key/tasks/main.yml | 10 + .../couchdb_cache_se_plugin/defaults/main.yml | 6 + .../couchdb_cache_se_plugin/handlers/main.yml | 3 + .../couchdb_cache_se_plugin/tasks/main.yml | 9 + smartgears/dataminer_app/defaults/main.yml | 50 ++++ smartgears/dataminer_app/handlers/main.yml | 3 + .../dataminer_app/tasks/dataminer-app.yml | 105 +++++++ .../dataminer_app/tasks/dataminer-upgrade.yml | 11 + .../dataminer_app/tasks/install-gcube-key.yml | 20 ++ smartgears/dataminer_app/tasks/main.yml | 6 + smartgears/dataminer_app/tasks/w52n-app.yml | 61 ++++ .../templates/dataminer-pom.xml.j2 | 15 + .../templates/maven-settings.xml.j2 | 93 ++++++ smartgears/dataminer_app/templates/pom.xml.j2 | 267 ++++++++++++++++++ .../dataminer_app/templates/wps_config.xml.j2 | 113 ++++++++ smartgears/generic_worker/defaults/main.yml | 9 + smartgears/generic_worker/handlers/main.yml | 3 + smartgears/generic_worker/tasks/main.yml | 9 + smartgears/install-gcube-key/tasks/main.yml | 19 ++ .../is_sweeper_se_plugin/defaults/main.yml | 5 + .../is_sweeper_se_plugin/handlers/main.yml | 3 + .../is_sweeper_se_plugin/tasks/main.yml | 9 + smartgears/smart_executor/defaults/main.yml | 24 ++ smartgears/smart_executor/handlers/main.yml | 3 + smartgears/smart_executor/tasks/main.yml | 41 +++ .../templates/smart_executor-container.xml.j2 | 34 +++ .../smartgears-nginx-frontend/meta/main.yml | 3 + .../smartgears-nginx-frontend/tasks/main.yml | 22 ++ ...aminer1-d-d4s.d4science.org-virtualhost.j2 | 65 +++++ ...aminer1-p-d4s.d4science.org-virtualhost.j2 | 1 + ...aminer2-d-d4s.d4science.org-virtualhost.j2 | 1 + ...aminer2-p-d4s.d4science.org-virtualhost.j2 | 1 + .../egi-d4s-smartgears-virtualhost.j2 | 1 + .../generic-smartgears-virtualhost.j2 | 58 ++++ .../tagme.d4science.org-virtualhost.j2 | 139 +++++++++ smartgears/smartgears/defaults/main.yml | 50 ++++ smartgears/smartgears/handlers/main.yml | 3 + smartgears/smartgears/tasks/main.yml | 3 + .../smartgears/tasks/smartgears-app.yml | 71 +++++ .../smartgears/templates/container.xml.j2 | 27 ++ .../templates/tomcat-instance-default.j2 | 2 + .../smartgears_egi_image/tasks/main.yml | 4 + .../tasks/setup-egi-image.yml | 9 + .../templates/get-gcube-keys.sh.j2 | 43 +++ .../templates/smartgears-setup.sh.j2 | 106 +++++++ .../defaults/main.yml | 7 + .../handlers/main.yml | 3 + .../tasks/main.yml | 9 + 53 files changed, 1661 insertions(+) create mode 100644 ghn-gcore/gcore-upgrade/tasks/main.yml create mode 100644 ghn-gcore/ghn-gcoreboot/tasks/main.yml create mode 100644 ghn-gcore/ghn-gcoreboot/templates/gcore.j2 create mode 100644 ghn-gcore/ghn-update/tasks/main.yml create mode 100644 ghn-gcore/provide-gcube-key/handlers/main.yml create mode 100644 ghn-gcore/provide-gcube-key/tasks/main.yml create mode 100644 smartgears/couchdb_cache_se_plugin/defaults/main.yml create mode 100644 smartgears/couchdb_cache_se_plugin/handlers/main.yml create mode 100644 smartgears/couchdb_cache_se_plugin/tasks/main.yml create mode 100644 smartgears/dataminer_app/defaults/main.yml create mode 100644 smartgears/dataminer_app/handlers/main.yml create mode 100644 smartgears/dataminer_app/tasks/dataminer-app.yml create mode 100644 smartgears/dataminer_app/tasks/dataminer-upgrade.yml create mode 100644 smartgears/dataminer_app/tasks/install-gcube-key.yml create mode 100644 smartgears/dataminer_app/tasks/main.yml create mode 100644 smartgears/dataminer_app/tasks/w52n-app.yml create mode 100644 smartgears/dataminer_app/templates/dataminer-pom.xml.j2 create mode 100644 smartgears/dataminer_app/templates/maven-settings.xml.j2 create mode 100644 smartgears/dataminer_app/templates/pom.xml.j2 create mode 100644 smartgears/dataminer_app/templates/wps_config.xml.j2 create mode 100644 smartgears/generic_worker/defaults/main.yml create mode 100644 smartgears/generic_worker/handlers/main.yml create mode 100644 smartgears/generic_worker/tasks/main.yml create mode 100644 smartgears/install-gcube-key/tasks/main.yml create mode 100644 smartgears/is_sweeper_se_plugin/defaults/main.yml create mode 100644 smartgears/is_sweeper_se_plugin/handlers/main.yml create mode 100644 smartgears/is_sweeper_se_plugin/tasks/main.yml create mode 100644 smartgears/smart_executor/defaults/main.yml create mode 100644 smartgears/smart_executor/handlers/main.yml create mode 100644 smartgears/smart_executor/tasks/main.yml create mode 100644 smartgears/smart_executor/templates/smart_executor-container.xml.j2 create mode 100644 smartgears/smartgears-nginx-frontend/meta/main.yml create mode 100644 smartgears/smartgears-nginx-frontend/tasks/main.yml create mode 100644 smartgears/smartgears-nginx-frontend/templates/dataminer1-d-d4s.d4science.org-virtualhost.j2 create mode 120000 smartgears/smartgears-nginx-frontend/templates/dataminer1-p-d4s.d4science.org-virtualhost.j2 create mode 120000 smartgears/smartgears-nginx-frontend/templates/dataminer2-d-d4s.d4science.org-virtualhost.j2 create mode 120000 smartgears/smartgears-nginx-frontend/templates/dataminer2-p-d4s.d4science.org-virtualhost.j2 create mode 120000 smartgears/smartgears-nginx-frontend/templates/egi-d4s-smartgears-virtualhost.j2 create mode 100644 smartgears/smartgears-nginx-frontend/templates/generic-smartgears-virtualhost.j2 create mode 100644 smartgears/smartgears-nginx-frontend/templates/tagme.d4science.org-virtualhost.j2 create mode 100644 smartgears/smartgears/defaults/main.yml create mode 100644 smartgears/smartgears/handlers/main.yml create mode 100644 smartgears/smartgears/tasks/main.yml create mode 100644 smartgears/smartgears/tasks/smartgears-app.yml create mode 100644 smartgears/smartgears/templates/container.xml.j2 create mode 100644 smartgears/smartgears/templates/tomcat-instance-default.j2 create mode 100644 smartgears/smartgears_egi_image/tasks/main.yml create mode 100644 smartgears/smartgears_egi_image/tasks/setup-egi-image.yml create mode 100644 smartgears/smartgears_egi_image/templates/get-gcube-keys.sh.j2 create mode 100755 smartgears/smartgears_egi_image/templates/smartgears-setup.sh.j2 create mode 100644 smartgears/social_data_indexer_se_plugin/defaults/main.yml create mode 100644 smartgears/social_data_indexer_se_plugin/handlers/main.yml create mode 100644 smartgears/social_data_indexer_se_plugin/tasks/main.yml diff --git a/ghn-gcore/gcore-upgrade/tasks/main.yml b/ghn-gcore/gcore-upgrade/tasks/main.yml new file mode 100644 index 00000000..cf17849b --- /dev/null +++ b/ghn-gcore/gcore-upgrade/tasks/main.yml @@ -0,0 +1,15 @@ +--- +- name: Download the Gcore upgrade script + become: True + become_user: '{{ item }}' + get_url: url={{ gcore_upgrade_script_url }} dest=/home/{{ item }} validate_certs=no + with_items: '{{ gcube_users }}' + tags: gcore_upgrade + +- name: Run the Gcore upgrade script + become: True + become_user: '{{ item }}' + shell: . /home/{{ item }}/.bashrc ; cd /home/{{ item }} ; chmod 555 ./{{ gcore_upgrade_script }} ; ./{{ gcore_upgrade_script }} {{ http_port }} {{ restart_on_upgrade }} force + with_items: '{{ gcube_users }}' + tags: gcore_upgrade + diff --git a/ghn-gcore/ghn-gcoreboot/tasks/main.yml b/ghn-gcore/ghn-gcoreboot/tasks/main.yml new file mode 100644 index 00000000..fd887bf3 --- /dev/null +++ b/ghn-gcore/ghn-gcoreboot/tasks/main.yml @@ -0,0 +1,19 @@ +--- +# Install the gcore boot script +- name: Install the gcore boot script + template: src=gcore.j2 dest=/etc/init.d/{{ item }} owner=root group=root mode=0755 + with_items: 'gcore' + when: + - gcoreboot_install + tags: [ 'gcoreboot' ] + +- name: Enable the gcore boot script + file: src=/etc/init.d/{{ item }} dest=/etc/rc2.d/S95gcore state=link + file: src=/etc/init.d/{{ item }} dest=/etc/rc3.d/S95gcore state=link + file: src=/etc/init.d/{{ item }} dest=/etc/rc0.d/K25gcore state=link + file: src=/etc/init.d/{{ item }} dest=/etc/rc1.d/K25gcore state=link + with_items: 'gcore' + when: + - gcoreboot_install + tags: [ 'gcoreboot' ] + diff --git a/ghn-gcore/ghn-gcoreboot/templates/gcore.j2 b/ghn-gcore/ghn-gcoreboot/templates/gcore.j2 new file mode 100644 index 00000000..2b23dbac --- /dev/null +++ b/ghn-gcore/ghn-gcoreboot/templates/gcore.j2 @@ -0,0 +1,33 @@ +#!/bin/bash +SELF=$(cd $(dirname $0); pwd -P)/$(basename $0) +case "${1:-''}" in + 'start') + test=$(cat /etc/passwd | cut -d":" -f1 | grep gcube) + for elemento in $test; do + echo "entering on $elemento" + path="/home/$elemento/startContainer.sh" + if [ -e "$path" ] + then + su $elemento -c "source $path" + fi + echo "exiting from $elemento" + done + ;; + 'stop') + test=$(cat /etc/passwd | cut -d":" -f1 | grep gcube) + for elemento in $test; do + echo "entering on $elemento" + path="/home/$elemento/stopContainer.sh" + if [ -e "$path" ] + then + su $elemento -c "source $path" + fi + echo "exiting from $elemento" + done + ;; +*) + echo "Usage: $SELF {start|stop}" + exit 1 + ;; +esac + diff --git a/ghn-gcore/ghn-update/tasks/main.yml b/ghn-gcore/ghn-update/tasks/main.yml new file mode 100644 index 00000000..c75b9cbf --- /dev/null +++ b/ghn-gcore/ghn-update/tasks/main.yml @@ -0,0 +1,27 @@ +--- +# Old stuff, do not use +# +- name: Stop the gcube service + shell: /home/{{ gcube_user }}/stopContainer.sh >/dev/null 2>&1 + #shell: su - {{ gcube_user }} /home/{{ gcube_user }}/gCore/bin/gcore-stop-container + ignore_errors: True + tags: + - ghn + +- name: Install the new ghn release + action: ghn_r_cluster user={{ gcube_user }} + tags: + - ghn + +- name: Replace the hostname in the gCore/etc/globus_wsrf_core/server-config.wsdd configuration file + lineinfile: dest=/home/{{ gcube_user }}/gCore/etc/globus_wsrf_core/server-config.wsdd regexp="" line="" + tags: + - ghn + - ghnconfig + +- name: Replace JAVA_HOME in /home/{{ gcube_user }}/.gcorerc + lineinfile: dest=/home/{{ gcube_user }}/.gcorerc regexp="^export JAVA_HOME=.*" line="export JAVA_HOME={{ jdk_java_home }}" + tags: + - ghn + - ghnconfig + - jdk diff --git a/ghn-gcore/provide-gcube-key/handlers/main.yml b/ghn-gcore/provide-gcube-key/handlers/main.yml new file mode 100644 index 00000000..8184f347 --- /dev/null +++ b/ghn-gcore/provide-gcube-key/handlers/main.yml @@ -0,0 +1,8 @@ +--- +- name: Restart ghn webservice + shell: /etc/init.d/gcore stop ; /etc/init.d/gcore start + when: + - restart_gcore_service + - ( has_tomcat is defined ) and ( not has_tomcat ) + ignore_errors: True + diff --git a/ghn-gcore/provide-gcube-key/tasks/main.yml b/ghn-gcore/provide-gcube-key/tasks/main.yml new file mode 100644 index 00000000..39049da5 --- /dev/null +++ b/ghn-gcore/provide-gcube-key/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Install the gcube keys + copy: src=/tmp/symm.key dest=/home/{{ item.0 }}/{{ gcube_conf_path }}/{{ item.1 }} owner={{ item.0 }} group={{ item.0 }} mode=0400 + with_nested: + - '{{ gcube_users }}' + - '{{ gcube_symm_key_file }}' + notify: Restart ghn webservice + tags: + - gcube_key + diff --git a/smartgears/couchdb_cache_se_plugin/defaults/main.yml b/smartgears/couchdb_cache_se_plugin/defaults/main.yml new file mode 100644 index 00000000..d1b6feca --- /dev/null +++ b/smartgears/couchdb_cache_se_plugin/defaults/main.yml @@ -0,0 +1,6 @@ +--- +couchdb_cache_se_plugin_install: False +#couchdb_cache_se_plugin_ver: 1.0.0-3.10.1 +couchdb_cache_se_plugin_ver: 1.1.0-3.11.0-128111 +couchdb_cache_se_plugin_name: 'couchdb-cache-se-plugin-{{ couchdb_cache_se_plugin_ver }}-jar-with-dependencies.jar' +couchdb_cache_se_plugin_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/accounting/couchdb-cache-se-plugin/{{ couchdb_cache_se_plugin_ver }}/{{ couchdb_cache_se_plugin_name }}' diff --git a/smartgears/couchdb_cache_se_plugin/handlers/main.yml b/smartgears/couchdb_cache_se_plugin/handlers/main.yml new file mode 100644 index 00000000..a9b4ffdd --- /dev/null +++ b/smartgears/couchdb_cache_se_plugin/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name='tomcat-instance-{{ smartgears_http_port }}' state=restarted diff --git a/smartgears/couchdb_cache_se_plugin/tasks/main.yml b/smartgears/couchdb_cache_se_plugin/tasks/main.yml new file mode 100644 index 00000000..0e9fdb3a --- /dev/null +++ b/smartgears/couchdb_cache_se_plugin/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Get the couchdb cache se plugin and install it inside the smart executor + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ couchdb_cache_se_plugin_url }} dest={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }}/WEB-INF/lib + when: couchdb_cache_se_plugin_install + notify: Restart smartgears + tags: [ 'smartgears', 'couchdb_cache_se', 'tomcat' ] + diff --git a/smartgears/dataminer_app/defaults/main.yml b/smartgears/dataminer_app/defaults/main.yml new file mode 100644 index 00000000..e13c2994 --- /dev/null +++ b/smartgears/dataminer_app/defaults/main.yml @@ -0,0 +1,50 @@ +--- +dataminer_app_install: False +dataminer_app_upgrade: False + +dataminer_52north_webapp_name: wps +dataminer_52north_webapp_context: '/{{ dataminer_52north_webapp_name }}' + +dataminer_52north_webapp_file: 52n-wps-webapp-3.3.2-20150523.010021-9.war +#dataminer_52north_webapp_url: 'http://build.dev.52north.org/jenkins/view/Processing/job/52n-wps_GIT-DEV_nightly_builds/lastSuccessfulBuild/org.n52.wps$52n-wps-webapp/artifact/org.n52.wps/52n-wps-webapp/3.3.2-20150523.010021-9/{{ dataminer_52north_webapp_file }}' +dataminer_52north_webapp_url: 'http://data.d4science.org/uri-resolver/id?fileName=wps-3.3.2.war&smp-id=565d67b7e4b0eacf4a0fc5ad&contentType=application%2Fx-tika-java-web-archive' +dataminer_maven_enable_gcube_snapshots: False +#dataminer_dataminer_jar_version: 1.0.1-3.10.0 +dataminer_dataminer_jar_version: 1.0.2-3.10.1 +dataminer_dataminer_jar_version_range: "[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)" + +dataminer_gcube_web_xml: "http://data.d4science.org/id?fileName=web.xml&smp-id=56615ae0e4b0158fcb561817&contentType=application%2Fxml" +dataminer_gcube_app_xml: "http://data.d4science.org/id?fileName=gcube-app.xml&smp-id=56615ae0e4b0158fcb561815&contentType=application%2Fxml" + +dataminer_jars_to_remove: + - 52n-wps-server-3.3.2-SNAPSHOT.jar + - 52n-wps-io-impl-3.3.1.jar + - 52n-wps-database-3.3.1.jar + - 52n-wps-server-3.3.1.jar + - 52n-wps-algorithm-3.3.1.jar + - 52n-wps-commons-3.3.1.jar + - 52n-wps-io-3.3.1.jar + - commons-beanutils-1.7.0.jar + - dom4j-1.6.jar + - jaxen-1.1.1.jar + - jcl-over-slf4j-1.6.4.jar + - jdom-1.0.jar + - logback-core-1.0.13.jar + - slf4j-api-1.7.5.jar + - spring-beans-2.0.6.jar + - spring-context-2.0.6.jar + - spring-core-2.0.6.jar + - registry-publisher-1.2.5-3.10.0.jar + +dataminer_ecocfg_svn_repo: 'http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/cfg' + +dataminer_ecocfg_parallelproc_svn_repo: 'http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/PARALLEL_PROCESSING' + +# Retrieve and modify in a template before each installation. +dataminer_wps_config: 'http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/wpscfg/wps_config.xml' + +# Those packages are needed to download and configure the dataminer application +dataminer_additional_packages: + - subversion + - maven + diff --git a/smartgears/dataminer_app/handlers/main.yml b/smartgears/dataminer_app/handlers/main.yml new file mode 100644 index 00000000..69c07b56 --- /dev/null +++ b/smartgears/dataminer_app/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name=tomcat-instance-{{ smartgears_http_port }} state=restarted diff --git a/smartgears/dataminer_app/tasks/dataminer-app.yml b/smartgears/dataminer_app/tasks/dataminer-app.yml new file mode 100644 index 00000000..d4e053b5 --- /dev/null +++ b/smartgears/dataminer_app/tasks/dataminer-app.yml @@ -0,0 +1,105 @@ +--- +- name: Install some packages needed to download the dataminer environment + apt: pkg={{ item }} state=present + with_items: '{{ dataminer_additional_packages }}' + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer' ] + +- name: Ensure that the maven configuration directory exists for the smartgears user + become: True + become_user: '{{ smartgears_user }}' + file: path={{ smartgears_user_home }}/.m2 state=directory mode=0755 + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Create a directory where to install the pom file needed to download the dataminer jar + become: True + become_user: '{{ smartgears_user }}' + file: path={{ smartgears_user_home }}/.m2/dataminer state=directory mode=0755 + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install the maven configuration + become: True + become_user: '{{ smartgears_user }}' + template: src=maven-settings.xml.j2 dest={{ smartgears_user_home }}/.m2/settings.xml mode=0640 + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install a fake maven POM file to download the dataminer app dependencies + become: True + become_user: '{{ smartgears_user }}' + template: src=pom.xml.j2 dest={{ smartgears_user_home }}/.m2/pom.xml mode=0640 + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install a fake maven POM file to download the dataminer jar + become: True + become_user: '{{ smartgears_user }}' + template: src=dataminer-pom.xml.j2 dest={{ smartgears_user_home }}/.m2/dataminer/pom.xml mode=0640 + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install the rapidminer-custom jar + become: True + become_user: '{{ smartgears_user }}' + shell: mvn org.apache.maven.plugins:maven-dependency-plugin:2.8:copy -Dartifact=rapidminer-custom:52n-wps-server:3.3.2 -DoutputDirectory={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib + args: + creates: '{{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib/52n-wps-server-3.3.2.jar' + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install the org.gcube.dataanalysis dataminer jar + become: True + become_user: '{{ smartgears_user }}' + shell: cd {{ smartgears_user_home }}/.m2/dataminer ; mvn org.apache.maven.plugins:maven-dependency-plugin:2.8:copy -Dartifact=org.gcube.dataanalysis:dataminer:{{ dataminer_dataminer_jar_version }} -DoutputDirectory={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib + args: + creates: '{{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib/dataminer-{{ dataminer_dataminer_jar_version }}.jar' + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install the org.gcube.dataanalysis dataminer jar dependencies + become: True + become_user: '{{ smartgears_user }}' + shell: cd {{ smartgears_user_home }}/.m2 ; mvn -DexcludeScope=provided org.apache.maven.plugins:maven-dependency-plugin:2.8:copy-dependencies -Dartifact=org.gcube.dataanalysis:dataminer:{{ dataminer_dataminer_jar_version }} -DoutputDirectory={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib + args: + creates: '{{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib/ecological-engine-1.8.6-3.9.0.jar' + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Remove the unwanted jars from the 52 north webapp distribution and from the org.gcube.dataanalysis dataminer dependencies + become: True + become_user: '{{ smartgears_user }}' + file: dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/lib/{{ item }} state=absent + with_items: '{{ dataminer_jars_to_remove }}' + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_deps_cleanup' ] + +- name: Download the ecocfg config from subversion + become: True + become_user: '{{ smartgears_user }}' + subversion: repo={{ dataminer_ecocfg_svn_repo }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/ecocfg force=yes + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Download the ecocfg parallel processing libraries from subversion + become: True + become_user: '{{ smartgears_user }}' + subversion: repo={{ dataminer_ecocfg_parallelproc_svn_repo }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/ecocfg/PARALLEL_PROCESSING force=yes + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Install the wps config file + become: True + become_user: '{{ smartgears_user }}' + template: src=wps_config.xml.j2 dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/config/wps_config.xml mode=0444 + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + diff --git a/smartgears/dataminer_app/tasks/dataminer-upgrade.yml b/smartgears/dataminer_app/tasks/dataminer-upgrade.yml new file mode 100644 index 00000000..4a41894a --- /dev/null +++ b/smartgears/dataminer_app/tasks/dataminer-upgrade.yml @@ -0,0 +1,11 @@ +--- +- name: If it is an upgrade, shut down the server + service: name=tomcat-instance-{{ smartgears_http_port }} state=stopped + when: dataminer_app_upgrade + tags: [ 'tomcat', 'dataminer', 'dataminer_upgrade' ] + +- name: If it is an upgrade, brutally remove the wps installation + file: dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }} state=absent + when: dataminer_app_upgrade + tags: [ 'tomcat', 'dataminer', 'dataminer_upgrade' ] + diff --git a/smartgears/dataminer_app/tasks/install-gcube-key.yml b/smartgears/dataminer_app/tasks/install-gcube-key.yml new file mode 100644 index 00000000..9d6e9464 --- /dev/null +++ b/smartgears/dataminer_app/tasks/install-gcube-key.yml @@ -0,0 +1,20 @@ +--- +- name: Install the production gcube keys + get_url: url={{ item.url }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/ecocfg/PARALLEL_PROCESSING owner={{ smartgears_user }} group={{ smartgears_user }} mode=0400 + with_items: + - '{{ gcube_prod_key_1 }}' + - '{{ gcube_prod_key_2 }}' + - '{{ gcube_prod_key_3 }}' + notify: Restart smartgears + when: install_gcube_prod_key + tags: [ 'dataminer', 'gcube_key' ] + +- name: Install the devel gcube keys + get_url: url={{ item.url }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/ecocfg/PARALLEL_PROCESSING owner={{ smartgears_user }} group={{ smartgears_user }} mode=0400 + with_items: + - '{{ gcube_dev_key_1 }}' + - '{{ gcube_dev_key_2 }}' + - '{{ gcube_dev_key_3 }}' + notify: Restart smartgears + when: install_gcube_dev_key + tags: [ 'dataminer', 'gcube_key' ] diff --git a/smartgears/dataminer_app/tasks/main.yml b/smartgears/dataminer_app/tasks/main.yml new file mode 100644 index 00000000..a18b39bd --- /dev/null +++ b/smartgears/dataminer_app/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- include: dataminer-upgrade.yml + when: dataminer_app_upgrade +- include: w52n-app.yml +- include: dataminer-app.yml +- include: install-gcube-key.yml diff --git a/smartgears/dataminer_app/tasks/w52n-app.yml b/smartgears/dataminer_app/tasks/w52n-app.yml new file mode 100644 index 00000000..16f668e3 --- /dev/null +++ b/smartgears/dataminer_app/tasks/w52n-app.yml @@ -0,0 +1,61 @@ +--- +# NOTE: Install as the smartgears user so we do not mess with the permissions +- name: Get the 52 north application war file + remote_user: root + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ dataminer_52north_webapp_url }} dest={{ smartgears_user_home }}/{{ dataminer_52north_webapp_file }} + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Create the 52 north webapp directory and subdirectories + remote_user: root + become: True + become_user: '{{ smartgears_user }}' + file: dest={{ smartgears_instance_path }}/webapps/{{ item }} state=directory mode=0755 + with_items: + - '{{ dataminer_52north_webapp_name }}' + - '{{ dataminer_52north_webapp_name }}/persistence' +# - '{{ dataminer_52north_webapp_name }}/ecocfg' + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Unarchive the 52 north webapp distribution + remote_user: root + become: True + become_user: '{{ smartgears_user }}' + unarchive: copy=no src={{ smartgears_user_home }}/{{ dataminer_52north_webapp_file }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }} + args: + creates: '{{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/classes/52N_license.txt' + register: install_52_north + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Remove the original web.xml file + remote_user: root + become: True + become_user: '{{ smartgears_user }}' + file: dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/web.xml state=absent + notify: Restart smartgears + when: ( install_52_north | changed ) + tags: [ 'tomcat', 'dataminer', 'wps' ] + +# Transform the 52n-wps app in a gcube enabled app +- name: Download the gcube web.xml config file + remote_user: root + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ dataminer_gcube_web_xml }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/web.xml + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] + +- name: Download the gcube gcube-app.xml config file + remote_user: root + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ dataminer_gcube_app_xml }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_52north_webapp_name }}/WEB-INF/gcube-app.xml + notify: Restart smartgears + when: dataminer_app_install + tags: [ 'tomcat', 'dataminer', 'wps' ] diff --git a/smartgears/dataminer_app/templates/dataminer-pom.xml.j2 b/smartgears/dataminer_app/templates/dataminer-pom.xml.j2 new file mode 100644 index 00000000..f7d409d0 --- /dev/null +++ b/smartgears/dataminer_app/templates/dataminer-pom.xml.j2 @@ -0,0 +1,15 @@ + + + 4.0.0 + org.gcube + dataminer + jar + {{ dataminer_dataminer_jar_version }} + + + org.gcube.dataanalysis + dataminer + {{ dataminer_dataminer_jar_version }} + + + diff --git a/smartgears/dataminer_app/templates/maven-settings.xml.j2 b/smartgears/dataminer_app/templates/maven-settings.xml.j2 new file mode 100644 index 00000000..97d5acf0 --- /dev/null +++ b/smartgears/dataminer_app/templates/maven-settings.xml.j2 @@ -0,0 +1,93 @@ + + + + + gcube-snapshots + + + + + + + gcube + + + gcube-snapshots + gCube Snapshots + http://maven.research-infrastructures.eu/nexus/content/repositories/gcube-snapshots + + false + + + {{ dataminer_maven_enable_gcube_snapshots }} + + + + gcube-releases + gCube Releases + http://maven.research-infrastructures.eu/nexus/content/repositories/gcube-releases + + false + + + true + + + + gcube-externals + gCube Externals + http://maven.research-infrastructures.eu/nexus/content/repositories/gcube-externals + + false + + + true + + + + + + + gcube-snapshots + gCube Snapshots + http://maven.research-infrastructures.eu/nexus/content/repositories/gcube-snapshots + + false + + + {{ dataminer_maven_enable_gcube_snapshots }} + + + + gcube-releases + gCube Releases + http://maven.research-infrastructures.eu/nexus/content/repositories/gcube-releases + + false + + + true + + + + gcube-externals + gCube Externals + http://maven.research-infrastructures.eu/nexus/content/repositories/gcube-externals + + false + + + true + + + + + + + + + gcube + + + diff --git a/smartgears/dataminer_app/templates/pom.xml.j2 b/smartgears/dataminer_app/templates/pom.xml.j2 new file mode 100644 index 00000000..31c169ac --- /dev/null +++ b/smartgears/dataminer_app/templates/pom.xml.j2 @@ -0,0 +1,267 @@ + + 4.0.0 + + maven-parent + org.gcube.tools + 1.0.0 + + + org.gcube.dataanalysis + dataminer + {{ dataminer_dataminer_jar_version }} + dataminer + dataminer + + ${project.basedir}/distro + + + + org.gcube.dataaccess.algorithms + database-rm-algorithms + [1.0.0-SNAPSHOT,1.4.0-SNAPSHOT) + + + slf4j-api + org.slf4j + + + common-encryption + org.gcube.core + + + + + ch.qos.logback + logback-classic + 1.1.2 + + + org.n52.wps + 52n-wps-io + [3.3.1,4.0.0) + + + slf4j-api + org.slf4j + + + + + org.n52.wps + 52n-wps-io-impl + [3.3.1,4.0.0) + + + slf4j-api + org.slf4j + + + + + org.n52.wps + 52n-wps-algorithm + [3.3.1,4.0.0) + + + slf4j-api + org.slf4j + + + + + org.n52.wps + 52n-wps-server + [3.3.1,4.0.0) + + + slf4j-api + org.slf4j + + + + + + org.gcube.dataanalysis + ecological-engine-wps-extension + [1.0.2-SNAPSHOT,2.0.0-SNAPSHOT) + + + slf4j-api + org.slf4j + + + + + org.gcube.dataanalysis + ecological-engine-geospatial-extensions + [1.3.2-SNAPSHOT,2.0.0-SNAPSHOT) + + + slf4j-api + org.slf4j + + + + + org.gcube.dataanalysis + ecological-engine-external-algorithms + [1.1.5-SNAPSHOT,2.0.0-SNAPSHOT) + + + slf4j-api + org.slf4j + + + common-utils-encryption + org.gcube.core + + + common-encryption + org.gcube.core + + + + + + org.gcube.dataanalysis + ecological-engine-smart-executor + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + slf4j-api + org.slf4j + + + + + + org.slf4j + slf4j-nop + 1.7.10 + + + slf4j-api + org.slf4j + + + + + junit + junit + 4.11 + + + + org.gcube.common + common-authorization + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + provided + + + slf4j-api + org.slf4j + + + common-utils-encryption + org.gcube.core + + + + + org.gcube.core + common-scope + [1.2.0-SNAPSHOT,2.0.0-SNAPSHOT) + provided + + + slf4j-api + org.slf4j + + + + + + + + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + true + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-profile + install + + copy-resources + + + target + + + ${distroDirectory} + true + + profile.xml + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.3 + + + ${distroDirectory}/descriptor.xml + + + + + servicearchive + install + + single + + + + + + + + + n52-releases + 52n Releases + http://52north.org/maven/repo/releases + + true + + + false + + + + diff --git a/smartgears/dataminer_app/templates/wps_config.xml.j2 b/smartgears/dataminer_app/templates/wps_config.xml.j2 new file mode 100644 index 00000000..f721481f --- /dev/null +++ b/smartgears/dataminer_app/templates/wps_config.xml.j2 @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.clusterers.DBSCAN +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.clusterers.KMEANS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.clusterers.LOF +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.clusterers.XMEANS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.evaluators.DISCREPANCY_ANALYSIS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.evaluators.HRS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.evaluators.MAPS_COMPARISON +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.evaluators.QUALITY_ANALYSIS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.generators.BIONYM +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.generators.CMSY +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.generators.ICCAT_VPA +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ABSENCE_CELLS_FROM_AQUAMAPS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.BIONYM_LOCAL +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.CSQUARE_COLUMN_CREATOR +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ESRI_GRID_EXTRACTION +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.FAO_OCEAN_AREA_COLUMN_CREATOR +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.FAO_OCEAN_AREA_COLUMN_CREATOR_FROM_QUADRANT +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.GENERIC_CHARTS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.GEO_CHART +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.HCAF_FILTER +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.MAX_ENT_NICHE_MODELLING +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_MOST_OBSERVED_SPECIES +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_MOST_OBSERVED_TAXA +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_SINGLE_SPECIES_DISTRIBUTION_PER_AREA +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_SPECIES_OBSERVATIONS_PER_LME_AREA +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_SPECIES_OBSERVATIONS_PER_MEOW_AREA +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_SPECIES_OBSERVATIONS_PER_YEAR +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OBIS_TAXA_OBSERVATIONS_PER_YEAR +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.OCCURRENCE_ENRICHMENT +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PRESENCE_CELLS_GENERATION +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.RASTER_DATA_PUBLISHER +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.SGVM_INTERPOLATION +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.SUBMITQUERY +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.TIMEEXTRACTION +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.TIMEEXTRACTION_TABLE +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.TIME_GEO_CHART +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.TIME_SERIES_ANALYSIS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.TIME_SERIES_CHARTS +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.XYEXTRACTOR +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.XYEXTRACTOR_TABLE +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ZEXTRACTION +org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.ZEXTRACTION_TABLE +org.n52.wps.demo.TestIO + + + + + + + + + diff --git a/smartgears/generic_worker/defaults/main.yml b/smartgears/generic_worker/defaults/main.yml new file mode 100644 index 00000000..757ee673 --- /dev/null +++ b/smartgears/generic_worker/defaults/main.yml @@ -0,0 +1,9 @@ +--- +generic_worker_install: False + +#smart_generic_worker_plugin_ver: 1.0.1-3.9.0 +#smart_generic_worker_plugin_ver: 1.0.2-3.10.1 +smart_generic_worker_plugin_ver: 1.0.3-3.11.0-128830 +smart_generic_worker_plugin_name: 'smart-generic-worker-{{ smart_generic_worker_plugin_ver }}-jar-with-dependencies.jar' +smart_generic_worker_plugin_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/dataanalysis/smart-generic-worker/{{ smart_generic_worker_plugin_ver }}/{{ smart_generic_worker_plugin_name }}' + diff --git a/smartgears/generic_worker/handlers/main.yml b/smartgears/generic_worker/handlers/main.yml new file mode 100644 index 00000000..a9b4ffdd --- /dev/null +++ b/smartgears/generic_worker/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name='tomcat-instance-{{ smartgears_http_port }}' state=restarted diff --git a/smartgears/generic_worker/tasks/main.yml b/smartgears/generic_worker/tasks/main.yml new file mode 100644 index 00000000..8010717b --- /dev/null +++ b/smartgears/generic_worker/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Get the smart generic worker plugin and install it inside the smart executor + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ smart_generic_worker_plugin_url }} dest={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }}/WEB-INF/lib + when: generic_worker_install + notify: Restart smartgears + tags: [ 'smartgears', 'generic_worker', 'tomcat' ] + diff --git a/smartgears/install-gcube-key/tasks/main.yml b/smartgears/install-gcube-key/tasks/main.yml new file mode 100644 index 00000000..9c17a5c8 --- /dev/null +++ b/smartgears/install-gcube-key/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Install the production gcube keys + get_url: url={{ item.url }} dest=/{{ gcube_tomcat_lib_dir }}/{{ item.name }} owner={{ d4science_user }} group={{ d4science_user }} mode=0400 + with_items: + - '{{ gcube_prod_key_1 }}' + - '{{ gcube_prod_key_2 }}' + - '{{ gcube_prod_key_3 }}' + when: install_gcube_prod_key + tags: gcube_key + +- name: Install the devel gcube keys + get_url: url={{ item.url }} dest=/{{ gcube_tomcat_lib_dir }}/{{ item.name }} owner={{ d4science_user }} group={{ d4science_user }} mode=0400 + with_items: + - '{{ gcube_dev_key_1 }}' + - '{{ gcube_dev_key_2 }}' + - '{{ gcube_dev_key_3 }}' + when: install_gcube_dev_key + tags: gcube_key + diff --git a/smartgears/is_sweeper_se_plugin/defaults/main.yml b/smartgears/is_sweeper_se_plugin/defaults/main.yml new file mode 100644 index 00000000..f662b5fd --- /dev/null +++ b/smartgears/is_sweeper_se_plugin/defaults/main.yml @@ -0,0 +1,5 @@ +--- +is_sweeper_se_plugin_install: False +is_sweeper_se_plugin_ver: 1.0.0-3.11.0-128833 +is_sweeper_se_plugin_name: 'is-sweeper-se-plugin-{{ is_sweeper_se_plugin_ver }}-jar-with-dependencies.jar' +is_sweeper_se_plugin_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/information-system/is-sweeper-se-plugin/{{ is_sweeper_se_plugin_ver }}/{{ is_sweeper_se_plugin_name }}' diff --git a/smartgears/is_sweeper_se_plugin/handlers/main.yml b/smartgears/is_sweeper_se_plugin/handlers/main.yml new file mode 100644 index 00000000..a9b4ffdd --- /dev/null +++ b/smartgears/is_sweeper_se_plugin/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name='tomcat-instance-{{ smartgears_http_port }}' state=restarted diff --git a/smartgears/is_sweeper_se_plugin/tasks/main.yml b/smartgears/is_sweeper_se_plugin/tasks/main.yml new file mode 100644 index 00000000..3100ea84 --- /dev/null +++ b/smartgears/is_sweeper_se_plugin/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Get the is-sweeper se plugin and install it inside the smart executor + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ is_sweeper_se_plugin_url }} dest={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }}/WEB-INF/lib + when: is_sweeper_se_plugin_install + notify: Restart smartgears + tags: [ 'smartgears', 'is_sweeper_se', 'tomcat' ] + diff --git a/smartgears/smart_executor/defaults/main.yml b/smartgears/smart_executor/defaults/main.yml new file mode 100644 index 00000000..83fa19a9 --- /dev/null +++ b/smartgears/smart_executor/defaults/main.yml @@ -0,0 +1,24 @@ +--- +smart_executor_install: False + +#smart_executor_version: 1.2.0-3.9.0 +#smart_executor_version: 1.3.0-3.10.1 +smart_executor_version: 1.3.0-3.11.0-128844 +smart_executor_name: smart-executor +smart_executor_file: '{{ smart_executor_name }}-{{ smart_executor_version }}.war' +smart_executor_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/vremanagement/smart-executor/{{ smart_executor_version }}/{{ smart_executor_file }}' + +smart_executor_context: '/{{ smart_executor_name }}' +smart_executor_contexts_list: + - '' + - '/BiodiversityLab' + - '/BiOnym' + - '/ScalableDataMining' + +# dev has two different contexts +#smart_executor_contexts_list: +# - '' +# - '/devVRE' + +# - '' +# - '/NextNext' diff --git a/smartgears/smart_executor/handlers/main.yml b/smartgears/smart_executor/handlers/main.yml new file mode 100644 index 00000000..a9b4ffdd --- /dev/null +++ b/smartgears/smart_executor/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name='tomcat-instance-{{ smartgears_http_port }}' state=restarted diff --git a/smartgears/smart_executor/tasks/main.yml b/smartgears/smart_executor/tasks/main.yml new file mode 100644 index 00000000..de1fe8a6 --- /dev/null +++ b/smartgears/smart_executor/tasks/main.yml @@ -0,0 +1,41 @@ +--- +- name: Remove the old smart executor files + file: path={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }} state=absent + when: + - smart_executor_install + - smartgears_upgrade + tags: [ 'smartgears', 'tomcat' ] + +# NOTE: Install as the smartgears user so we do not mess with the permissions +- name: Get the smart executor plugin + get_url: url={{ smart_executor_url }} dest={{ smartgears_user_home }}/{{ smart_executor_file }} + when: smart_executor_install + tags: [ 'smartgears', 'smart_executor', 'tomcat' ] + +- name: Create the smart executor working directory + file: path={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }} state=directory owner={{ smartgears_user }} group={{ smartgears_user }} + when: smart_executor_install + tags: [ 'smartgears', 'smart_executor', 'tomcat' ] + +- name: Unarchive the smartexecutor distribution + become: True + become_user: '{{ smartgears_user }}' + unarchive: copy=no src={{ smartgears_user_home }}/{{ smart_executor_file }} dest={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }} creates={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }}/WEB-INF/lib + when: smart_executor_install + notify: Restart smartgears + tags: [ 'smartgears', 'smart_executor', 'tomcat' ] + +- name: Install the smartgears configuration file + template: src=smart_executor-container.xml.j2 dest={{ smartgears_install_path }}/container.xml owner={{ item.user }} group={{ item.user }} + with_items: '{{ tomcat_m_instances }}' + notify: Restart smartgears + when: smart_executor_install + register: smartexec_containerxml_state + tags: [ 'smartgears', 'smart_executor', 'smart_executor_conf', 'tomcat' ] + +- name: Remove the smartgears application state if the configuration changed + file: dest={{ smartgears_install_path }}/state state=absent + notify: Restart smartgears + when: ( smartexec_containerxml_state | changed ) + tags: [ 'smartgears', 'smart_executor', 'smart_executor_conf', 'tomcat' ] + diff --git a/smartgears/smart_executor/templates/smart_executor-container.xml.j2 b/smartgears/smart_executor/templates/smart_executor-container.xml.j2 new file mode 100644 index 00000000..eaec0a22 --- /dev/null +++ b/smartgears/smart_executor/templates/smart_executor-container.xml.j2 @@ -0,0 +1,34 @@ + + {{ smartgears_hostname }} +{%if setup_nginx %} +{%if https_port is defined %} + {{ https_port }} +{% else %} + {{ http_port }} +{% endif %} +{% else %} + {{ item.http_port }} +{% endif %} + {{ smartgears_infrastructure_name }} + {{ smartgears_vo_name }} + + + {{ smartgears_country }} + {{ smartgears_location }} + 41.9000 + 12.5000 + + + + + 60 + +{% if smart_executor_install %} + +{% for context in smart_executor_contexts_list %} + /{{ smartgears_infrastructure_name }}/{{ smartgears_vo_name }}{{ context }} +{% endfor %} + +{% endif %} + + diff --git a/smartgears/smartgears-nginx-frontend/meta/main.yml b/smartgears/smartgears-nginx-frontend/meta/main.yml new file mode 100644 index 00000000..f1749521 --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: '../../library/roles/nginx' diff --git a/smartgears/smartgears-nginx-frontend/tasks/main.yml b/smartgears/smartgears-nginx-frontend/tasks/main.yml new file mode 100644 index 00000000..c0c4640e --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Install the nginx virtualhosts +- name: Install the nginx virtualhosts + template: src={{ item.servername }}-virtualhost.j2 dest=/etc/nginx/sites-available/{{ item.servername }} owner=root group=root mode=0444 + with_items: '{{ tomcat_m_instances }}' + when: not smartgears_install_generic_virthost + notify: Reload nginx + tags: [ 'nginx', 'virtualhost' ] + +- name: Install the nginx smartexecutor virtualhost + template: src=generic-smartgears-virtualhost.j2 dest=/etc/nginx/sites-available/{{ item.servername }} owner=root group=root mode=0444 + with_items: '{{ tomcat_m_instances }}' + when: smartgears_install_generic_virthost + notify: Reload nginx + tags: [ 'nginx', 'virtualhost' ] + +- name: Enable the nginx virtualhosts + file: src=/etc/nginx/sites-available/{{ item.servername }} dest=/etc/nginx/sites-enabled/{{ item.servername }}.conf state=link + with_items: '{{ tomcat_m_instances }}' + notify: Reload nginx + tags: [ 'nginx', 'virtualhost' ] + diff --git a/smartgears/smartgears-nginx-frontend/templates/dataminer1-d-d4s.d4science.org-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/dataminer1-d-d4s.d4science.org-virtualhost.j2 new file mode 100644 index 00000000..f62bc421 --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/dataminer1-d-d4s.d4science.org-virtualhost.j2 @@ -0,0 +1,65 @@ +server { + listen {{ http_port }}; + server_name {{ item.servername }}; + log_format forwarderLog '$remote_addr forwarded for $http_x_forwarded_for - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"'; + log_format shortForwarderLog '$http_x_forwarded_for - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'; + access_log /var/log/nginx/{{ item.servername }}_access.log shortForwarderLog; + + # This is the default for nginx on Ubuntu 12.04 + root /usr/share/nginx/www/; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/www; + } + + location = /favicon.ico { + log_not_found off; + access_log off; + } + + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + + location /logs { + alias /var/log/nginx; + allow {{ analytics_ip }}; # analytics.d4science.org + deny all; + } + # don't send the nginx version number in error pages and Server header + server_tokens off; + + # Proxy stuff + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_buffer_size {{ nginx_proxy_buffer_size }}; + proxy_buffers {{ nginx_proxy_buffers }}; + proxy_busy_buffers_size {{ nginx_proxy_busy_buffers_size }}; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_redirect {{ nginx_proxy_redirect }}; + proxy_buffering {{ nginx_proxy_buffering }}; + proxy_connect_timeout {{ nginx_proxy_connect_timeout }}; + proxy_read_timeout {{ nginx_proxy_read_timeout }}; + proxy_send_timeout {{ nginx_proxy_send_timeout }}; +{% for instance in tomcat_m_instances %} +{% for context in instance.app_contexts %} + location /{{ context }} { + proxy_pass http://localhost:{{ item.http_port }}/{{ context }}; + } +{% endfor %} +{% endfor %} + location {{ dataminer_52north_webapp_context }} { + proxy_pass http://localhost:{{ item.http_port }}{{ dataminer_52north_webapp_context }}; + } + +} diff --git a/smartgears/smartgears-nginx-frontend/templates/dataminer1-p-d4s.d4science.org-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/dataminer1-p-d4s.d4science.org-virtualhost.j2 new file mode 120000 index 00000000..170b0000 --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/dataminer1-p-d4s.d4science.org-virtualhost.j2 @@ -0,0 +1 @@ +dataminer1-d-d4s.d4science.org-virtualhost.j2 \ No newline at end of file diff --git a/smartgears/smartgears-nginx-frontend/templates/dataminer2-d-d4s.d4science.org-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/dataminer2-d-d4s.d4science.org-virtualhost.j2 new file mode 120000 index 00000000..170b0000 --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/dataminer2-d-d4s.d4science.org-virtualhost.j2 @@ -0,0 +1 @@ +dataminer1-d-d4s.d4science.org-virtualhost.j2 \ No newline at end of file diff --git a/smartgears/smartgears-nginx-frontend/templates/dataminer2-p-d4s.d4science.org-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/dataminer2-p-d4s.d4science.org-virtualhost.j2 new file mode 120000 index 00000000..65e3032e --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/dataminer2-p-d4s.d4science.org-virtualhost.j2 @@ -0,0 +1 @@ +dataminer1-p-d4s.d4science.org-virtualhost.j2 \ No newline at end of file diff --git a/smartgears/smartgears-nginx-frontend/templates/egi-d4s-smartgears-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/egi-d4s-smartgears-virtualhost.j2 new file mode 120000 index 00000000..170b0000 --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/egi-d4s-smartgears-virtualhost.j2 @@ -0,0 +1 @@ +dataminer1-d-d4s.d4science.org-virtualhost.j2 \ No newline at end of file diff --git a/smartgears/smartgears-nginx-frontend/templates/generic-smartgears-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/generic-smartgears-virtualhost.j2 new file mode 100644 index 00000000..e20b786d --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/generic-smartgears-virtualhost.j2 @@ -0,0 +1,58 @@ +server { + listen {{ http_port }}; + server_name {{ item.servername }}; + access_log on; + + # This is the default for nginx on Ubuntu 12.04 + root /usr/share/nginx/www/; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/www; + } + + location = /favicon.ico { + log_not_found off; + access_log off; + } + + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + + # don't send the nginx version number in error pages and Server header + server_tokens off; + + # Proxy stuff + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_buffer_size {{ nginx_proxy_buffer_size }}; + proxy_buffers {{ nginx_proxy_buffers }}; + proxy_busy_buffers_size {{ nginx_proxy_busy_buffers_size }}; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_redirect {{ nginx_proxy_redirect }}; + proxy_buffering {{ nginx_proxy_buffering }}; + proxy_connect_timeout {{ nginx_proxy_connect_timeout }}; + proxy_read_timeout {{ nginx_proxy_read_timeout }}; + proxy_send_timeout {{ nginx_proxy_send_timeout }}; +{% for instance in tomcat_m_instances %} +{% for context in instance.app_contexts %} + location /{{ context }} { + proxy_pass http://localhost:{{ item.http_port }}/{{ context }}; + } +{% endfor %} +{% endfor %} +{% if smart_executor_install %} + location {{ smart_executor_context }} { + proxy_pass http://localhost:{{ item.http_port }}{{ smart_executor_context }}; + } +{% endif %} + +} diff --git a/smartgears/smartgears-nginx-frontend/templates/tagme.d4science.org-virtualhost.j2 b/smartgears/smartgears-nginx-frontend/templates/tagme.d4science.org-virtualhost.j2 new file mode 100644 index 00000000..40cc4a32 --- /dev/null +++ b/smartgears/smartgears-nginx-frontend/templates/tagme.d4science.org-virtualhost.j2 @@ -0,0 +1,139 @@ +server { + listen {{ http_port }}; +{% if letsencrypt_acme_install is defined and letsencrypt_acme_install %} + location ^~ /.well-known/acme-challenge { + proxy_pass http://127.0.0.1:{{ letsencrypt_acme_standalone_port }}/.well-known/acme-challenge; + access_log /var/log/nginx/letsencrypt_access.log; + error_log /var/log/nginx/letsencrypt_error.log; + } +{% endif %} +{% if not http_ssl_enabled %} + server_name {{ ansible_fqdn }}; + client_max_body_size 100M; + + # This is the default for nginx on Ubuntu 14.04 + root /usr/share/nginx/html/; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location = /favicon.ico { + log_not_found off; + access_log off; + } + + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + + # don't send the nginx version number in error pages and Server header + server_tokens off; + + # Proxy stuff + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_buffer_size {{ nginx_proxy_buffer_size }}; + proxy_buffers {{ nginx_proxy_buffers }}; + proxy_busy_buffers_size {{ nginx_proxy_busy_buffers_size }}; + proxy_set_header X-Forwarded-Host $remote_addr; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_redirect {{ nginx_proxy_redirect }}; + proxy_buffering {{ nginx_proxy_buffering }}; + proxy_connect_timeout {{ nginx_proxy_connect_timeout }}; + proxy_read_timeout {{ nginx_proxy_read_timeout }}; + proxy_send_timeout {{ nginx_proxy_send_timeout }}; +{% for instance in tomcat_m_instances %} +{% for context in instance.app_contexts %} + location /{{ context }} { + proxy_pass http://localhost:{{ item.http_port }}/{{ context }}; + } +{% endfor %} +{% endfor %} + location / { + proxy_pass http://localhost:{{ smartgears_http_port }}; + } +{% else %} + return 301 https://{{ ansible_fqdn }}$request_uri; +{% endif %} + +} + +{% if http_ssl_enabled %} +server { + listen {{ https_port }} ssl; + + server_name {{ ansible_fqdn }}; + client_max_body_size 100M; + access_log /var/log/nginx/access_ssl.log; + error_log /var/log/nginx/error_ssl.log; + + # This is the default for nginx on Ubuntu 14.04 + root /usr/share/nginx/html/; + + ssl_certificate {{ letsencrypt_acme_certs_dir }}/fullchain; + ssl_certificate_key {{ letsencrypt_acme_certs_dir }}/privkey; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"; + ssl_prefer_server_ciphers on; + ssl_stapling on; + ssl_stapling_verify on; + add_header Strict-Transport-Security max-age=15768000; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location = /favicon.ico { + log_not_found off; + access_log off; + } + + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + + # don't send the nginx version number in error pages and Server header + server_tokens off; + + # Proxy stuff + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_buffer_size {{ nginx_proxy_buffer_size }}; + proxy_buffers {{ nginx_proxy_buffers }}; + proxy_busy_buffers_size {{ nginx_proxy_busy_buffers_size }}; + proxy_set_header X-Forwarded-Host $remote_addr; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_redirect {{ nginx_proxy_redirect }}; + proxy_buffering {{ nginx_proxy_buffering }}; + proxy_connect_timeout {{ nginx_proxy_connect_timeout }}; + proxy_read_timeout {{ nginx_proxy_read_timeout }}; + proxy_send_timeout {{ nginx_proxy_send_timeout }}; +{% for instance in tomcat_m_instances %} +{% for context in instance.app_contexts %} + location /{{ context }} { + proxy_pass http://localhost:{{ item.http_port }}/{{ context }}; + } +{% endfor %} +{% endfor %} + location / { + proxy_pass http://localhost:{{ smartgears_http_port }}; + } +} +{% endif %} diff --git a/smartgears/smartgears/defaults/main.yml b/smartgears/smartgears/defaults/main.yml new file mode 100644 index 00000000..9ddaba88 --- /dev/null +++ b/smartgears/smartgears/defaults/main.yml @@ -0,0 +1,50 @@ +--- +gcube_repository: 'gcube-staging' +gcube_repository_stagin_test: 'gcube-staging-test' + +# Set it to true from the command line when you want to upgrade a +# distribution version +smartgears_upgrade: False +smartgears_force_install: False + +smartgears_node: True +smartgears_install_generic_virthost: True + +smartgears_data_fs: /home +smartgears_user: '{{ d4science_user }}' +smartgears_user_home: '{{ d4science_user_home }}' +smartgears_instance_path: '{{ smartgears_user_home }}/tomcat' +smartgears_install_path: '{{ smartgears_user_home }}/SmartGears' +#smartgears_distribution_version: 1.2.6-3.10.0 +#smartgears_distribution_version: 1.2.7-3.10.1 +smartgears_distribution_version: 1.2.7-3.11.0-125799 +smartgears_file: 'smartgears-distribution-{{ smartgears_distribution_version }}.tar.gz' +smartgears_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/distribution/smartgears-distribution/{{ smartgears_distribution_version }}/{{ smartgears_file }}' +smartgears_mode: online +# Production infra and VO +smartgears_infrastructure_name: "d4science.research-infrastructures.eu" +smartgears_vo_name: gCubeApps +smartgears_hostname: '{{ ansible_fqdn }}' +smartgears_country: it +smartgears_location: pisa + +smartgears_http_port: 9000 +smartgears_service_name: 'tomcat-instance-{{ smartgears_http_port }}' + +# The iptables rules use this +http_port: '{{ smartgears_http_port }}' + +tomcat_m_webapps_unpack: True +tomcat_m_instances: + - { http_enabled: True, http_port: '{{ smartgears_http_port }}', http_address: '0.0.0.0', ajp_enabled: False, ajp_port: '8109', ajp_address: '127.0.0.1', restart_timeout: '{{ tomcat_m_restart_timeout }}', shutdown_port: '-1', java_home: '{{ jdk_java_home }}', user: '{{ smartgears_user }}', user_home: '{{ smartgears_user_home }}', user_shell: '{{ tomcat_m_default_user_shell }}', instance_path: '{{ smartgears_instance_path }}', max_threads: '{{ tomcat_m_max_threads }}', autodeploy: '{{ tomcat_m_webapps_autodeploy }}', unpack: '{{ tomcat_m_webapps_unpack }}', default_conf: True, java_opts: '{{ tomcat_m_java_opts }}', java_gc_opts: '{{ tomcat_m_java_gc_opts }}', other_java_opts: '{{ tomcat_m_other_java_opts }}', remote_debugging: '{{ tomcat_m_enable_remote_debugging }}', remote_debugging_port: '{{ smartgears_debugging_port }}' , access_log_enabled: True, log_rotation_freq: daily, log_retain: 30, allowed_hosts: [ '0.0.0.0/0' ], app_contexts: [ 'whn-manager' ], servername: '{{ ansible_fqdn }}' } + +# To enable debugging: +# - Set +# tomcat_m_enable_remote_debugging: True +# Rerun the playbook using --tags=tomcat_default +# +smartgears_debugging_port: '{{ tomcat_m_remote_debugging_port }}' +# iptables: +# tcp_rules: True +# tcp: +# - { port: '{{ smartgears_debugging_port }}', allowed_hosts: [ '{{ network.isti }}', '{{ network.nmis }}', '{{ network.eduroam }}' ] } diff --git a/smartgears/smartgears/handlers/main.yml b/smartgears/smartgears/handlers/main.yml new file mode 100644 index 00000000..a9b4ffdd --- /dev/null +++ b/smartgears/smartgears/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name='tomcat-instance-{{ smartgears_http_port }}' state=restarted diff --git a/smartgears/smartgears/tasks/main.yml b/smartgears/smartgears/tasks/main.yml new file mode 100644 index 00000000..cc3b8031 --- /dev/null +++ b/smartgears/smartgears/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- include: smartgears-app.yml + diff --git a/smartgears/smartgears/tasks/smartgears-app.yml b/smartgears/smartgears/tasks/smartgears-app.yml new file mode 100644 index 00000000..8458025d --- /dev/null +++ b/smartgears/smartgears/tasks/smartgears-app.yml @@ -0,0 +1,71 @@ +--- +- name: Remove the smartgears distribution archive to force un upgrade + file: dest={{ item }} state=absent + with_items: + - '{{ smartgears_user_home }}/{{ smartgears_file }}' + - '{{ smartgears_user_home }}/smartgears-distribution-{{ smartgears_distribution_version }}' + when: smartgears_force_install + tags: [ 'smartgears', 'tomcat' ] + +- name: Get the smartgears distribution + get_url: url={{ smartgears_url }} dest={{ smartgears_user_home }}/{{ smartgears_file }} + register: new_smartgears_distribution + tags: [ 'smartgears', 'tomcat' ] + +- name: Unarchive the smartgears distribution + become: True + become_user: '{{ d4science_user }}' + unarchive: copy=no src={{ smartgears_user_home }}/{{ smartgears_file }} dest={{ smartgears_user_home }} creates={{ smartgears_user_home }}/smartgears-distribution-{{ smartgears_distribution_version }} + when: ( new_smartgears_distribution | changed ) + tags: [ 'smartgears', 'tomcat' ] + +- name: Stop the smartgears container before launching the upgrade script + become: True + become_user: '{{ d4science_user }}' + shell: /home/{{ d4science_user }}/stopContainer.sh + when: smartgears_upgrade + tags: [ 'smartgears', 'tomcat', 'stop_smartgears_service' ] + +- name: Install the additional tomcat default file + template: src=tomcat-instance-default.j2 dest=/etc/default/tomcat-instance-{{ item.http_port }}.local + with_items: '{{ tomcat_m_instances }}' + notify: Restart smartgears + tags: [ 'smartgears', 'tomcat', 'smartgears_env' ] + +- name: Set the additional tomcat default file as the smartgears user .bashrc + become: True + become_user: '{{ d4science_user }}' + template: src=tomcat-instance-default.j2 dest={{ item.user_home }}/.bashrc mode=0444 + with_items: '{{ tomcat_m_instances }}' + tags: [ 'smartgears', 'tomcat', 'smartgears_env' ] + +- name: Install the smartgears app + shell: cd {{ item.user_home }}/smartgears-distribution-{{ smartgears_distribution_version }} ; . /{{ item.user_home }}/.bashrc ; ./install -s tomcat ; chown -R gcube:gcube smartgears-distribution-{{ smartgears_distribution_version }} SmartGears/ tomcat/lib/ tomcat/webapps/ + with_items: '{{ tomcat_m_instances }}' + when: ( new_smartgears_distribution | changed ) + register: install_smartgears_app + notify: Restart smartgears + tags: [ 'smartgears', 'tomcat' ] + +- name: Install the smartgears configuration file + template: src=container.xml.j2 dest={{ smartgears_install_path }}/container.xml owner={{ item.user }} group={{ item.user }} + with_items: '{{ tomcat_m_instances }}' + when: not generic_worker_install + register: containerxml_state + notify: Restart smartgears + tags: [ 'smartgears', 'smartgears_conf', 'tomcat' ] + +- name: Remove the smartgears application state if the configuration changed + file: dest={{ smartgears_install_path }}/state state=absent + when: ( containerxml_state | changed ) + notify: Restart smartgears + tags: [ 'smartgears', 'smartgears_conf', 'tomcat' ] + +- name: Remove some wrong symbolic links created by the install/upgrade script + file: dest={{ smartgears_install_path }}/state state=absent + with_items: + - '{{ smartgears_install_path }}/lib/lib' + - '{{ smartgears_install_path }}/apps/webapps' + when: smartgears_upgrade + tags: [ 'smartgears', 'smartgears_conf', 'tomcat' ] + diff --git a/smartgears/smartgears/templates/container.xml.j2 b/smartgears/smartgears/templates/container.xml.j2 new file mode 100644 index 00000000..3649a3e3 --- /dev/null +++ b/smartgears/smartgears/templates/container.xml.j2 @@ -0,0 +1,27 @@ + + + {{ smartgears_hostname }} +{%if setup_nginx %} +{%if https_port is defined %} + {{ https_port }} +{% else %} + {{ http_port }} +{% endif %} +{% else %} + {{ item.http_port }} +{% endif %} + {{ smartgears_infrastructure_name }} + {{ smartgears_vo_name }} + + + {{ smartgears_country }} + {{ smartgears_location }} + 41.9000 + 12.5000 + + + + + 60 + + diff --git a/smartgears/smartgears/templates/tomcat-instance-default.j2 b/smartgears/smartgears/templates/tomcat-instance-default.j2 new file mode 100644 index 00000000..b8da26b4 --- /dev/null +++ b/smartgears/smartgears/templates/tomcat-instance-default.j2 @@ -0,0 +1,2 @@ +export GHN_HOME={{ smartgears_user_home }}/SmartGears +export WEB_CONTAINER_HOME={{ smartgears_instance_path }} diff --git a/smartgears/smartgears_egi_image/tasks/main.yml b/smartgears/smartgears_egi_image/tasks/main.yml new file mode 100644 index 00000000..fe7df81e --- /dev/null +++ b/smartgears/smartgears_egi_image/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- include: setup-egi-image.yml + when: egi_image + diff --git a/smartgears/smartgears_egi_image/tasks/setup-egi-image.yml b/smartgears/smartgears_egi_image/tasks/setup-egi-image.yml new file mode 100644 index 00000000..1a7b19e4 --- /dev/null +++ b/smartgears/smartgears_egi_image/tasks/setup-egi-image.yml @@ -0,0 +1,9 @@ +--- +- name: Install the EGI image setup script + template: src=smartgears-setup.sh.j2 dest=/usr/local/bin/smartgears-setup owner=root group=root mode=0555 + tags: [ 'smartgears', 'tomcat', 'egi', 'egi_image' ] + +- name: Install the EGI gcube keys installer script + template: src=get-gcube-keys.sh.j2 dest=/usr/local/bin/get-gcube-keys owner=root group=root mode=0500 + tags: [ 'smartgears', 'tomcat', 'egi', 'egi_image' ] + diff --git a/smartgears/smartgears_egi_image/templates/get-gcube-keys.sh.j2 b/smartgears/smartgears_egi_image/templates/get-gcube-keys.sh.j2 new file mode 100644 index 00000000..e798768b --- /dev/null +++ b/smartgears/smartgears_egi_image/templates/get-gcube-keys.sh.j2 @@ -0,0 +1,43 @@ +#!/bin/bash + +SM_DIR={{ smartgears_install_path }} +SM_LIB_DIR=${SM_DIR}/lib +# The dataminer wants the gcube keys in a different directory too +SM_DATAMINER_PARALLEL_PROC={{ smartgears_user_home }}/tomcat/webapps/wps/ecocfg/PARALLEL_PROCESSING +LOG_FILE=/var/log/smartgears-gcubekeys-setup.log + +INFRA_NAME=$1 +> $LOG_FILE + +if [ ! -d $SM_LIB_DIR ] ; then + echo "$0: The lib directory: $SM_LIB_DIR does not exist, aborting" >> $LOG_FILE + exit 1 +else + pushd $SM_LIB_DIR >/dev/null 2>&1 +fi + +if [ "$INFRA_NAME" == "gcube" ] ; then + wget {{ gcube_dkey_1 }} --no-check-certificate -o /dev/null -O gcube.gcubekey + wget {{ gcube_dkey_2 }} --no-check-certificate -o /dev/null -O devsec.gcubekey + wget {{ gcube_dkey_3 }} --no-check-certificate -o /dev/null -O devNext.gcubekey +elif [ "$INFRA_NAME" == "d4science.research-infrastructures.eu" ] ; then + wget {{ gcube_pkey_1 }} --no-check-certificate -o /dev/null -O d4science.research-infrastructures.eu.gcubekey + wget {{ gcube_pkey_2 }} --no-check-certificate -o /dev/null -O FARM.gcubekey + wget {{ gcube_pkey_3 }} --no-check-certificate -o /dev/null -O gCubeApps.gcubekey + chmod 400 $SM_LIB_DIR/*.gcubekey +else + echo "The Infra name is unknown, no keys downloaded" >> $LOG_FILE + exit 0 +fi +popd >/dev/null 2>&1 +chmod 400 $SM_LIB_DIR/*.gcubekey >> $LOG_FILE 2>&1 +chown gcube:root $SM_LIB_DIR/*.gcubekey >> $LOG_FILE 2>&1 +if [ -d $SM_DATAMINER_PARALLEL_PROC ] ; then + echo "This is a dataminer. Copy the keys inside the dataminer conf dir" >> $LOG_FILE 2>&1 + cp -a $SM_LIB_DIR/*.gcubekey $SM_DATAMINER_PARALLEL_PROC >> $LOG_FILE 2>&1 + chmod 400 $SM_DATAMINER_PARALLEL_PROC/*.gcubekey >> $LOG_FILE 2>&1 + chown gcube:root $SM_DATAMINER_PARALLEL_PROC/*.gcubekey >> $LOG_FILE 2>&1 +fi + +exit 0 + diff --git a/smartgears/smartgears_egi_image/templates/smartgears-setup.sh.j2 b/smartgears/smartgears_egi_image/templates/smartgears-setup.sh.j2 new file mode 100755 index 00000000..4f89d4aa --- /dev/null +++ b/smartgears/smartgears_egi_image/templates/smartgears-setup.sh.j2 @@ -0,0 +1,106 @@ +#!/bin/bash + +SM_DIR={{ smartgears_install_path }} +SM_CONF_F=${SM_DIR}/container.xml +SM_LIB_DIR=${SM_DIR}/lib +SM_STATE_DIR=${SM_DIR}/state/ +GET_KEYS_SCRIPT=/usr/local/bin/get-gcube-keys +SVC=tomcat-instance-{{ smartgears_http_port }} +LOG_FILE=/var/log/smartgears-node-setup.log +HOSTS_FILE=/etc/hosts +FQDN_HOST= +PUB_FQDN_HOST= +PUBLIC_IP= +MAIN_IP= +RETVAL=0 + +{% raw %} +> $LOG_FILE +if [ -z "${HOST}" ] ; then + echo "No hostname was passed from the outside. Trying to find it out by ourselves" >> $LOG_FILE + FQDN_HOST=$( /bin/hostname -f ) + if [ -z "${FQDN_HOST}" ] ; then + echo "The machine has no FQDN. Revert to plain hostname" >> $LOG_FILE + HOST=$( /bin/hostname ) + echo "Setting the plain hostname: $HOST" >> $LOG_FILE + else + HOST=$FQDN_HOST + fi +fi +# We don't know if we have been assigned a private or public IP address +PUBLIC_IP=$( wget http://ipecho.net/plain -qO - ) +MAIN_IP=$( ip addr show scope global dev eth0 | grep inet | awk '{ print $2 }' | cut -d "/" -f 1 ) +echo "Our main IP is ${MAIN_IP}" >> $LOG_FILE +if [ -z $PUBLIC_IP ] ; then + # We have some kind of problem + echo "We cannot obtain our public IP from outside. Using our main IP address as public IP" >> $LOG_FILE + PUBLIC_IP=$MAIN_IP +else + echo "Our public IP is ${PUBLIC_IP}" >> $LOG_FILE +fi +# Try to know our public FQDN +TMP_FQDN_HOST=$( host $PUBLIC_IP ) +RET_FQDN=$? +if [ $RET_FQDN -eq 0 ] ; then + PUB_FQDN_HOST=$( host $PUBLIC_IP | awk '{print $5}' | sed -e 's/\.$//g' ) +fi +# Set the hosts file with the new data +grep -v $HOST $HOSTS_FILE > $HOSTS_FILE.tmp +echo "${PUBLIC_IP} ${FQDN_HOST} ${PUB_FQDN_HOST} ${HOST}" >> $HOSTS_FILE.tmp +mv $HOSTS_FILE.tmp $HOSTS_FILE +chmod 644 $HOSTS_FILE +chown root:root $HOSTS_FILE + +if [ -z "${INFRA_NAME}" ] ; then + echo "The infrastructure name is void" >> $LOG_FILE + RETVAL=1 +fi +if [ -z "${SCOPE}" ] ; then + echo "The scope is void" >> $LOG_FILE + RETVAL=1 +fi +if [ -z "${COUNTRY}" ] ; then + echo "The country code is void" >> $LOG_FILE + RETVAL=1 +elif [ ${#COUNTRY} -ne 2 ] ; then + echo "The country code is wrong. It must be two characters long" >> $LOG_FILE + RETVAL=1 +fi +if [ $RETVAL -eq 1 ] ; then + echo "Exiting because of errors. The SmartExecutor will not start" >> $LOG_FILE + exit 1 +fi + +# We use the public FQDN if there's one. Otherwise we use the public IP +if [ ! -z ${PUB_FQDN_HOST} ] ; then + sed -i -e "s#@SMARTGEARS_HOSTNAME@#${PUB_FQDN_HOST}#g" $SM_CONF_F +else + sed -i -e "s#@SMARTGEARS_HOSTNAME@#${PUBLIC_IP}#g" $SM_CONF_F +fi +sed -i -e "s#@SMARTGEARS_INFRASTRUCTURE_NAME@#${INFRA_NAME}#g" $SM_CONF_F +sed -i -e "s#@SMARTGEARS_VO_NAME@#${SCOPE}#g" $SM_CONF_F +sed -i -e "s#@SMARTGEARS_COUNTRY@#${COUNTRY}#g" $SM_CONF_F +sed -i -e "s#@SMARTGEARS_LOCATION@#${LOCATION}#g" $SM_CONF_F +{% endraw %} + +# Get and install the gcube security keys +if [ -x $GET_KEYS_SCRIPT ] ; then + echo "Getting the infrastructure keys" >> $LOG_FILE + $GET_KEYS_SCRIPT ${INFRA_NAME} >> $LOG_FILE + rm -f $GET_KEYS_SCRIPT + echo "Done" >> $LOG_FILE +else + echo "Not getting the infrastructure keys, the script is not available." >> $LOG_FILE +fi + +/etc/init.d/${SVC} stop +rm -fr ${SM_STATE_DIR}/* +rm -f ${SM_DIR}/ghn.log +sleep 20 +/usr/sbin/update-rc.d ${SVC} enable +/etc/init.d/${SVC} start + +exit 0 + + + diff --git a/smartgears/social_data_indexer_se_plugin/defaults/main.yml b/smartgears/social_data_indexer_se_plugin/defaults/main.yml new file mode 100644 index 00000000..5e6032bd --- /dev/null +++ b/smartgears/social_data_indexer_se_plugin/defaults/main.yml @@ -0,0 +1,7 @@ +--- +gcube_repository: 'gcube-staging' +social_data_indexer_se_plugin_install: False +#social_data_indexer_se_plugin_ver: 1.0.0-3.11.0-124419 +social_data_indexer_se_plugin_ver: 1.0.0-3.11.0-125977 +social_data_indexer_se_plugin_name: 'social-data-indexer-se-plugin-{{ social_data_indexer_se_plugin_ver }}-jar-with-dependencies.jar' +social_data_indexer_se_plugin_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}/org/gcube/socialnetworking/social-data-indexer-se-plugin/{{ social_data_indexer_se_plugin_ver }}/{{ social_data_indexer_se_plugin_name }}' diff --git a/smartgears/social_data_indexer_se_plugin/handlers/main.yml b/smartgears/social_data_indexer_se_plugin/handlers/main.yml new file mode 100644 index 00000000..a9b4ffdd --- /dev/null +++ b/smartgears/social_data_indexer_se_plugin/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart smartgears + service: name='tomcat-instance-{{ smartgears_http_port }}' state=restarted diff --git a/smartgears/social_data_indexer_se_plugin/tasks/main.yml b/smartgears/social_data_indexer_se_plugin/tasks/main.yml new file mode 100644 index 00000000..e3ec9ba2 --- /dev/null +++ b/smartgears/social_data_indexer_se_plugin/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Get the couchdb cache se plugin and install it inside the smart executor + become: True + become_user: '{{ smartgears_user }}' + get_url: url={{ social_data_indexer_se_plugin_url }} dest={{ smartgears_instance_path }}/webapps/{{ smart_executor_name }}/WEB-INF/lib + when: social_data_indexer_se_plugin_install + notify: Restart smartgears + tags: [ 'smartgears', 'social_data_indexer_se', 'tomcat' ] +