Merge branch 'master' of gitorious.research-infrastructures.eu:infrastructure-management/ansible-playbooks

python3-env/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+py3_env_install: False
+py3_ppa: 'ppa:deadsnakes/ppa'
+py3_env_version: 3.6
+py3_env_pkgs_state: present
+py3_env_site: False
+
+py3_env_basic_pkgs:
+  - python-pip
+
+py3_env_dpkg:
+  - 'python{{ py3_env_version }}'
+  - 'python{{ py3_env_version }}-venv'
+  - 'python{{ py3_env_version }}-dev'

python3-env/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+- block:
+  - name: Install the python 3 ppa repository
+    apt_repository: repo={{ py3_ppa }} update_cache=yes state=present
+
+  - name: Install the python3 deb packages
+    apt: name={{ py3_env_dpkg }} state={{ py_env_pkgs_state }} update_cache=yes cache_valid_time=600
+
+  - name: Install the correct pip3 version
+    shell: python{{ py3_env_version }} -m ensurepip && pip{{ py3_env_version }} install --upgrade pip setuptools wheel
+
+  - name: Install a list of pip packages
+    pip: executable=pip{{ py3_env_version }} name={{ py3_env_pip_pkgs }} state=latest
+
+  - name: Install a list of versioned pip packages
+    pip: executable=pip{{ py3_env_version }} name={{ item.pkg }} version={{ item.version }}
+    with_items: '{{ py3_env_versioned_pip_pkgs | default ([]) }}'
+
+  when:
+    - py3_env_install
+    - is_trusty
+  tags: [ "python", "py3_env", "py3_env_pkgs", 'python3' ]
+

shibbolet-sp-2/defaults/main.yml

@@ -6,5 +6,9 @@ shibbolet_pkg_list:
   - shibboleth-sp2-utils
   - curl
 
+shibbolet_pkg_list_trusty:
+  - shibboleth-sp2-schemas
+  - curl
+
 shibbolet_apache_module_pkg:
   - libapache2-mod-shib2

shibbolet-sp-2/tasks/main.yml

@@ -1,12 +1,15 @@
 ---
 - block:
   - name: Install the shibbolet packages
-    apt: pkg={{ item }} state={{ shibbolet_pkg_state }} cache_valid_time=1800 update_cache=yes
+    apt: pkg={{ shibbolet_pkg_list }} state={{ shibbolet_pkg_state }} cache_valid_time=1800 update_cache=yes
-    with_items: '{{ shibbolet_pkg_list }}'
+    when: is_not_trusty
+
+  - name: Install the shibbolet packages on Trusty
+    apt: pkg={{ shibbolet_pkg_list_trusty }} state={{ shibbolet_pkg_state }} cache_valid_time=1800 update_cache=yes
+    when: is_not_trusty
 
   - name: Install the shibbolet apache module
-    apt: pkg={{ item }} state={{ shibbolet_pkg_state }} cache_valid_time=1800 update_cache=yes
+    apt: pkg={{ shibbolet_apache_module_pkg }} state={{ shibbolet_pkg_state }} cache_valid_time=1800 update_cache=yes
-    with_items: '{{ shibbolet_apache_module_pkg }}'
 
   - name: Enable the apache shib2 module
     apache2_module: name=shib2 state=present

smartgears/dataminer_app/meta/main.yml

@@ -3,3 +3,4 @@ dependencies:
   - { role: ../../library/roles/smartgears/smartgears-service }
   - { role: ../../library/roles/conda, when: dataminer_conda_install }
   - { role: ../../library/roles/hdf5, when: dataminer_hdf5 }
+  - { role: ../../library/roles/python3-env, when: py3_env_install }

smartgears/smartgears-nginx-frontend/defaults/main.yml

@@ -10,5 +10,9 @@ smartgears_tomcat_manager_exposed: False
 smartgears_tomcat_manager_access_acls:
   - { policy: 'allow', address: '0.0.0.0/0' }
 
+# You can use smartgears_special_redirect to redirect whatever URL to another one, even changing the hostname (it must be an alias of the machine hostname if you want to maintain it local)
+#smartgears_special_redirect:
+#  - { source_uri: '/', dest_host: 'hostname-alias', dest_url: 'service-url'}
+
 smartgears_nginx_expose_tomcat_logs: False
 smartgears_nginx_rw_html_root: False

smartgears/smartgears-nginx-frontend/templates/generic-smartgears-virtualhost.j2

@@ -67,6 +67,7 @@ server {
     {% for instance in tomcat_m_instances %}
     {% for context in instance.app_contexts %}
     {% if context != "whn-manager" %}
+    {% if context != "uri-resolver" %}
     {% if context != '' %}
     location /{{ context }} {
         {% if varnish_install is defined and varnish_install %}
@@ -95,6 +96,7 @@ server {
     }
     {% endif %}
     {% endif %}
+    {% endif %}
     {% endfor %}
     {% endfor %}
     {% if smart_executor_install is defined and smart_executor_install %}
@@ -157,16 +159,25 @@ server {
         {% if smartgears_nginx_cors_enabled %}
         include /etc/nginx/snippets/nginx-cors.conf;
         {% endif %}
-        proxy_pass http://127.0.0.1:{{ smartgears_http_port }}/uri-resolver/;
+        rewrite ^ $request_uri;
+        rewrite ^/(.*) uri-resolver/$1 break;
+        return 400; #if the second rewrite won't match
+        proxy_pass http://127.0.0.1:{{ smartgears_http_port }}/$uri;
     }
     {% endif %}
+    {% if smartgears_special_redirect is defined %}
+    {% for target in smartgears_special_redirect %}
+    location {{ target.source_uri }} {
+        return 301 http://{{ target.dest_host }}/{{ target.dest_url }};
+    }
+    {% endfor %}
+    {% endif %}
 
     {% else %}
     location / {
         return 301 https://{{ item.servername }}$request_uri;
     }
     {% endif %}
-
 }
 
 {% if letsencrypt_acme_install %}
@@ -238,6 +249,7 @@ server {
     {% for instance in tomcat_m_instances %}
     {% for context in instance.app_contexts %}
     {% if context != "whn-manager" %}
+    {% if context != "uri-resolver" %}
     {% if context != '' %}
     location /{{ context }} {
         {% if varnish_install is defined and varnish_install %}
@@ -266,6 +278,7 @@ server {
     }
     {% endif %}
     {% endif %}
+    {% endif %}
     {% endfor %}
     {% endfor %}
     {% if smart_executor_install is defined and smart_executor_install %}
@@ -320,16 +333,28 @@ server {
         allow {{ ip }};
         {% endfor %}
         deny all;
-        proxy_pass http://127.0.0.1:{{ smartgears_http_port }}/uri-resolver/geonetwork;
+        rewrite ^ $request_uri;
+        rewrite ^/(.*) uri-resolver/$1 break;
+        return 400; #if the second rewrite won't match
+        proxy_pass http://127.0.0.1:{{ smartgears_http_port }}/$uri;
     }
 
     location / {
         {% if smartgears_nginx_cors_enabled %}
         include /etc/nginx/snippets/nginx-cors.conf;
         {% endif %}
-        proxy_pass http://127.0.0.1:{{ smartgears_http_port }}/uri-resolver/;
+        rewrite ^ $request_uri;
+        rewrite ^/(.*) uri-resolver/$1 break;
+        return 400; #if the second rewrite won't match
+        proxy_pass http://127.0.0.1:{{ smartgears_http_port }}/$uri;
     }
     {% endif %}
-
+    {% if smartgears_special_redirect is defined %}
+    {% for target in smartgears_special_redirect %}
+    location {{ target.source_uri }} {
+        return 301 http://{{ target.dest_host }}/{{ target.dest_url }};
+    }
+    {% endfor %}
+    {% endif %}
 }
 {% endif %}

smartgears/smartgears-service/vars/main.yml

@@ -4,4 +4,4 @@ smartgears_install_generic_virthost: True
 
 tomcat_m_webapps_unpack: True
 tomcat_m_instances:
-  - {  http_enabled: True, http_port: '{{ smartgears_http_port }}', http_address: '{{ smartgears_tomcat_listen_address }}', ajp_enabled: False, ajp_port: '8109', ajp_address: '127.0.0.1', restart_timeout: '{{ tomcat_m_restart_timeout }}', shutdown_port: '-1', java_home: '{{ jdk_java_home }}', user: '{{ smartgears_user }}', user_home: '{{ smartgears_user_home }}', user_shell: '{{ tomcat_m_default_user_shell }}', instance_path: '{{ smartgears_instance_path }}', max_threads: '{{ tomcat_m_max_threads }}', autodeploy: '{{ tomcat_m_webapps_autodeploy }}', unpack: '{{ tomcat_m_webapps_unpack }}', default_conf: True, java_opts: '{{ tomcat_m_java_opts }}', java_gc_opts: '{{ tomcat_m_java_gc_opts }}', other_java_opts: '{{ tomcat_m_other_java_opts }}', jmx_enabled: '{{ tomcat_m_jmx_enabled }}', jmx_disable_additional_ports: '{{ tomcat_m_jmx_disable_additional_ports }}', jmx_port: '{{ tomcat_m_jmx_port }}', jmx_use_ssl: '{{ tomcat_m_jmx_use_ssl }}', jmx_auth_enabled: '{{ tomcat_m_jmx_auth_enabled }}', jmx_auth_dir: '{{ tomcat_m_instances_base_path }}/{{ smartgears_http_port }}/conf', jmx_monitorpass: '{{ tomcat_m_jmx_monitorpass | default(omit) }}', jmx_controlpass: '{{ tomcat_m_jmx_controlpass | default(omit) }}', catalina_tmp_directory: '{{ smartgears_tomcat_tmp_dir }}', remote_debugging: '{{ tomcat_m_enable_remote_debugging }}', remote_debugging_port: '{{ smartgears_debugging_port }}', access_log_enabled: True, log_rotation_freq: daily, log_retain: 30, allowed_hosts: [ '0.0.0.0/0' ], app_contexts: '{{ smartgears_tomcat_contexts }}', web_document_root: '{{ smartgears_web_document_root }}' , skip_smartgears: False, smartgears_id: 1, servername: '{{ ansible_fqdn }}' }
+  - {  http_enabled: True, http_port: '{{ smartgears_http_port }}', http_address: '{{ smartgears_tomcat_listen_address }}', ajp_enabled: False, ajp_port: '8109', ajp_address: '127.0.0.1', restart_timeout: '{{ tomcat_m_restart_timeout }}', shutdown_port: '-1', java_home: '{{ jdk_java_home }}', user: '{{ smartgears_user }}', user_home: '{{ smartgears_user_home }}', user_shell: '{{ tomcat_m_default_user_shell }}', instance_path: '{{ smartgears_instance_path }}', max_threads: '{{ tomcat_m_max_threads }}', autodeploy: '{{ tomcat_m_webapps_autodeploy }}', unpack: '{{ tomcat_m_webapps_unpack }}', default_conf: True, java_opts: '{{ tomcat_m_java_opts }}', java_gc_opts: '{{ tomcat_m_java_gc_opts }}', other_java_opts: '{{ tomcat_m_other_java_opts }}', jmx_enabled: '{{ tomcat_m_jmx_enabled }}', jmx_disable_additional_ports: '{{ tomcat_m_jmx_disable_additional_ports }}', jmx_port: '{{ tomcat_m_jmx_port }}', jmx_use_ssl: '{{ tomcat_m_jmx_use_ssl }}', jmx_auth_enabled: '{{ tomcat_m_jmx_auth_enabled }}', jmx_auth_dir: '{{ tomcat_m_instances_base_path }}/{{ smartgears_http_port }}/conf', jmx_monitorpass: '{{ tomcat_m_jmx_monitorpass | default(omit) }}', jmx_controlpass: '{{ tomcat_m_jmx_controlpass | default(omit) }}', catalina_tmp_directory: '{{ smartgears_tomcat_tmp_dir }}', remote_debugging: '{{ tomcat_m_enable_remote_debugging }}', remote_debugging_port: '{{ smartgears_debugging_port }}', access_log_enabled: True, log_rotation_freq: daily, log_retain: 30, allowed_hosts: [ '0.0.0.0/0' ], app_contexts: '{{ smartgears_tomcat_contexts }}', web_document_root: '{{ smartgears_web_document_root }}' , skip_smartgears: False, smartgears_id: 1, servername: '{{ smartgears_tomcat_servername }}' }

smartgears/smartgears/defaults/main.yml

@@ -71,6 +71,7 @@ smartgears_logback_template: True
 
 smartgears_tomcat_contexts: [ '' ]
 smartgears_tomcat_listen_address: '127.0.0.1'
+smartgears_tomcat_servername: '{{ ansible_fqdn }}'
 
 # The iptables rules use this
 http_port: '{{ smartgears_http_port }}'
@@ -80,7 +81,7 @@ smartgears_web_document_root: '/usr/share/nginx/html/'
 
 tomcat_m_webapps_unpack: True
 tomcat_m_instances:
-  - {  http_enabled: True, http_port: '{{ smartgears_http_port }}', http_address: '{{ smartgears_tomcat_listen_address }}', ajp_enabled: False, ajp_port: '8109', ajp_address: '127.0.0.1', restart_timeout: '{{ tomcat_m_restart_timeout }}', shutdown_port: '-1', java_home: '{{ jdk_java_home }}', user: '{{ smartgears_user }}', user_home: '{{ smartgears_user_home }}', user_shell: '{{ tomcat_m_default_user_shell }}', instance_path: '{{ smartgears_instance_path }}', max_threads: '{{ tomcat_m_max_threads }}', autodeploy: '{{ tomcat_m_webapps_autodeploy }}', unpack: '{{ tomcat_m_webapps_unpack }}', default_conf: True, java_opts: '{{ tomcat_m_java_opts }}', java_gc_opts: '{{ tomcat_m_java_gc_opts }}', other_java_opts: '{{ tomcat_m_other_java_opts }}', jmx_enabled: '{{ tomcat_m_jmx_enabled }}', jmx_disable_additional_ports: '{{ tomcat_m_jmx_disable_additional_ports }}', jmx_port: '{{ tomcat_m_jmx_port }}', jmx_use_ssl: '{{ tomcat_m_jmx_use_ssl }}', jmx_auth_enabled: '{{ tomcat_m_jmx_auth_enabled }}', jmx_auth_dir: '{{ tomcat_m_instances_base_path }}/{{ smartgears_http_port }}/conf', jmx_monitorpass: '{{ tomcat_m_jmx_monitorpass | default(omit) }}', jmx_controlpass: '{{ tomcat_m_jmx_controlpass | default(omit) }}', catalina_tmp_directory: '{{ smartgears_tomcat_tmp_dir }}', remote_debugging: '{{ tomcat_m_enable_remote_debugging }}', remote_debugging_port: '{{ smartgears_debugging_port }}', access_log_enabled: True, log_rotation_freq: daily, log_retain: 30, allowed_hosts: [ '0.0.0.0/0' ], app_contexts: '{{ smartgears_tomcat_contexts }}', web_document_root: '{{ smartgears_web_document_root }}' , skip_smartgears: False, smartgears_id: 1, servername: '{{ ansible_fqdn }}' }
+  - {  http_enabled: True, http_port: '{{ smartgears_http_port }}', http_address: '{{ smartgears_tomcat_listen_address }}', ajp_enabled: False, ajp_port: '8109', ajp_address: '127.0.0.1', restart_timeout: '{{ tomcat_m_restart_timeout }}', shutdown_port: '-1', java_home: '{{ jdk_java_home }}', user: '{{ smartgears_user }}', user_home: '{{ smartgears_user_home }}', user_shell: '{{ tomcat_m_default_user_shell }}', instance_path: '{{ smartgears_instance_path }}', max_threads: '{{ tomcat_m_max_threads }}', autodeploy: '{{ tomcat_m_webapps_autodeploy }}', unpack: '{{ tomcat_m_webapps_unpack }}', default_conf: True, java_opts: '{{ tomcat_m_java_opts }}', java_gc_opts: '{{ tomcat_m_java_gc_opts }}', other_java_opts: '{{ tomcat_m_other_java_opts }}', jmx_enabled: '{{ tomcat_m_jmx_enabled }}', jmx_disable_additional_ports: '{{ tomcat_m_jmx_disable_additional_ports }}', jmx_port: '{{ tomcat_m_jmx_port }}', jmx_use_ssl: '{{ tomcat_m_jmx_use_ssl }}', jmx_auth_enabled: '{{ tomcat_m_jmx_auth_enabled }}', jmx_auth_dir: '{{ tomcat_m_instances_base_path }}/{{ smartgears_http_port }}/conf', jmx_monitorpass: '{{ tomcat_m_jmx_monitorpass | default(omit) }}', jmx_controlpass: '{{ tomcat_m_jmx_controlpass | default(omit) }}', catalina_tmp_directory: '{{ smartgears_tomcat_tmp_dir }}', remote_debugging: '{{ tomcat_m_enable_remote_debugging }}', remote_debugging_port: '{{ smartgears_debugging_port }}', access_log_enabled: True, log_rotation_freq: daily, log_retain: 30, allowed_hosts: [ '0.0.0.0/0' ], app_contexts: '{{ smartgears_tomcat_contexts }}', web_document_root: '{{ smartgears_web_document_root }}' , skip_smartgears: False, smartgears_id: 1, servername: '{{ smartgears_tomcat_servername }}' }
 
 # To enable debugging:
 # - Set

tomcat-multiple-instances/templates/tomcat-server.xml.j2

@@ -56,12 +56,16 @@
 {% if item.http_enabled %}
     <!-- A http "Connector" using the shared thread pool-->
     <Connector executor="tomcatThreadPool"
-	           enableLookups="false"
+	       enableLookups="false"
                maxQueueSize="{{ item.max_queue_size | default(32767) }}"
                maxThreads="{{ item.max_threads }}" connectionTimeout="60000"
-	           URIEncoding="UTF-8" bindOnInit="false" address="{{ item.http_address }}"
+	       URIEncoding="UTF-8"
+               bindOnInit="false" address="{{ item.http_address }}"
                port="{{ item.http_port }}" protocol="HTTP/1.1"
                maxPostSize="{{ item.max_post_size | default(104857600) }}"
+               useBodyEncodingForURI="true"
+               maxHttpHeaderSize="8192"
+               disableUploadTimeout="true"
                {% if tomcat_m_reverse_proxy_name_enabled %}
                proxyName="{{ tomcat_m_reverse_proxy_name }}"
                proxyPort="{{ tomcat_m_reverse_proxy_port }}"
@@ -72,9 +76,12 @@
 {% if item.ajp_enabled %}
     <!-- Define an AJP 1.3 Connector on port {{ tomcat_ajp_port }} -->
     <Connector port="{{ item.ajp_port }}" protocol="AJP/1.3"
-	           enableLookups="false"
+	       enableLookups="false"
                address="{{ item.ajp_address }}"
                URIEncoding="UTF-8"
+               useBodyEncodingForURI="true"
+               maxHttpHeaderSize="8192"
+               disableUploadTimeout="true"
                maxQueueSize="{{ item.max_queue_size | default(32767) }}"
                maxThreads="{{ item.max_threads }}"
                connectionTimeout="60000"
@@ -83,7 +90,8 @@
                proxyName="{{ tomcat_m_reverse_proxy_name }}"
                proxyPort="{{ tomcat_m_reverse_proxy_port }}"
                {% endif %}
-	           URIEncoding="UTF-8" bindOnInit="false" />
+	       URIEncoding="UTF-8"
+               bindOnInit="false" />
 {% endif %}
     
     <Engine name="Catalina" defaultHost="localhost">

tomcat/templates/tomcat-server.xml.j2

@@ -79,25 +79,30 @@
 {% if tomcat_http_enabled %}
     <!-- A http "Connector" using the shared thread pool-->
     <Connector executor="tomcatThreadPool"
-	           enableLookups="false"
+	       enableLookups="false"
                maxQueueSize="{{ tomcat_max_queue_size | default(32767) }}"
                maxThreads="{{ tomcat_max_threads }}" connectionTimeout="60000"
-	           URIEncoding="UTF-8" bindOnInit="false" address="{{ tomcat_http_address }}"
+	       URIEncoding="UTF-8"
+               useBodyEncodingForURI="true"
+               bindOnInit="false" address="{{ tomcat_http_address }}"
                port="{{ tomcat_http_port }}" protocol="HTTP/1.1"
                maxPostSize="{{ tomcat_max_post_size | default(104857600) }}"
                {% if tomcat_reverse_proxy_name_enabled %}
                proxyName="{{ tomcat_reverse_proxy_name }}"
                proxyPort="{{ tomcat_reverse_proxy_port }}"
                {% endif %}
+               maxHttpHeaderSize="8192"
+               disableUploadTimeout="true"
                />
 
 {% endif %}
 {% if tomcat_ajp_enabled %}
     <!-- Define an AJP 1.3 Connector on port {{ tomcat_ajp_port }} -->
     <Connector port="{{ tomcat_ajp_port }}" protocol="AJP/1.3"
-	           enableLookups="false" address="{{ tomcat_ajp_address }}"
+	       enableLookups="false" address="{{ tomcat_ajp_address }}"
                maxQueueSize="{{ tomcat_max_queue_size | default(32767) }}"
                URIEncoding="UTF-8"
+               useBodyEncodingForURI="true"
                maxThreads="{{ tomcat_max_threads }}"
                connectionTimeout="60000"
                maxPostSize="{{ tomcat_max_post_size | default(104857600) }}"
@@ -105,7 +110,9 @@
                proxyName="{{ tomcat_reverse_proxy_name }}"
                proxyPort="{{ tomcat_reverse_proxy_port }}"
                {% endif %}
-	           URIEncoding="UTF-8" bindOnInit="false" />
+               maxHttpHeaderSize="8192"
+               disableUploadTimeout="true"
+	       bindOnInit="false" />
 {% endif %}
     
     <!-- An Engine represents the entry point (within Catalina) that processes