Merge branch 'master' of gitorious.research-infrastructures.eu:infrastructure-management/ansible-playbooks

solr/defaults/main.yml

@@ -1,13 +1,13 @@
 ---
 # solr
 solr_install: False
+solr_version: 6.6.0  
 solr_http_port: 8983
-
-solr_version: 5.5.4
 solr_service: -solr
 solr_config_name: hindex
 solr_shards: 1
 solr_instance: '{{ solr_service }}'
+solr_log_dir: /var/log/solr
 solr_log_level: INFO
 solr_http_port_1: '{{ solr_http_port }}'
 solr_zoo_port: 9983
@@ -17,8 +17,11 @@ solr_jmx_port_1: 8601
 solr_user: solr
 solr_group: solr
 # We need to define this one because we are using the tomcat multiple instances role
-solr_base_dir: /opt/solr-files
+solr_base_dir: /opt
-solr_data_dir: '{{ solr_base_dir }}/solr'
+solr_server_dir: '{{ solr_base_dir }}/solr'
+solr_data_dir: '{{ solr_server_dir }}/server/solr'
+solr_collections_base_dir: '{{ solr_data_dir }}/data/solr'
+solr_pid_dir: '{{ solr_server_dir }}/bin'
 solr_heap:  1024M
 solr_mode: solrcloud
 solr_download_dir: '{{ solr_base_dir }}/download'
@@ -29,4 +32,16 @@ solr_jar_files:
   - contrib/analysis-extras/lib/icu4*.jar
   - contrib/analysis-extras/lucene-libs/lucene-analyzers-icu*.jar
 
+solr_set_gc_options: False
+#solr_gc_tune: '-XX:NewRatio=3 -XX:SurvivorRatio=4'
+solr_gc_tune: ''
 
+solr_multicore: False
+solr_cores:
+  - collection1
+
+solr_cloud_mode: True
+# When we want to use an external zookeeper cluster, set this one to True
+solr_zk_external_cluster: False
+solr_zk_hosts: '127.0.0.1:2181'
+solr_zk_timeout: 15000

solr/files/solr

@@ -34,29 +34,27 @@
 
 # Where you extracted the Solr distribution bundle
 SOLR_INSTALL_DIR="/opt/solr"
-# Specify the user to run Solr as; if not set, then Solr will run as root.
-# Running Solr as root is not recommended for production environments
-
-# Path to an include file that defines environment specific settings to override default
-# variables used by the bin/solr script. It's highly recommended to define this script so
-# that you can keep the Solr binary files separated from live files (pid, logs, index data, etc)
-# see bin/solr.in.sh for an example
-SOLR_ENV="/etc/default/solr"
-
-RUNAS="solr"
-
-if [ ! -f "$SOLR_ENV" ]; then
-  echo "$SOLR_ENV not found! Please check the SOLR_ENV setting in your $0 script."
-  exit 1
-else
-	. $SOLR_ENV
-fi
 
 if [ ! -d "$SOLR_INSTALL_DIR" ]; then
   echo "$SOLR_INSTALL_DIR not found! Please check the SOLR_INSTALL_DIR setting in your $0 script."
   exit 1
 fi
 
+# Path to an include file that defines environment specific settings to override default
+# variables used by the bin/solr script. It's highly recommended to define this script so
+# that you can keep the Solr binary files separated from live files (pid, logs, index data, etc)
+# see bin/solr.in.sh for an example
+SOLR_ENV="/etc/default/solr.in.sh"
+
+if [ ! -f "$SOLR_ENV" ]; then
+  echo "$SOLR_ENV not found! Please check the SOLR_ENV setting in your $0 script."
+  exit 1
+fi
+
+# Specify the user to run Solr as; if not set, then Solr will run as root.
+# Running Solr as root is not recommended for production environments
+RUNAS="solr"
+
 # verify the specified run as user exists
 runas_uid="`id -u "$RUNAS"`"
 if [ $? -ne 0 ]; then

solr/handlers/main.yml

@@ -1,4 +1,4 @@
 ---
-- name: tomcat solr restart
+- name: solr restart
-  service: name=tomcat-instance-'{{ solr_http_port }}' state=restarted sleep=20
+  service: name=solr state=restarted
 

solr/tasks/main.yml

@@ -1,43 +1,56 @@
 ---
 - block:
-
     - name: Create a solr user
-      become: False
       user: name={{ solr_user }} home={{ solr_base_dir }} createhome=yes shell=/bin/bash
 
-    - name: Create the solr service and download directories
-      file: path={{ item }} state=directory
-      with_items: [ '{{ solr_download_dir }}' ]
-
     - name: Download solr
       get_url: url='{{ solr_download_url }}' dest={{ solr_download_dir }}/solr-{{ solr_version }}.tgz validate_certs=no
 
     - name: Unarchive solr release
       unarchive: src={{ solr_download_dir }}/solr-{{ solr_version }}.tgz dest={{ solr_base_dir }} copy=no
+      args:
+        creates: '{{ solr_base_dir }}/bin/solr'
 
     - name: Link to the latest release
-      file: src=solr-{{ solr_version }} dest={{ solr_data_dir }} state=link
+      file: src={{ solr_base_dir }}/solr-{{ solr_version }} dest={{ solr_server_dir }} state=link
       
-    - name: create solr lib dir
+    - name: Create the solr service, download, data and logs directories
-      file: path={{ solr_data_dir }}/server/solr/lib state=directory
+      file: path={{ item }} state=directory owner={{ solr_user }} group={{ solr_user }}
+      with_items:
+        - '{{ solr_download_dir }}'
+        - '{{ solr_data_dir }}'
+        - '{{ solr_pid_dir }}'
+        - '{{ solr_log_dir }}'
+
+    - name: Fix the permissions of the solr server directory
+      file: dest={{ solr_data_dir }} state=directory owner={{ solr_user }} group={{ solr_user }} recurse=yes
+
+    - name: Create the solr lib directory
+      file: dest={{ solr_data_dir }}/lib state=directory
+
+    - name: Install the zookeeper configuration
+      template: src={{ item }}.j2 dest={{ solr_data_dir }}/{{ item }}
+      with_items:
+        - solr.xml
+        - zoo.cfg
+      notify: solr restart
 
     - name: copy extra libraries
-      shell: cd {{ solr_data_dir }}; cp -u {{ item }} {{ solr_data_dir }}/server/solr/lib
+      shell: cd {{ solr_server_dir }}; cp -u {{ item }} {{ solr_data_dir }}/lib
       with_items: '{{ solr_jar_files }}'
 
     - name: install the solr init script
-      become: False
       copy: src=solr  dest=/etc/init.d/solr mode=0755 owner=root group=root
 
     - name: Install the solr default
-      become: False
+      template: src=solr-default.j2 dest=/etc/default/solr.in.sh mode=0644 owner=root group=root
-      template: src=solr-default.j2 dest=/etc/default/solr mode=0644 owner=root group=root
+      notify: solr restart
 
     - name: Start and enable the solr service
-      become: False
       service: name=solr state=started enabled=yes
+      ignore_errors: True
 
   become: True
-  become_user: '{{ solr_user }}' 
+  become_user: root
   when: solr_install
   tags: solr

solr/templates/solr-default.j2

@@ -1,6 +1,3 @@
-SOLR_INSTALL_DIR="{{ solr_data_dir }}"
-RUNAS="{{ solr_user }}"
-
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -17,11 +14,19 @@ RUNAS="{{ solr_user }}"
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+SOLR_INSTALL_DIR="{{ solr_data_dir }}"
+
 # By default the script will use JAVA_HOME to determine which java
 # to use, but you can set a specific path for Solr to use without
 # affecting other Java applications on your server/workstation.
 #SOLR_JAVA_HOME=""
 
+# This controls the number of seconds that the solr script will wait for
+# Solr to stop gracefully or Solr to start.  If the graceful stop fails,
+# the script will forcibly stop Solr.  If the start fails, the script will
+# give up waiting and display the last few lines of the logfile.
+#SOLR_STOP_WAIT="180"
+
 # Increase Java Heap as needed to support your indexing / query needs
 SOLR_HEAP="{{ solr_heap }}"
 
@@ -29,33 +34,33 @@ SOLR_HEAP="{{ solr_heap }}"
 # Comment out SOLR_HEAP if you are using this though, that takes precedence
 #SOLR_JAVA_MEM="-Xms512m -Xmx512m"
 
-# Enable verbose GC logging
+# Enable verbose GC logging...
+#  * If this is unset, various default options will be selected depending on which JVM version is in use
+#  * For java8 or lower: if this is set, additional params will be added to specify the log file & rotation
+#  * For java9 or higher: each included opt param that starts with '-Xlog:gc', but does not include an output
+#    specifier, will have a 'file' output specifier (as well as formatting & rollover options) appended,
+#    using the effective value of the SOLR_LOGS_DIR.
+#
+#GC_LOG_OPTS='-Xlog:gc*'  # (java9)
+#GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \
+#  -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"
+{% if solr_set_gc_options %}
 GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \
--XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"
+  -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"
+{% endif %}
 
 # These GC settings have shown to work well for a number of common Solr workloads
-GC_TUNE="-XX:NewRatio=3 \
+GC_TUNE="{{ solr_gc_tune }}"
--XX:SurvivorRatio=4 \
--XX:TargetSurvivorRatio=90 \
--XX:MaxTenuringThreshold=8 \
--XX:+UseConcMarkSweepGC \
--XX:+UseParNewGC \
--XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \
--XX:+CMSScavengeBeforeRemark \
--XX:PretenureSizeThreshold=64m \
--XX:+UseCMSInitiatingOccupancyOnly \
--XX:CMSInitiatingOccupancyFraction=50 \
--XX:CMSMaxAbortablePrecleanTime=6000 \
--XX:+CMSParallelRemarkEnabled \
--XX:+ParallelRefProcEnabled"
 
 # Set the ZooKeeper connection string if using an external ZooKeeper ensemble
 # e.g. host1:2181,host2:2181/chroot
 # Leave empty if not using SolrCloud
-#ZK_HOST=""
+{% if solr_zk_external_cluster %}
+ZK_HOST="{{ solr_zk_hosts }}"
+{% endif %}
 
 # Set the ZooKeeper client timeout (for SolrCloud mode)
-#ZK_CLIENT_TIMEOUT="15000"
+ZK_CLIENT_TIMEOUT="{{ solr_zk_timeout }}"
 
 # By default the start script uses "localhost"; override the hostname here
 # for production SolrCloud environments to control the hostname exposed to cluster state
@@ -75,7 +80,9 @@ ENABLE_REMOTE_JMX_OPTS="false"
 # Set the thread stack size
 SOLR_OPTS="$SOLR_OPTS -Xss256k"
 
-SOLR_MODE={{ solr_mode }}
+{% if not solr_zk_external_cluster %}
+export SOLR_MODE={{ solr_mode }}
+{% endif %}
 
 # Anything you add to the SOLR_OPTS variable will be included in the java
 # start command line as-is, in ADDITION to other options. If you specify the
@@ -86,30 +93,41 @@ SOLR_MODE={{ solr_mode }}
 
 # Location where the bin/solr script will save PID files for running instances
 # If not set, the script will create PID files in $SOLR_TIP/bin
-#SOLR_PID_DIR=
+SOLR_PID_DIR={{ solr_pid_dir }}
 
 # Path to a directory for Solr to store cores and their data. By default, Solr will use server/solr
 # If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
-#SOLR_HOME=
+SOLR_HOME={{ solr_data_dir }}
 
 # Solr provides a default Log4J configuration properties file in server/resources
 # however, you may want to customize the log settings and file appender location
 # so you can point the script to use a different log4j.properties file
 #LOG4J_PROPS=/var/solr/log4j.properties
 
+# Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
+# This is an alternative to changing the rootLogger in log4j.properties
+SOLR_LOG_LEVEL={{ solr_log_level }}
+
 # Location where Solr should write logs to; should agree with the file appender
 # settings in server/resources/log4j.properties
-#SOLR_LOGS_DIR=
+SOLR_LOGS_DIR={{ solr_log_dir }}
+
+# Enables log rotation, cleanup, and archiving during start. Setting SOLR_LOG_PRESTART_ROTATION=false will skip start
+# time rotation of logs, and the archiving of the last GC and console log files. It does not affect Log4j configuration.
+# This pre-startup rotation may need to be disabled depending how much you customize the default logging setup.
+#SOLR_LOG_PRESTART_ROTATION=true
 
 # Sets the port Solr binds to, default is 8983
-#SOLR_PORT=8983
+SOLR_PORT={{ solr_http_port }}
 
 # Uncomment to set SSL-related system properties
 # Be sure to update the paths to the correct keystore for your environment
-#SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
+#SOLR_SSL_KEY_STORE=/home/shalin/work/oss/shalin-lusolr/solr/server/etc/solr-ssl.keystore.jks
 #SOLR_SSL_KEY_STORE_PASSWORD=secret
-#SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
+#SOLR_SSL_KEY_STORE_TYPE=JKS
+#SOLR_SSL_TRUST_STORE=/home/shalin/work/oss/shalin-lusolr/solr/server/etc/solr-ssl.keystore.jks
 #SOLR_SSL_TRUST_STORE_PASSWORD=secret
+#SOLR_SSL_TRUST_STORE_TYPE=JKS
 #SOLR_SSL_NEED_CLIENT_AUTH=false
 #SOLR_SSL_WANT_CLIENT_AUTH=false
 
@@ -117,10 +135,21 @@ SOLR_MODE={{ solr_mode }}
 # otherwise keep them commented and the above values will automatically be set for HTTP clients
 #SOLR_SSL_CLIENT_KEY_STORE=
 #SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
+#SOLR_SSL_CLIENT_KEY_STORE_TYPE=
 #SOLR_SSL_CLIENT_TRUST_STORE=
 #SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
+#SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
 
 # Settings for authentication
-#SOLR_AUTHENTICATION_CLIENT_CONFIGURER=
+# Please configure only one of SOLR_AUTHENTICATION_CLIENT_CONFIGURER or SOLR_AUTH_TYPE parameters
-#SOLR_AUTHENTICATION_OPTS=
+#SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthConfigurer"
+#SOLR_AUTH_TYPE="basic"
+#SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks"
+
+# Settings for ZK ACL
+#SOLR_ZK_CREDS_AND_ACLS="-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider \
+#  -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider \
+#  -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD \
+#  -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD"
+#SOLR_OPTS="$SOLR_OPTS $SOLR_ZK_CREDS_AND_ACLS"
 

solr/templates/solr.xml.j2

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+   This is an example of a simple "solr.xml" file for configuring one or 
+   more Solr Cores, as well as allowing Cores to be added, removed, and 
+   reloaded via HTTP requests.
+
+   More information about options available in this configuration file, 
+   and Solr Core administration can be found online:
+   http://wiki.apache.org/solr/CoreAdmin
+-->
+
+<solr>
+
+  <solrcloud>
+    {% if not solr_multicore %}
+    <str name="host">{{ ansible_fqdn }}</str>
+    <int name="hostPort">{{ solr_http_port_1 }}</int>
+    <str name="hostContext">${hostContext:solr}</str>
+    <int name="zkClientTimeout">{{ solr_zk_timeout }}</int>
+    <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+    {% if solr_cloud_mode %}
+    <str name="zkHost">{{ solr_zk_hosts }}</str>
+    {% endif %}
+    {% endif %}
+  </solrcloud>
+
+  <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
+    <int name="socketTimeout">${socketTimeout:0}</int>
+    <int name="connTimeout">${connTimeout:0}</int>
+  </shardHandlerFactory>
+
+  {% if solr_multicore %}
+  <cores adminPath="/admin/cores">
+    {% for core in solr_cores %}
+    
+    <core name="{{ core }}" instanceDir="{{ core }}">
+      <property name="dataDir" value="{{ solr_collections_base_dir }}/{{ core }}" />
+    </core>
+    
+    {% endfor %}
+  </cores>
+  {% endif %}
+
+
+</solr>

solr/templates/zoo.cfg.j2

@@ -0,0 +1,17 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+
+# the directory where the snapshot is stored.
+dataDir={{ solr_zookeeper_data_dir }}
+# NOTE: Solr defaults the dataDir to <solrHome>/zoo_data
+
+# the port at which the clients will connect
+clientPort={{ solr_zoo_port }}
+# NOTE: Solr sets this based on zkRun / zkHost params
+

thredds/defaults/main.yml

@@ -10,7 +10,7 @@ thredds_file: '{{ thredds_name }}-{{ thredds_version }}.{{ thredds_extension }}'
 thredds_repository_url: 'https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases'
 thredds_download_directory: /srv
 thredds_data_content_dir: /data/thredds
-thredds_public_netcdf_dir: '{{ thredds_data_content_dir }}/public/netcdf/'
+thredds_public_netcdf_dir: '{{ thredds_data_content_dir }}/thredds/public/netcdf/'
 thredds_server_name: 'Thredds instance'
 thredds_logo_url: '/thredds/threddsIcon.gif'
 thredds_logo_alt_text: '{{ thredds_server_name }}'

thredds/tasks/main.yml

@@ -22,6 +22,8 @@
       with_items:
         - threddsConfig.xml
         - catalog.xml
+      notify: tomcat instances restart
+      tags: [ 'thredds', 'tomcat', 'thredds_conf' ]
 
     - name: Get the thredds war file
       maven_artifact: artifact_id={{ thredds_name }} version={{ thredds_version | default(omit) }} group_id={{ thredds_group_id }} extension={{ thredds_extension | default('war') }} repository_url={{ thredds_repository_url }} dest={{ thredds_download_directory }}/{{ thredds_file }}