diff --git a/solr/defaults/main.yml b/solr/defaults/main.yml index 55475ed1..53e26bfc 100644 --- a/solr/defaults/main.yml +++ b/solr/defaults/main.yml @@ -1,13 +1,13 @@ --- # solr solr_install: False +solr_version: 6.6.0 solr_http_port: 8983 - -solr_version: 5.5.4 solr_service: -solr solr_config_name: hindex solr_shards: 1 solr_instance: '{{ solr_service }}' +solr_log_dir: /var/log/solr solr_log_level: INFO solr_http_port_1: '{{ solr_http_port }}' solr_zoo_port: 9983 @@ -17,8 +17,11 @@ solr_jmx_port_1: 8601 solr_user: solr solr_group: solr # We need to define this one because we are using the tomcat multiple instances role -solr_base_dir: /opt/solr-files -solr_data_dir: '{{ solr_base_dir }}/solr' +solr_base_dir: /opt +solr_server_dir: '{{ solr_base_dir }}/solr' +solr_data_dir: '{{ solr_server_dir }}/server/solr' +solr_collections_base_dir: '{{ solr_data_dir }}/data/solr' +solr_pid_dir: '{{ solr_server_dir }}/bin' solr_heap: 1024M solr_mode: solrcloud solr_download_dir: '{{ solr_base_dir }}/download' @@ -29,4 +32,16 @@ solr_jar_files: - contrib/analysis-extras/lib/icu4*.jar - contrib/analysis-extras/lucene-libs/lucene-analyzers-icu*.jar +solr_set_gc_options: False +#solr_gc_tune: '-XX:NewRatio=3 -XX:SurvivorRatio=4' +solr_gc_tune: '' +solr_multicore: False +solr_cores: + - collection1 + +solr_cloud_mode: True +# When we want to use an external zookeeper cluster, set this one to True +solr_zk_external_cluster: False +solr_zk_hosts: '127.0.0.1:2181' +solr_zk_timeout: 15000 diff --git a/solr/files/solr b/solr/files/solr index 7412bb36..e73e0d68 100755 --- a/solr/files/solr +++ b/solr/files/solr @@ -34,29 +34,27 @@ # Where you extracted the Solr distribution bundle SOLR_INSTALL_DIR="/opt/solr" -# Specify the user to run Solr as; if not set, then Solr will run as root. -# Running Solr as root is not recommended for production environments - -# Path to an include file that defines environment specific settings to override default -# variables used by the bin/solr script. It's highly recommended to define this script so -# that you can keep the Solr binary files separated from live files (pid, logs, index data, etc) -# see bin/solr.in.sh for an example -SOLR_ENV="/etc/default/solr" - -RUNAS="solr" - -if [ ! -f "$SOLR_ENV" ]; then - echo "$SOLR_ENV not found! Please check the SOLR_ENV setting in your $0 script." - exit 1 -else - . $SOLR_ENV -fi if [ ! -d "$SOLR_INSTALL_DIR" ]; then echo "$SOLR_INSTALL_DIR not found! Please check the SOLR_INSTALL_DIR setting in your $0 script." exit 1 fi +# Path to an include file that defines environment specific settings to override default +# variables used by the bin/solr script. It's highly recommended to define this script so +# that you can keep the Solr binary files separated from live files (pid, logs, index data, etc) +# see bin/solr.in.sh for an example +SOLR_ENV="/etc/default/solr.in.sh" + +if [ ! -f "$SOLR_ENV" ]; then + echo "$SOLR_ENV not found! Please check the SOLR_ENV setting in your $0 script." + exit 1 +fi + +# Specify the user to run Solr as; if not set, then Solr will run as root. +# Running Solr as root is not recommended for production environments +RUNAS="solr" + # verify the specified run as user exists runas_uid="`id -u "$RUNAS"`" if [ $? -ne 0 ]; then diff --git a/solr/handlers/main.yml b/solr/handlers/main.yml index 15331111..77b22e49 100644 --- a/solr/handlers/main.yml +++ b/solr/handlers/main.yml @@ -1,4 +1,4 @@ --- -- name: tomcat solr restart - service: name=tomcat-instance-'{{ solr_http_port }}' state=restarted sleep=20 +- name: solr restart + service: name=solr state=restarted diff --git a/solr/tasks/main.yml b/solr/tasks/main.yml index 989aa940..eeafb974 100644 --- a/solr/tasks/main.yml +++ b/solr/tasks/main.yml @@ -1,43 +1,56 @@ --- - block: - - name: Create a solr user - become: False user: name={{ solr_user }} home={{ solr_base_dir }} createhome=yes shell=/bin/bash - - name: Create the solr service and download directories - file: path={{ item }} state=directory - with_items: [ '{{ solr_download_dir }}' ] - - name: Download solr get_url: url='{{ solr_download_url }}' dest={{ solr_download_dir }}/solr-{{ solr_version }}.tgz validate_certs=no - name: Unarchive solr release unarchive: src={{ solr_download_dir }}/solr-{{ solr_version }}.tgz dest={{ solr_base_dir }} copy=no + args: + creates: '{{ solr_base_dir }}/bin/solr' - name: Link to the latest release - file: src=solr-{{ solr_version }} dest={{ solr_data_dir }} state=link + file: src={{ solr_base_dir }}/solr-{{ solr_version }} dest={{ solr_server_dir }} state=link - - name: create solr lib dir - file: path={{ solr_data_dir }}/server/solr/lib state=directory + - name: Create the solr service, download, data and logs directories + file: path={{ item }} state=directory owner={{ solr_user }} group={{ solr_user }} + with_items: + - '{{ solr_download_dir }}' + - '{{ solr_data_dir }}' + - '{{ solr_pid_dir }}' + - '{{ solr_log_dir }}' + + - name: Fix the permissions of the solr server directory + file: dest={{ solr_data_dir }} state=directory owner={{ solr_user }} group={{ solr_user }} recurse=yes + + - name: Create the solr lib directory + file: dest={{ solr_data_dir }}/lib state=directory + + - name: Install the zookeeper configuration + template: src={{ item }}.j2 dest={{ solr_data_dir }}/{{ item }} + with_items: + - solr.xml + - zoo.cfg + notify: solr restart - name: copy extra libraries - shell: cd {{ solr_data_dir }}; cp -u {{ item }} {{ solr_data_dir }}/server/solr/lib + shell: cd {{ solr_server_dir }}; cp -u {{ item }} {{ solr_data_dir }}/lib with_items: '{{ solr_jar_files }}' - name: install the solr init script - become: False copy: src=solr dest=/etc/init.d/solr mode=0755 owner=root group=root - name: Install the solr default - become: False - template: src=solr-default.j2 dest=/etc/default/solr mode=0644 owner=root group=root + template: src=solr-default.j2 dest=/etc/default/solr.in.sh mode=0644 owner=root group=root + notify: solr restart - name: Start and enable the solr service - become: False service: name=solr state=started enabled=yes + ignore_errors: True become: True - become_user: '{{ solr_user }}' + become_user: root when: solr_install tags: solr diff --git a/solr/templates/solr-default.j2 b/solr/templates/solr-default.j2 index 8901769e..615e68d5 100644 --- a/solr/templates/solr-default.j2 +++ b/solr/templates/solr-default.j2 @@ -1,6 +1,3 @@ -SOLR_INSTALL_DIR="{{ solr_data_dir }}" -RUNAS="{{ solr_user }}" - # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -17,11 +14,19 @@ RUNAS="{{ solr_user }}" # See the License for the specific language governing permissions and # limitations under the License. +SOLR_INSTALL_DIR="{{ solr_data_dir }}" + # By default the script will use JAVA_HOME to determine which java # to use, but you can set a specific path for Solr to use without # affecting other Java applications on your server/workstation. #SOLR_JAVA_HOME="" +# This controls the number of seconds that the solr script will wait for +# Solr to stop gracefully or Solr to start. If the graceful stop fails, +# the script will forcibly stop Solr. If the start fails, the script will +# give up waiting and display the last few lines of the logfile. +#SOLR_STOP_WAIT="180" + # Increase Java Heap as needed to support your indexing / query needs SOLR_HEAP="{{ solr_heap }}" @@ -29,33 +34,33 @@ SOLR_HEAP="{{ solr_heap }}" # Comment out SOLR_HEAP if you are using this though, that takes precedence #SOLR_JAVA_MEM="-Xms512m -Xmx512m" -# Enable verbose GC logging +# Enable verbose GC logging... +# * If this is unset, various default options will be selected depending on which JVM version is in use +# * For java8 or lower: if this is set, additional params will be added to specify the log file & rotation +# * For java9 or higher: each included opt param that starts with '-Xlog:gc', but does not include an output +# specifier, will have a 'file' output specifier (as well as formatting & rollover options) appended, +# using the effective value of the SOLR_LOGS_DIR. +# +#GC_LOG_OPTS='-Xlog:gc*' # (java9) +#GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \ +# -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" +{% if solr_set_gc_options %} GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \ --XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" + -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" +{% endif %} # These GC settings have shown to work well for a number of common Solr workloads -GC_TUNE="-XX:NewRatio=3 \ --XX:SurvivorRatio=4 \ --XX:TargetSurvivorRatio=90 \ --XX:MaxTenuringThreshold=8 \ --XX:+UseConcMarkSweepGC \ --XX:+UseParNewGC \ --XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \ --XX:+CMSScavengeBeforeRemark \ --XX:PretenureSizeThreshold=64m \ --XX:+UseCMSInitiatingOccupancyOnly \ --XX:CMSInitiatingOccupancyFraction=50 \ --XX:CMSMaxAbortablePrecleanTime=6000 \ --XX:+CMSParallelRemarkEnabled \ --XX:+ParallelRefProcEnabled" +GC_TUNE="{{ solr_gc_tune }}" # Set the ZooKeeper connection string if using an external ZooKeeper ensemble # e.g. host1:2181,host2:2181/chroot # Leave empty if not using SolrCloud -#ZK_HOST="" +{% if solr_zk_external_cluster %} +ZK_HOST="{{ solr_zk_hosts }}" +{% endif %} # Set the ZooKeeper client timeout (for SolrCloud mode) -#ZK_CLIENT_TIMEOUT="15000" +ZK_CLIENT_TIMEOUT="{{ solr_zk_timeout }}" # By default the start script uses "localhost"; override the hostname here # for production SolrCloud environments to control the hostname exposed to cluster state @@ -75,7 +80,9 @@ ENABLE_REMOTE_JMX_OPTS="false" # Set the thread stack size SOLR_OPTS="$SOLR_OPTS -Xss256k" -SOLR_MODE={{ solr_mode }} +{% if not solr_zk_external_cluster %} +export SOLR_MODE={{ solr_mode }} +{% endif %} # Anything you add to the SOLR_OPTS variable will be included in the java # start command line as-is, in ADDITION to other options. If you specify the @@ -86,30 +93,41 @@ SOLR_MODE={{ solr_mode }} # Location where the bin/solr script will save PID files for running instances # If not set, the script will create PID files in $SOLR_TIP/bin -#SOLR_PID_DIR= +SOLR_PID_DIR={{ solr_pid_dir }} # Path to a directory for Solr to store cores and their data. By default, Solr will use server/solr # If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml -#SOLR_HOME= +SOLR_HOME={{ solr_data_dir }} # Solr provides a default Log4J configuration properties file in server/resources # however, you may want to customize the log settings and file appender location # so you can point the script to use a different log4j.properties file #LOG4J_PROPS=/var/solr/log4j.properties +# Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO +# This is an alternative to changing the rootLogger in log4j.properties +SOLR_LOG_LEVEL={{ solr_log_level }} + # Location where Solr should write logs to; should agree with the file appender # settings in server/resources/log4j.properties -#SOLR_LOGS_DIR= +SOLR_LOGS_DIR={{ solr_log_dir }} + +# Enables log rotation, cleanup, and archiving during start. Setting SOLR_LOG_PRESTART_ROTATION=false will skip start +# time rotation of logs, and the archiving of the last GC and console log files. It does not affect Log4j configuration. +# This pre-startup rotation may need to be disabled depending how much you customize the default logging setup. +#SOLR_LOG_PRESTART_ROTATION=true # Sets the port Solr binds to, default is 8983 -#SOLR_PORT=8983 +SOLR_PORT={{ solr_http_port }} # Uncomment to set SSL-related system properties # Be sure to update the paths to the correct keystore for your environment -#SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks +#SOLR_SSL_KEY_STORE=/home/shalin/work/oss/shalin-lusolr/solr/server/etc/solr-ssl.keystore.jks #SOLR_SSL_KEY_STORE_PASSWORD=secret -#SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks +#SOLR_SSL_KEY_STORE_TYPE=JKS +#SOLR_SSL_TRUST_STORE=/home/shalin/work/oss/shalin-lusolr/solr/server/etc/solr-ssl.keystore.jks #SOLR_SSL_TRUST_STORE_PASSWORD=secret +#SOLR_SSL_TRUST_STORE_TYPE=JKS #SOLR_SSL_NEED_CLIENT_AUTH=false #SOLR_SSL_WANT_CLIENT_AUTH=false @@ -117,10 +135,21 @@ SOLR_MODE={{ solr_mode }} # otherwise keep them commented and the above values will automatically be set for HTTP clients #SOLR_SSL_CLIENT_KEY_STORE= #SOLR_SSL_CLIENT_KEY_STORE_PASSWORD= +#SOLR_SSL_CLIENT_KEY_STORE_TYPE= #SOLR_SSL_CLIENT_TRUST_STORE= #SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD= +#SOLR_SSL_CLIENT_TRUST_STORE_TYPE= # Settings for authentication -#SOLR_AUTHENTICATION_CLIENT_CONFIGURER= -#SOLR_AUTHENTICATION_OPTS= +# Please configure only one of SOLR_AUTHENTICATION_CLIENT_CONFIGURER or SOLR_AUTH_TYPE parameters +#SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthConfigurer" +#SOLR_AUTH_TYPE="basic" +#SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks" + +# Settings for ZK ACL +#SOLR_ZK_CREDS_AND_ACLS="-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider \ +# -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider \ +# -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD \ +# -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD" +#SOLR_OPTS="$SOLR_OPTS $SOLR_ZK_CREDS_AND_ACLS" diff --git a/solr/templates/solr.xml.j2 b/solr/templates/solr.xml.j2 new file mode 100644 index 00000000..73d00829 --- /dev/null +++ b/solr/templates/solr.xml.j2 @@ -0,0 +1,62 @@ + + + + + + + + + {% if not solr_multicore %} + {{ ansible_fqdn }} + {{ solr_http_port_1 }} + ${hostContext:solr} + {{ solr_zk_timeout }} + ${genericCoreNodeNames:true} + {% if solr_cloud_mode %} + {{ solr_zk_hosts }} + {% endif %} + {% endif %} + + + + ${socketTimeout:0} + ${connTimeout:0} + + + {% if solr_multicore %} + + {% for core in solr_cores %} + + + + + + {% endfor %} + + {% endif %} + + + diff --git a/solr/templates/zoo.cfg.j2 b/solr/templates/zoo.cfg.j2 new file mode 100644 index 00000000..321156f7 --- /dev/null +++ b/solr/templates/zoo.cfg.j2 @@ -0,0 +1,17 @@ +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 + +# the directory where the snapshot is stored. +dataDir={{ solr_zookeeper_data_dir }} +# NOTE: Solr defaults the dataDir to /zoo_data + +# the port at which the clients will connect +clientPort={{ solr_zoo_port }} +# NOTE: Solr sets this based on zkRun / zkHost params + diff --git a/thredds/defaults/main.yml b/thredds/defaults/main.yml index 9ad026a1..b9715de8 100644 --- a/thredds/defaults/main.yml +++ b/thredds/defaults/main.yml @@ -10,7 +10,7 @@ thredds_file: '{{ thredds_name }}-{{ thredds_version }}.{{ thredds_extension }}' thredds_repository_url: 'https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases' thredds_download_directory: /srv thredds_data_content_dir: /data/thredds -thredds_public_netcdf_dir: '{{ thredds_data_content_dir }}/public/netcdf/' +thredds_public_netcdf_dir: '{{ thredds_data_content_dir }}/thredds/public/netcdf/' thredds_server_name: 'Thredds instance' thredds_logo_url: '/thredds/threddsIcon.gif' thredds_logo_alt_text: '{{ thredds_server_name }}' diff --git a/thredds/tasks/main.yml b/thredds/tasks/main.yml index a94063a7..d65f0214 100644 --- a/thredds/tasks/main.yml +++ b/thredds/tasks/main.yml @@ -22,6 +22,8 @@ with_items: - threddsConfig.xml - catalog.xml + notify: tomcat instances restart + tags: [ 'thredds', 'tomcat', 'thredds_conf' ] - name: Get the thredds war file maven_artifact: artifact_id={{ thredds_name }} version={{ thredds_version | default(omit) }} group_id={{ thredds_group_id }} extension={{ thredds_extension | default('war') }} repository_url={{ thredds_repository_url }} dest={{ thredds_download_directory }}/{{ thredds_file }}