From eab1f9899c5219e4de4b4464361217f34d4d08f2 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Mon, 17 Dec 2018 18:21:46 +0100 Subject: [PATCH] Vocbench: add the php service and install the vocbench api application. See https://support.d4science.org/issues/12495 --- ELK/kibana/defaults/main.yml | 39 +++++++ ELK/kibana/handlers/main.yml | 5 + ELK/kibana/tasks/elasticsearch.yml | 28 +++++ ELK/kibana/tasks/main.yml | 14 +++ ELK/kibana/tasks/plugins.yml | 14 +++ ELK/kibana/templates/elasticsearch-default.j2 | 74 +++++++++++++ ELK/kibana/templates/elasticsearch.yml.j2 | 101 ++++++++++++++++++ ELK/kibana/templates/jvm.options.j2 | 97 +++++++++++++++++ 8 files changed, 372 insertions(+) create mode 100644 ELK/kibana/defaults/main.yml create mode 100644 ELK/kibana/handlers/main.yml create mode 100644 ELK/kibana/tasks/elasticsearch.yml create mode 100644 ELK/kibana/tasks/main.yml create mode 100644 ELK/kibana/tasks/plugins.yml create mode 100644 ELK/kibana/templates/elasticsearch-default.j2 create mode 100644 ELK/kibana/templates/elasticsearch.yml.j2 create mode 100644 ELK/kibana/templates/jvm.options.j2 diff --git a/ELK/kibana/defaults/main.yml b/ELK/kibana/defaults/main.yml new file mode 100644 index 00000000..4dade064 --- /dev/null +++ b/ELK/kibana/defaults/main.yml @@ -0,0 +1,39 @@ +--- +elasticsearch_repo_key: https://packages.elastic.co/GPG-KEY-elasticsearch +elasticsearch_major_version: 2 +elasticsearch_repo_version: '{{ elasticsearch_major_version }}.x' +elasticsearch_repo: 'deb http://packages.elastic.co/elasticsearch/{{ elasticsearch_repo_version }}/debian stable main' +#elasticsearch_major_version: 5 +#elasticsearch_repo: 'deb https://artifacts.elastic.co/packages/{{ elasticsearch_repo_version }}/apt stable main' +elasticsearch_packages: + - elasticsearch + +elasticsearch_package_state: 'present' +elasticsearch_cluster_name: 'Elasticsearch Cluster' +elasticsearch_enabled: True +elasticsearch_http_port: 9200 +elasticsearch_data_dir: /var/lib/elasticsearch +elasticsearch_log_dir: /var/log/elasticsearch +elasticsearch_bind_ip: 0.0.0.0 +elasticsearch_discovery_host_list: '["127.0.0.1", "[::1]"]' +elasticsearch_define_majority_of_nodes: True +elasticsearch_majority_of_nodes: 1 +elasticsearch_real_cluster: False +elasticsearch_recover_after_nodes: 3 +elasticsearch_max_local_storage_nodes: 1 +elasticsearch_destructive_requires_name: 'true' +elasticsearch_define_heap_size: False +elasticsearch_heap_size: 2g +elasticsearch_additional_java_opts: '-server -Djava.awt.headless=true' +elasticsearch_max_open_files: 65536 + +# bin/plugin for 2.x, bin/elasticsearch-plugin for 5.x +#elasticsearch_plugin_bin: /usr/share/elasticsearch/bin/elasticsearch-plugin +elasticsearch_plugin_bin: /usr/share/elasticsearch/bin/plugin +#elasticsearch_plugins: +# - { name: 'royrusso/elasticsearch-HQ', state: 'present' } + +# elasticsearch 5 not supported yet, so set it to False when installing 5.x +elasticsearch_hq_install: True +elasticsearch_hq_plugin: + - { name: 'royrusso/elasticsearch-HQ', state: 'present' } diff --git a/ELK/kibana/handlers/main.yml b/ELK/kibana/handlers/main.yml new file mode 100644 index 00000000..834bce5c --- /dev/null +++ b/ELK/kibana/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart elasticsearch + service: name=elasticsearch state=restarted enabled=yes + when: elasticsearch_enabled + diff --git a/ELK/kibana/tasks/elasticsearch.yml b/ELK/kibana/tasks/elasticsearch.yml new file mode 100644 index 00000000..ae9d0742 --- /dev/null +++ b/ELK/kibana/tasks/elasticsearch.yml @@ -0,0 +1,28 @@ +--- +- block: + - name: Install the elasticsearch repo key + apt_key: url={{ elasticsearch_repo_key }} state=present + + - name: Install the elasticsearch deb repository + apt_repository: repo='{{ elasticsearch_repo }}' state=present update_cache=yes + + - name: Install the elasticsearch deb packages + apt: name='{{ item }}' state={{ elasticsearch_package_state }} update_cache=yes cache_valid_time=1800 + with_items: '{{ elasticsearch_packages }}' + + - name: Install the elasticsearch startup default + template: src=elasticsearch-default.j2 dest=/etc/default/elasticsearch owner=root group=elasticsearch mode=0640 + notify: Restart elasticsearch + + - name: Install the elasticsearch JVM options + template: src=jvm.options.j2 dest=/etc/elasticsearch/jvm.options owner=root group=elasticsearch mode=0640 + notify: Restart elasticsearch + tags: [ 'ELK', 'elasticsearch', 'elk', 'elasticsearch_conf' ] + + - name: Install the elasticsearch configuration + template: src=elasticsearch.yml.j2 dest=/etc/elasticsearch/elasticsearch.yml owner=root group=elasticsearch mode=0640 + notify: Restart elasticsearch + tags: [ 'ELK', 'elasticsearch', 'elk', 'elasticsearch_conf' ] + + + tags: [ 'ELK', 'elasticsearch', 'elk' ] diff --git a/ELK/kibana/tasks/main.yml b/ELK/kibana/tasks/main.yml new file mode 100644 index 00000000..e4f63cdb --- /dev/null +++ b/ELK/kibana/tasks/main.yml @@ -0,0 +1,14 @@ +--- +- import_tasks: elasticsearch.yml +- import_tasks: plugins.yml + +- name: Ensure that elasticsearch is enabled and running + service: name=elasticsearch state=started enabled=yes + when: elasticsearch_enabled + tags: [ 'ELK', 'elasticsearch', 'elk' ] + +- name: Ensure that elasticsearch is disabled and stopped + service: name=elasticsearch state=stopped enabled=no + when: not elasticsearch_enabled + tags: [ 'ELK', 'elasticsearch', 'elk' ] + diff --git a/ELK/kibana/tasks/plugins.yml b/ELK/kibana/tasks/plugins.yml new file mode 100644 index 00000000..b274db5d --- /dev/null +++ b/ELK/kibana/tasks/plugins.yml @@ -0,0 +1,14 @@ +--- +- block: + - name: Manage a list of elasticsearch plugins + elasticsearch_plugin: name="{{ item.name }}" state={{ item.state }} plugin_bin={{ elasticsearch_plugin_bin }} url={{ item.url|default(omit) }} version={{ item.version|default(omit) }} + with_items: '{{ elasticsearch_plugins | default ([]) }}' + + - name: Install the elasticsearch HQ plugin + elasticsearch_plugin: name="{{ item.name }}" state={{ item.state }} plugin_bin={{ elasticsearch_plugin_bin }} url={{ item.url|default(omit) }} version={{ item.version|default(omit) }} + with_items: '{{ elasticsearch_hq_plugin | default ([]) }}' + when: elasticsearch_hq_install + + when: elasticsearch_plugins is defined + tags: [ 'elasticsearch', 'es_plugins' ] + diff --git a/ELK/kibana/templates/elasticsearch-default.j2 b/ELK/kibana/templates/elasticsearch-default.j2 new file mode 100644 index 00000000..8f1ff171 --- /dev/null +++ b/ELK/kibana/templates/elasticsearch-default.j2 @@ -0,0 +1,74 @@ +################################ +# Elasticsearch +################################ + +# Elasticsearch home directory +#ES_HOME=/usr/share/elasticsearch + +# Elasticsearch configuration directory +#CONF_DIR=/etc/elasticsearch + +# Elasticsearch data directory +DATA_DIR={{ elasticsearch_data_dir }} + +# Elasticsearch logs directory +LOG_DIR={{ elasticsearch_log_dir }} + +# Elasticsearch PID directory +#PID_DIR=/var/run/elasticsearch + +{% if elasticsearch_major_version <= 2 %} +# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g +ES_HEAP_SIZE={{ elasticsearch_heap_size }} +# Heap new generation +#ES_HEAP_NEWSIZE= + +# Maximum direct memory +#ES_DIRECT_SIZE= + +# Additional Java OPTS +ES_JAVA_OPTS="{{ elasticsearch_additional_java_opts }}" +{% endif %} +# Configure restart on package upgrade (true, every other setting will lead to not restarting) +#ES_RESTART_ON_UPGRADE=true + +# Path to the GC log file +#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log + +################################ +# Elasticsearch service +################################ + +# SysV init.d +# +# When executing the init script, this user will be used to run the elasticsearch service. +# The default value is 'elasticsearch' and is declared in the init.d file. +# Note that this setting is only used by the init script. If changed, make sure that +# the configured user can read and write into the data, work, plugins and log directories. +# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service +ES_USER=elasticsearch +ES_GROUP=elasticsearch + +# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process +ES_STARTUP_SLEEP_TIME=5 + +################################ +# System properties +################################ + +# Specifies the maximum file descriptor number that can be opened by this process +# When using Systemd, this setting is ignored and the LimitNOFILE defined in +# /usr/lib/systemd/system/elasticsearch.service takes precedence +MAX_OPEN_FILES={{ elasticsearch_max_open_files }} + +# The maximum number of bytes of memory that may be locked into RAM +# Set to "unlimited" if you use the 'bootstrap.mlockall: true' option +# in elasticsearch.yml (ES_HEAP_SIZE must also be set). +# When using Systemd, the LimitMEMLOCK property must be set +# in /usr/lib/systemd/system/elasticsearch.service +MAX_LOCKED_MEMORY=unlimited + +# Maximum number of VMA (Virtual Memory Areas) a process can own +# When using Systemd, this setting is ignored and the 'vm.max_map_count' +# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf +#MAX_MAP_COUNT=262144 diff --git a/ELK/kibana/templates/elasticsearch.yml.j2 b/ELK/kibana/templates/elasticsearch.yml.j2 new file mode 100644 index 00000000..0858b9ea --- /dev/null +++ b/ELK/kibana/templates/elasticsearch.yml.j2 @@ -0,0 +1,101 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please see the documentation for further information on configuration options: +# +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: {{ elasticsearch_cluster_name }} +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: {{ ansible_fqdn }} +# +# Add custom attributes to the node: +# +# node.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: {{ elasticsearch_data_dir }} +# +# Path to log files: +# +path.logs: {{ elasticsearch_log_dir }} +# +{% if elasticsearch_major_version <= 2 %} +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +bootstrap.mlockall: true +# +# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory +# available on the system and that the owner of the process is allowed to use this limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +{% endif %} +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +network.host: {{ elasticsearch_bind_ip }} +# +# Set a custom port for HTTP: +# +http.port: {{ elasticsearch_http_port }} +# +# For more information, see the documentation at: +# +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when new node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +discovery.zen.ping.unicast.hosts: {{ elasticsearch_discovery_host_list }} +# +{% if elasticsearch_define_majority_of_nodes %} +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +# +discovery.zen.minimum_master_nodes: {{ elasticsearch_majority_of_nodes }} +# +# For more information, see the documentation at: +# +# +{% if elasticsearch_real_cluster %} +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +gateway.recover_after_nodes: {{ elasticsearch_recover_after_nodes }} +# +# For more information, see the documentation at: +# +# +{% endif %} +{% endif %} +# ---------------------------------- Various ----------------------------------- +# +# Disable starting multiple nodes on a single system: +# +node.max_local_storage_nodes: {{ elasticsearch_max_local_storage_nodes }} +# +# Require explicit names when deleting indices: +# +action.destructive_requires_name: {{ elasticsearch_destructive_requires_name }} + diff --git a/ELK/kibana/templates/jvm.options.j2 b/ELK/kibana/templates/jvm.options.j2 new file mode 100644 index 00000000..508946d9 --- /dev/null +++ b/ELK/kibana/templates/jvm.options.j2 @@ -0,0 +1,97 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{ elasticsearch_heap_size }} +-Xmx{{ elasticsearch_heap_size }} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# disable calls to System#gc +-XX:+DisableExplicitGC + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM +-server + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# flag to explicitly tell Netty to not use unsafe +-Dio.netty.noUnsafe=true + +-Dlog4j2.disable.jmx=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps + +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true