Merge pull request 'ELK: add tasks and templates that manage kibana, and nginx in front of it.' (#157) from adellam/ansible-roles:master into master

library/roles/ELK/elasticHQ/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: '../../library/roles/nginx', when: elastic_hq_use_nginx_proxy }
+  - { role: '../../../library/roles/nginx', when: elastic_hq_use_nginx_proxy | bool }

library/roles/ELK/elasticsearch/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+elasticsearch_install: True
 elasticsearch_repo_key: https://packages.elastic.co/GPG-KEY-elasticsearch
 elasticsearch_repo_version: '{{ elasticsearch_major_version }}.x'
 #elasticsearch_major_version: 2
@@ -11,10 +12,28 @@ elasticsearch_repo: 'deb https://artifacts.elastic.co/packages/{{ elasticsearch_
 elasticsearch_packages:
   - elasticsearch
 
+elasticsearch_kibana_install: False
+elasticsearch_kibana_enabled: True
+elasticsearch_kibana_proxy: False
+elasticsearch_kibana_nginx_proxy: True
+elasticsearch_kibana_packages:
+  - kibana
+
+elasticsearch_kibana_http_port: 5601
+elasticsearch_kibana_bind_ip: 127.0.0.1
+elasticsearch_kibana_serverpath: ''
+elasticsearch_kibana_servername: '{{ ansible_fqdn }}'
+elasticsearch_kibana_elasticsearch_url: 'http://localhost:9200'
+elasticsearch_kibana_preserve_host: 'false'
+elasticsearch_kibana_ssl_enabled: False
+elasticsearch_kibana_rundir: /run/kibana
+
 elasticsearch_package_state: 'present'
 elasticsearch_cluster_name: 'Elasticsearch Cluster'
 elasticsearch_enabled: True
 elasticsearch_http_port: 9200
+elasticsearch_transport_min_port: 9300
+elasticsearch_transport_max_port: 9400
 elasticsearch_data_dir: /var/lib/elasticsearch
 elasticsearch_log_dir: /var/log/elasticsearch
 elasticsearch_bind_ip: 0.0.0.0

library/roles/ELK/elasticsearch/handlers/main.yml

@@ -1,5 +1,11 @@
 ---
 - name: Restart elasticsearch
   service: name=elasticsearch state=restarted enabled=yes
-  when: elasticsearch_enabled
+  when: elasticsearch_enabled | bool
+  ignore_errors: True
+
+- name: Restart kibana
+  service: name=kibana state=restarted enabled=yes
+  when: elasticsearch_kibana_enabled | bool
+
 

library/roles/ELK/elasticsearch/meta/main.yml

@@ -1,3 +1,4 @@
 ---
 dependencies:
   - { role: '../../../library/roles/openjdk' }
+  - { role: '../../../library/roles/nginx', when: elasticsearch_kibana_nginx_proxy | bool }

library/roles/ELK/elasticsearch/tasks/elasticsearch.yml

@@ -1,25 +1,23 @@
 ---
-- block:
-    - name: Install the elasticsearch repo key
-      apt_key: url={{ elasticsearch_repo_key }} state=present
-
-    - name: Install the elasticsearch deb repository
-      apt_repository: repo='{{ elasticsearch_repo }}' state=present update_cache=yes
-
+- name: Elasticsearch installation
+  block:
     - name: Install the elasticsearch deb packages
       apt: name='{{ elasticsearch_packages }}' state={{ elasticsearch_package_state }} update_cache=yes cache_valid_time=1800
   
     - name: Install the elasticsearch startup default
       template: src=elasticsearch-default.j2 dest=/etc/default/elasticsearch owner=root group=elasticsearch mode=0640
+      register: elasticsearch_default
       notify: Restart elasticsearch
 
     - name: Install the elasticsearch JVM options
       template: src=jvm.options.j2 dest=/etc/elasticsearch/jvm.options owner=root group=elasticsearch mode=0640
+      register: elasticsearch_jvm_opts
       notify: Restart elasticsearch
       tags: [ 'ELK', 'elasticsearch', 'elk', 'elasticsearch_conf' ]
 
     - name: Install the elasticsearch configuration
       template: src=elasticsearch.yml.j2 dest=/etc/elasticsearch/elasticsearch.yml owner=root group=elasticsearch mode=0640
+      register: elasticsearch_configuration
       notify: Restart elasticsearch
       tags: [ 'ELK', 'elasticsearch', 'elk', 'elasticsearch_conf' ]
 

library/roles/ELK/elasticsearch/tasks/elasticsearch_plugins.yml

@@ -0,0 +1,16 @@
+---
+- block:
+    - name: Manage a list of elasticsearch plugins
+      elasticsearch_plugin: name="{{ item.name }}" state={{ item.state }} plugin_bin={{ elasticsearch_plugin_bin }} url={{ item.url|default(omit) }} version={{ item.version|default(omit) }}
+      with_items: '{{ elasticsearch_plugins | default ([]) }}'
+
+    - name: Install the elasticsearch HQ plugin
+      elasticsearch_plugin: name="{{ item.name }}" state={{ item.state }} plugin_bin={{ elasticsearch_plugin_bin }} url={{ item.url|default(omit) }} version={{ item.version|default(omit) }}
+      with_items: '{{ elasticsearch_hq_plugin | default ([]) }}'
+      when:
+        - elasticsearch_major_version <= 2
+        - elasticsearch_hq_install
+
+  when: elasticsearch_plugins is defined
+  tags: [ 'elasticsearch', 'es_plugins' ]
+    

library/roles/ELK/elasticsearch/tasks/elasticsearch_service.yml

@@ -0,0 +1,11 @@
+---
+- name: Ensure that elasticsearch is enabled and running
+  service: name=elasticsearch state=started enabled=yes
+  when: elasticsearch_enabled | bool
+  tags: [ 'ELK', 'elasticsearch', 'elk' ]
+  
+- name: Ensure that elasticsearch is disabled and stopped
+  service: name=elasticsearch state=stopped enabled=no
+  when: not elasticsearch_enabled | bool
+  tags: [ 'ELK', 'elasticsearch', 'elk' ]
+  

library/roles/ELK/elasticsearch/tasks/elk_repo.yml

@@ -0,0 +1,10 @@
+---
+- name: ELK repository
+  block:
+    - name: Install the elasticsearch repo key
+      apt_key: url={{ elasticsearch_repo_key }} state=present
+
+    - name: Install the elasticsearch deb repository
+      apt_repository: repo='{{ elasticsearch_repo }}' state=present update_cache=yes
+
+  tags: [ 'ELK', 'elasticsearch', 'elk' ]

library/roles/ELK/elasticsearch/tasks/kibana.yml

@@ -0,0 +1,50 @@
+---
+- name: Kibana x509 certificate management
+  block:
+    - name: Create the acme hooks directory if it does not yet exist
+      file: dest={{ letsencrypt_acme_sh_services_scripts_dir }} state=directory owner=root group=root
+
+    - name: Create the kibana pki subdir
+      file: dest={{ pki_dir }}/kibana state=directory owner=root group=kibana mode=0750
+
+    - name: Check if the global certificate private key exists
+      stat: path={{ letsencrypt_acme_certs_dir }}/privkey
+      register: kibana_privkey
+
+    - name: Check if the kibana certificate private key exists under the pki directory
+      stat: path={{ pki_dir }}/kibana/privkey
+      register: kibana_pki_privkey
+
+    - name: Copy the private key into the expected place if it is not already there
+      copy: src={{ letsencrypt_acme_certs_dir }}/privkey dest={{ pki_dir }}/kibana/privkey remote_src=yes owner=root group=kibana mode=0440
+      when:
+        - kibana_privkey.stat.exists
+        - not kibana_pki_privkey.stat.exists
+
+    - name: Install the kibana hook for letsencrypt
+      template: src=kibana-letsencrypt-hook.sh.j2 dest=/usr/lib/acme/hooks/kibana owner=root group=root mode=0550
+
+  when:
+    - elasticsearch_kibana_ssl_enabled | bool
+    - letsencrypt_acme_install is defined and letsencrypt_acme_install | bool
+  tags: [ 'ELK', 'elasticsearch', 'elk', 'kibana' ]
+
+- name: Kibana installation
+  block:
+    - name: Install the Kibana packages
+      apt: name='{{ elasticsearch_kibana_packages }}' state={{ elasticsearch_package_state }} update_cache=yes cache_valid_time=1800
+
+    - name: Install the kibana systemd configuration to manage the rundir directory
+      template: src=kibana_rundir.conf.j2 dest=/usr/lib/tmpfiles.d/kibana.conf owner=root group=root mode=0644
+      register: reconfigure_systemd
+
+    - name: Reload the systemd configuration
+      systemd: daemon_reload=yes
+
+    - name: Install the Kibana configuration
+      template: src=kibana.yml.j2 dest=/etc/kibana/kibana.yml owner=root group=kibana mode=0640
+      register: kibana_configuration
+      notify: Restart kibana
+      tags: [ 'ELK', 'elasticsearch', 'elk', 'kibana', 'kibana_conf' ]
+  
+  tags: [ 'ELK', 'elasticsearch', 'elk', 'kibana' ]

library/roles/ELK/elasticsearch/tasks/kibana_service.yml

@@ -0,0 +1,11 @@
+---
+- name: Ensure that kibana is enabled and running
+  service: name=kibana state=started enabled=yes
+  when: elasticsearch_kibana_enabled | bool
+  tags: [ 'ELK', 'elasticsearch', 'elk', 'kibana' ]
+  
+- name: Ensure that kibana is disabled and stopped
+  service: name=kibana state=stopped enabled=no
+  when: not elasticsearch_kibana_enabled | bool
+  tags: [ 'ELK', 'elasticsearch', 'elk', 'kibana' ]
+  

library/roles/ELK/elasticsearch/tasks/main.yml

@@ -1,14 +1,13 @@
 ---
+- import_tasks: elk_repo.yml
 - import_tasks: elasticsearch.yml
-- import_tasks: plugins.yml
+  when: elasticsearch_install | bool
+- import_tasks: elasticsearch_plugins.yml
+  when: elasticsearch_install | bool
+- import_tasks: elasticsearch_service.yml
+  when: elasticsearch_install | bool
+- import_tasks: kibana.yml
+  when: elasticsearch_kibana_install | bool
+- import_tasks: kibana_service.yml
+  when: elasticsearch_kibana_install | bool
 
-- name: Ensure that elasticsearch is enabled and running
-  service: name=elasticsearch state=started enabled=yes
-  when: elasticsearch_enabled
-  tags: [ 'ELK', 'elasticsearch', 'elk' ]
-  
-- name: Ensure that elasticsearch is disabled and stopped
-  service: name=elasticsearch state=stopped enabled=no
-  when: not elasticsearch_enabled
-  tags: [ 'ELK', 'elasticsearch', 'elk' ]
-  

library/roles/ELK/elasticsearch/templates/elasticsearch.yml.j2

@@ -22,6 +22,13 @@ cluster.name: {{ elasticsearch_cluster_name }}
 #
 node.name: {{ ansible_fqdn }}
 
+{% if elasticsearch_kibana_proxy %}
+# This node is bein used by kibana as proxy to a cluster
+node.master: false
+node.data: false
+node.ingest: false
+{% endif %}
+
 {% if elasticsearch_major_version >= 7 %}
 cluster.initial_master_nodes:
 {% for n in elasticsearch_bootstrap_known_masters %}
@@ -60,11 +67,21 @@ bootstrap.mlockall: true
 #
 # Set the bind address to a specific IP (IPv4 or IPv6):
 #
+{% if elasticsearch_kibana_proxy %}
+network.host: localhost
+{% else %}
 network.host: {{ elasticsearch_bind_ip }}
+{% endif %}
 #
 # Set a custom port for HTTP:
 #
 http.port: {{ elasticsearch_http_port }}
+
+# by default transport.host refers to network.host
+transport.host: {{ elasticsearch_bind_ip }}
+{% if elasticsearch_major_version >= 6 %}
+transport.tcp.port: {{ elasticsearch_transport_min_port }} - {{ elasticsearch_transport_max_port }}
+{% endif %}
 #
 # For more information, see the documentation at:
 # <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>

library/roles/ELK/elasticsearch/templates/kibana-letsencrypt-hook.sh.j2

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+H_NAME=$( hostname -f )
+LE_SERVICES_SCRIPT_DIR=/usr/lib/acme/hooks
+LE_CERTS_DIR="/var/lib/acme/live/$H_NAME"
+LE_LOG_DIR=/var/log/letsencrypt
+KIBANA_CERTDIR=/etc/pki/kibana
+KIBANA_KEYFILE="$KIBANA_CERTDIR/privkey"
+DATE=$( date )
+
+[ ! -d $KIBANA_CERTDIR ] && mkdir -p $KIBANA_CERTDIR
+[ ! -d $LE_LOG_DIR ] && mkdir $LE_LOG_DIR
+echo "$DATE" >> $LE_LOG_DIR/kibana.log
+
+{% if letsencrypt_acme_install %}
+LE_ENV_FILE=/etc/default/letsencrypt
+{% endif %}
+{% if letsencrypt_acme_sh_install %}
+LE_ENV_FILE=/etc/default/acme_sh_request_env
+{% endif %}
+if [ -f "$LE_ENV_FILE" ] ; then
+    . "$LE_ENV_FILE"
+else
+    echo "No letsencrypt default file" >> $LE_LOG_DIR/kibana.log
+fi
+
+echo "Building the new certificate file" >> $LE_LOG_DIR/kibana.log
+cp -f ${LE_CERTS_DIR}/privkey ${KIBANA_KEYFILE}
+chmod 440 ${KIBANA_KEYFILE}
+chgrp kibana ${KIBANA_KEYFILE}
+
+echo "Reload the kibana service" >> $LE_LOG_DIR/kibana.log
+systemctl restart kibana >> $LE_LOG_DIR/kibana.log 2>&1
+
+
+echo "Done." >> $LE_LOG_DIR/kibana.log
+
+exit 0

library/roles/ELK/elasticsearch/templates/kibana.yml.j2

@@ -0,0 +1,108 @@
+# Kibana is served by a back end server. This setting specifies the port to use.
+server.port: {{ elasticsearch_kibana_http_port }}
+
+# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
+# The default is 'localhost', which usually means remote machines will not be able to connect.
+# To allow connections from remote users, set this parameter to a non-loopback address.
+server.host: "{{ elasticsearch_kibana_bind_ip }}"
+
+# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
+# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
+# to Kibana. This setting cannot end in a slash.
+server.basePath: "{{ elasticsearch_kibana_serverpath }}"
+
+# The maximum payload size in bytes for incoming server requests.
+#server.maxPayloadBytes: 1048576
+
+# The Kibana server's name.  This is used for display purposes.
+server.name: "{{ elasticsearch_kibana_servername }}"
+
+# The URL of the Elasticsearch instance to use for all your queries.
+elasticsearch.url: "{{ elasticsearch_kibana_elasticsearch_url }}"
+
+# When this setting's value is true Kibana uses the hostname specified in the server.host
+# setting. When the value of this setting is false, Kibana uses the hostname of the host
+# that connects to this Kibana instance.
+elasticsearch.preserveHost: {{ elasticsearch_kibana_preserve_host }}
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
+# dashboards. Kibana creates a new index if the index doesn't already exist.
+#kibana.index: ".kibana"
+
+# The default application to load.
+#kibana.defaultAppId: "discover"
+
+# If your Elasticsearch is protected with basic authentication, these settings provide
+# the username and password that the Kibana server uses to perform maintenance on the Kibana
+# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
+# is proxied through the Kibana server.
+#elasticsearch.username: "user"
+#elasticsearch.password: "pass"
+
+{% if letsencrypt_acme_install is defined and letsencrypt_acme_install %}
+{% if elasticsearch_kibana_ssl_enabled %}
+# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
+# These settings enable SSL for outgoing requests from the Kibana server to the browser.
+server.ssl.enabled: true
+server.ssl.certificate: {{ letsencrypt_acme_certs_dir }}/fullchain
+server.ssl.key: {{ pki_dir }}/kibana/privkey
+{% endif %}
+{% endif %}
+
+# Optional settings that provide the paths to the PEM-format SSL certificate and key files.
+# These files validate that your Elasticsearch backend uses the same key files.
+#elasticsearch.ssl.certificate: /path/to/your/client.crt
+#elasticsearch.ssl.key: /path/to/your/client.key
+
+# Optional setting that enables you to specify a path to the PEM file for the certificate
+# authority for your Elasticsearch instance.
+#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
+
+# To disregard the validity of SSL certificates, change this setting's value to 'none'.
+#elasticsearch.ssl.verificationMode: full
+
+# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
+# the elasticsearch.requestTimeout setting.
+#elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
+# must be a positive integer.
+#elasticsearch.requestTimeout: 30000
+
+# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
+# headers, set this value to [] (an empty list).
+#elasticsearch.requestHeadersWhitelist: [ authorization ]
+
+# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
+# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
+#elasticsearch.customHeaders: {}
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
+#elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
+#elasticsearch.startupTimeout: 5000
+
+# Specifies the path where Kibana creates the process ID file.
+pid.file: {{ elasticsearch_kibana_rundir }}/kibana.pid
+
+# Enables you specify a file where Kibana stores log output.
+#logging.dest: stdout
+
+# Set the value of this setting to true to suppress all logging output.
+#logging.silent: false
+
+# Set the value of this setting to true to suppress all logging output other than error messages.
+#logging.quiet: false
+
+# Set the value of this setting to true to log all events, including system usage information
+# and all requests.
+#logging.verbose: false
+
+# Set the interval in milliseconds to sample system and process performance
+# metrics. Minimum is 100ms. Defaults to 5000.
+#ops.interval: 5000
+
+# The default locale. This locale can be used in certain circumstances to substitute any missing
+# translations.
+#i18n.defaultLocale: "en"

library/roles/ELK/elasticsearch/templates/kibana_rundir.conf.j2

@@ -0,0 +1 @@
+d {{ elasticsearch_kibana_rundir }} 0775 kibana kibana