library/roles/postgresql: tons of fixes to the postgresql/pgpool tasks. Now the pgpool HA is handled.

postgresql/defaults/main.yml

@@ -12,7 +12,7 @@ psql_pgpool_pkg_state: installed
 #
 # See the features matrix here: http://www.postgresql.org/about/featurematrix/
 #
-psql_version: 9.4
+psql_version: 9.5
 psql_db_host: localhost
 psql_db_port: 5432
 psql_db_size_w: 150000000
@@ -54,7 +54,7 @@ postgresql_pkgs:
 
 psql_ansible_needed_pkgs:
   - python-psycopg2
-#  - libpq-dev
+
 psql_db_name: db_name
 psql_db_user: db_user
 psql_db_pwd: "We cannot save the password into the repository. Use another variable and change pgpass.j2 accordingly. Encrypt the file that contains the variable with ansible-vault"
@@ -75,6 +75,7 @@ postgresql_pgpool_pkgs:
 # pgpool-II
 pgpool_pkgs:
   - pgpool2
+  - iputils-arping
 
 pgpool_enabled: True
 pgpool_listen_addresses: 'localhost'
@@ -126,6 +127,20 @@ pgpool_memqcache_memcached_host: localhost
 pgpool_memqcache_memcached_port: 11211
 pgpool_memqcache_expire: 0
 pgpool_memqcache_auto_cache_invalidation: 'on'
+# HA and watchdog
+pgpool_use_watchdog: 'off'
+pgpool_wd_trusted_servers: 'localhost,localhost'
+pgpool_wd_port: 9000
+pgpool_wd_priority: 1
+# Warning: setting pgpool_wd_heartbeat_mode to False enables
+# the 'query mode' that is untested and not working without manual intervention
+pgpool_wd_heartbeat_mode: True
+pgpool_wd_heartbeat_port: 9694
+pgpool_wd_heartbeat_keepalive_int: 3
+pgpool_wd_heartbeat_deadtime: 30
+pgpool_wd_heartbeat_dest0: 'localhost'
+pgpool_wd_heartbeat_dest0_port: '{{ pgpool_wd_heartbeat_port }}'
+#pgpool_wd_authkey: 'set it inside a vault file'
 
 # SSL as a special case
 pgpool_enable_ssl: False
@@ -134,6 +149,8 @@ pgpool_ssl_key: /etc/pki/pgpool2/pgpool2.key
 pgpool_ssl_cert: '/var/lib/acme/live/{{ ansible_fqdn }}/cert'
 pgpool_ssl_ca: '/var/lib/acme/live/{{ ansible_fqdn }}/chain'
 pgpool_ssl_ca_dir: /etc/ssl/certs
+pgpool_virtual_ip: 127.0.0.1
+pgpool_virtual_netmask: 24
 
 # WAL files archiving is mandatory for pgpool recovery
 psql_wal_files_archiving_enabled: '{{ psql_pgpool_install }}'

postgresql/files/arping_script

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Exec /usr/bin/arping as root via sudo
+
+RETVAL=
+CMD=/usr/bin/arping
+
+sudo $CMD $@
+RETVAL=$?
+
+exit $RETVAL
+

postgresql/files/ip_script

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Exec /sbin/ip as root via sudo
+
+RETVAL=
+CMD=/sbin/ip
+
+sudo $CMD $@
+RETVAL=$?
+
+exit $RETVAL
+

postgresql/tasks/main.yml

@@ -5,8 +5,6 @@
   when: psql_postgresql_install
 - include: postgis.yml
   when: postgres_install_gis_extensions
-- include: postgres_pgpool.yml
-  when: psql_pgpool_install
 - include: postgresql-config.yml
   when: psql_postgresql_install
 - include: postgresql-ssl-config.yml
@@ -19,6 +17,10 @@
   when:
     - psql_postgresql_install
     - psql_db_data is defined
+- include: postgresql-service-status.yml
+  when: psql_postgresql_install
+- include: postgres_pgpool.yml
+  when: psql_pgpool_install
 - include: manage_pg_db.yml
   when:
     - psql_postgresql_install

postgresql/tasks/manage_pg_db.yml

@@ -12,7 +12,7 @@
   become_user: postgres
   postgresql_db: db={{ item.name }} port={{ psql_db_port }} encoding={{ item.encoding }} owner={{ item.user }} template=template0 state={{ item.state | default('present') }}
   with_items: '{{ psql_db_data | default(omit) }}'
-  when: item.managedb
+  when: item.managedb | default(True)
   tags: [ 'postgresql', 'postgres', 'pg_db' ]
 
 - name: Only set a db user password. Mostly for the postgresql user

postgresql/tasks/packages.yml

@@ -10,13 +10,3 @@
   with_items: '{{ psql_ansible_needed_pkgs }}'
   tags: [ 'postgresql', 'postgres' ]
 
-- name: Ensure that the postgresql server is started
-  service: name=postgresql state=started enabled=yes
-  when: postgresql_enabled
-  tags: [ 'postgresql', 'postgres' ]
-
-- name: Ensure that the postgresql server is stopped and disabled
-  service: name=postgresql state=stopped enabled=no
-  when: not postgresql_enabled
-  tags: [ 'postgresql', 'postgres' ]
-

postgresql/tasks/pgpool-ii.yml

@@ -1,18 +1,16 @@
 ---
+- block:
     - name: Install the pgpool package
       apt: name={{ item }} state={{ psql_pgpool_pkg_state }}
       with_items: '{{ pgpool_pkgs }}'
-  tags: [ 'postgresql', 'postgres', 'pgpool' ]
 
     - name: Configure pcp
       #template: src=pcp.conf.j2 dest=/etc/pgpool2/pcp.conf owner=root group=postgres mode=0640
       shell: pwd=`pg_md5 {{ pcp_pwd }}` ; echo "{{ pgpool_pcp_user }}:${pwd}" > /etc/pgpool2/pcp.conf ; chmod 640 /etc/pgpool2/pcp.conf; chown root:postgres /etc/pgpool2/pcp.conf 
-  tags: [ 'postgresql', 'postgres', 'pgpool', 'pcp_conf', 'pgpool_conf' ]
   
     - name: Install the pgpool configuration file
       template: src=pgpool.conf.j2 dest=/etc/pgpool2/pgpool.conf owner=root group=postgres mode=0640
       notify: Restart pgpool2
-  tags: [ 'postgresql', 'postgres', 'pgpool', 'pgpool_conf' ]
 
     - name: Give access to the remote postgresql clients
       lineinfile: name=/etc/pgpool2/pool_hba.conf regexp="^host {{ item.0.name }} {{ item.0.user }} {{ item.1 }}.*$" line="host {{ item.0.name }} {{ item.0.user }} {{ item.1 }} md5"
@@ -23,19 +21,60 @@
         - psql_db_data is defined
         - item.1 is defined
       notify: Reload pgpool2
-  tags: [ 'postgresql', 'postgres', 'pgpool', 'pgpool_conf' ]
+  
+    - name: Create the pki directory to store the pgpool key
+      file: dest=/etc/pki/pgpool2 state=directory owner=postgres group=postgres mode=0750
+      when: pgpool_enable_ssl
+
+    - name: Create a pgpool accessible ssl key file if it does not exist
+      copy: src=/var/lib/acme/live/{{ ansible_fqdn }}/privkey dest=/etc/pki/pgpool2/pgpool2.key owner=postgres group=postgres mode=0400 remote_src=True
+      when: pgpool_enable_ssl
 
     - name: Install the pool_passwd configuration file
       shell: cd /etc/pgpool2 ; pg_md5 -m -u {{ item.user }} {{ item.pwd }} ; chown root:postgres /etc/pgpool2/pool_passwd ; chmod 660 /etc/pgpool2/pool_passwd
       with_items: '{{ psql_db_data | default([]) }}'
-  tags: [ 'postgresql', 'postgres', 'pgpool', 'pgpool_conf' ]
+  
+    - name: Install the sudoers config that permits pgpool change the network configuration during a failover
+      template: src=pgpool-wd-sudoers.j2 dest=/etc/sudoers.d/pgpool-wd owner=root group=root mode=0440
+
+    - name: Install the ip script that manage the network configuration during a failover
+      copy: src={{ item }} dest=/sbin/{{ item }} owner=root group=root mode=0755
+      with_items:
+        - ip_script
+
+    - name: Install the arping scripts that manage the network configuration during a failover
+      copy: src={{ item }} dest=/usr/local/bin/{{ item }} owner=root group=root mode=0755
+      with_items:
+        - arping_script
 
     - name: Start and enable pgpool2
       service: name=pgpool2 state=started enabled=yes
-  when: pgpool_enabled
-  tags: [ 'postgresql', 'postgres', 'pgpool' ]
 
+  when: pgpool_enabled 
+  tags: [ 'postgresql', 'postgres', 'pgpool', 'pcp_conf', 'pgpool_conf' ]
+
+
+- block:
     - name: Stop and disable pgpool2
       service: name=pgpool2 state=stopped enabled=no
+
+    - name: Install the pgpool packages
+      apt: name={{ item }} state=absent
+      with_items: '{{ pgpool_pkgs }}'
+
+    - name: Remove the pgpool failover sudoers file
+      file: dest=/etc/sudoers.d/pgpool-wd state=absent
+
+    - name: Remove the pgpool configuration directory
+      file: dest=/etc/pgpool2 state=absent
+
+    - name: Remove the scripts that manage the network configuration during a failover
+      file: dest={{ item }} state=absent
+      with_items:
+        - /sbin/ip_script
+        - /usr/local/bin/arping_script
+
   when: not pgpool_enabled
   tags: [ 'postgresql', 'postgres', 'pgpool' ]
+
+

postgresql/tasks/postgresql-service-status.yml

@@ -0,0 +1,11 @@
+---
+- name: Ensure that the postgresql server is started
+  service: name=postgresql state=started enabled=yes
+  when: postgresql_enabled
+  tags: [ 'postgresql', 'postgres' ]
+
+- name: Ensure that the postgresql server is stopped and disabled
+  service: name=postgresql state=stopped enabled=no
+  when: not postgresql_enabled
+  tags: [ 'postgresql', 'postgres' ]
+

postgresql/tasks/postgresql-ssl-config.yml

@@ -1,8 +1,15 @@
 ---
+- block:
     - name: Setup ssl in the postgresql configuration
       action: configfile path=/etc/postgresql/{{ psql_version }}/main/postgresql.conf key={{ item.name }} value="'{{ item.value }}'"
       with_items: '{{ psql_conf_ssl_parameters }}'
-  when: psql_enable_ssl
       notify: Restart postgresql
-  tags: [ 'postgresql', 'postgres', 'pg_conf' ]
 
+    - name: Create the pki directory to store the postgresql key
+      file: dest=/etc/pki/postgresql state=directory owner=postgres group=postgres mode=0750
+
+    - name: Create a postgres accessible ssl key file if it does not exist
+      copy: src=/var/lib/acme/live/{{ ansible_fqdn }}/privkey dest=/etc/pki/postgresql/postgresql.key owner=postgres group=postgres mode=0400 remote_src=True
+
+  when: psql_enable_ssl
+  tags: [ 'postgresql', 'postgres', 'pg_conf' ]

postgresql/templates/pgpool-wd-sudoers.j2

@@ -0,0 +1,3 @@
+{{ pgpool_recovery_user }}  ALL=(ALL) NOPASSWD:   /bin/ip
+{{ pgpool_recovery_user }}  ALL=(ALL) NOPASSWD:   /usr/bin/arping
+

postgresql/templates/pgpool.conf.j2

@@ -437,13 +437,13 @@ client_idle_limit_in_recovery = {{ pgpool_client_idle_limit_in_recovery }}
 
 # - Enabling -
 
-use_watchdog = off
+use_watchdog = {{ pgpool_use_watchdog }}
                                     # Activates watchdog
                                     # (change requires restart)
 
 # -Connection to up stream servers -
 
-trusted_servers = ''
+trusted_servers = '{{ pgpool_wd_trusted_servers }}'
                                     # trusted server list which are used
                                     # to confirm network connection
                                     # (hostA,hostB,hostC,...)
@@ -454,36 +454,36 @@ ping_path = '/bin'
 
 # - Watchdog communication Settings -
 
-wd_hostname = ''
+wd_hostname = '{{ ansible_default_ipv4.address }}'
                                     # Host name or IP address of this watchdog
                                     # (change requires restart)
 wd_port = 9000
                                     # port number for watchdog service
                                     # (change requires restart)
-wd_authkey = ''
+wd_authkey = '{{ pgpool_wd_authkey }}'
                                     # Authentication key for watchdog communication
                                     # (change requires restart)
 
 # - Virtual IP control Setting -
 
-delegate_IP = ''
+delegate_IP = '{{ pgpool_virtual_ip }}'
                                     # delegate IP address
                                     # If this is empty, virtual IP never bring up. 
                                     # (change requires restart)
 ifconfig_path = '/sbin'
                                     # ifconfig command path
                                     # (change requires restart)
-if_up_cmd = 'ifconfig eth0:0 inet $_IP_$ netmask 255.255.255.0'
+if_up_cmd = 'ip_script addr add {{ pgpool_virtual_ip }}/{{ pgpool_virtual_netmask }} dev {{ ansible_default_ipv4.alias }}'
                                     # startup delegate IP command
                                     # (change requires restart)
-if_down_cmd = 'ifconfig eth0:0 down'
+if_down_cmd = 'ip_script addr del {{ pgpool_virtual_ip }}/{{ pgpool_virtual_netmask }} dev {{ ansible_default_ipv4.alias }}'
                                     # shutdown delegate IP command
                                     # (change requires restart)
 
-arping_path = '/usr/sbin'           # arping command path
+arping_path = '/usr/local/bin'           # arping command path
                                     # (change requires restart)
 
-arping_cmd = 'arping -U $_IP_$ -w 1'
+arping_cmd = 'arping_script -U $_IP_$ -w 1'
                                     # arping command
                                     # (change requires restart)
 
@@ -511,22 +511,25 @@ wd_interval = 10
                                     # lifecheck interval (sec) > 0
                                     # (change requires restart)
 
+wd_priority = {{ pgpool_wd_priority }}
+
+{% if pgpool_wd_heartbeat_mode %}
 # -- heartbeat mode --
 
-wd_heartbeat_port = 9694
+wd_heartbeat_port = {{ pgpool_wd_heartbeat_port }}
                                     # Port number for receiving heartbeat signal
                                     # (change requires restart)
-wd_heartbeat_keepalive = 2
+wd_heartbeat_keepalive = {{ pgpool_wd_heartbeat_keepalive_int }}
                                     # Interval time of sending heartbeat signal (sec)
                                     # (change requires restart)
-wd_heartbeat_deadtime = 30
+wd_heartbeat_deadtime = {{ pgpool_wd_heartbeat_deadtime }}
                                     # Deadtime interval for heartbeat signal (sec)
                                     # (change requires restart)
-heartbeat_destination0 = 'host0_ip1'
+heartbeat_destination0 = '{{ pgpool_wd_heartbeat_dest0 }}'
                                     # Host name or IP address of destination 0
                                     # for sending heartbeat signal.
                                     # (change requires restart)
-heartbeat_destination_port0 = 9694 
+heartbeat_destination_port0 = {{ pgpool_wd_heartbeat_dest0_port }}
                                     # Port number of destination 0 for sending
                                     # heartbeat signal. Usually this is the
                                     # same as wd_heartbeat_port.
@@ -543,6 +546,7 @@ heartbeat_device0 = ''
 #heartbeat_destination_port1 = 9694
 #heartbeat_device1 = ''
 
+{% else %}
 # -- query mode --
 
 wd_life_point = 3
@@ -561,21 +565,19 @@ wd_lifecheck_password = ''
                                     # Password for watchdog user in lifecheck
                                     # (change requires restart)
 
+
+{% endif %}
 # - Other pgpool Connection Settings -
 
-#other_pgpool_hostname0 = 'host0'
+other_pgpool_hostname0 = '{{ pgpool_wd_heartbeat_dest0 }}'
                                     # Host name or IP address to connect to for other pgpool 0
                                     # (change requires restart)
-#other_pgpool_port0 = 5432
+other_pgpool_port0 = {{ pgpool_port }}
                                     # Port number for othet pgpool 0
                                     # (change requires restart)
-#other_wd_port0 = 9000
+other_wd_port0 = {{ pgpool_wd_port }}
                                     # Port number for othet watchdog 0
                                     # (change requires restart)
-#other_pgpool_hostname1 = 'host1'
-#other_pgpool_port1 = 5432
-#other_wd_port1 = 9000
-
 
 #------------------------------------------------------------------------------
 # OTHERS