library/roles/postgresql: First steps of a pgpool configuration that can recovery. Lots of parts are still missing.

postgresql/defaults/main.yml

@@ -94,10 +94,15 @@ pgpool_replication_mode: 'on'
 pgpool_replicate_select: 'off'
 pgpool_insert_lock: 'on'
 pgpool_lobj_lock_table: ''
-pgpool_replication_stop_on_mismatch: 'off'
+pgpool_replication_stop_on_mismatch: 'on'
 pgpool_failover_if_affected_tuples_mismatch: 'off'
 pgpool_load_balance_mode: 'on'
 pgpool_ignore_leading_white_space: 'on'
+pgpool_recovery_user: postgres
+# pgpool_recovery_user_pwd: use a vault file for this one
+pgpool_recovery_stage1_script: pgpool_recovery_stage_1
+pgpool_recovery_stage2_script: pgpool_recovery_stage_2
+pgpool_remote_start_script: postgresql_remote_start
 pgpool_white_function_list: ''
 pgpool_black_function_list: 'nextval,setval'
 pgpool_allow_sql_comments: 'on'

postgresql/tasks/configure-access.yml

@@ -17,6 +17,12 @@
   notify: Reload postgresql
   tags: [ 'postgresql', 'postgres', 'pg_hba' ]
 
+# No conditionals, it is needed to perform base backups when the WAL archive is active
+- name: Give local access with replication privileges to the postgres user
+  lineinfile: name=/etc/postgresql/{{ psql_version }}/main/pg_hba.conf regexp="^local replication postgres peer" line="local replication postgres peer"
+  notify: Reload postgresql
+  tags: [ 'postgresql', 'postgres', 'pg_hba' ]
+
 - name: Set the postgresql listen port
   action: configfile path=/etc/postgresql/{{ psql_version }}/main/postgresql.conf key=port value="{{ psql_db_port }}"
   notify: Restart postgresql

postgresql/tasks/manage_pg_db.yml

@@ -19,11 +19,3 @@
     - ( item.createdb is not defined or item.createdb )
   tags: [ 'postgresql', 'postgres', 'pg_db' ]
 
-# - name: Add postgres extensions to the databases, if needed
-#   become: True
-#   become_user: postgres
-#   postgresql_ext: name={{ item.1 }} db={{ item.0.name }} port={{ psql_db_port }}
-#   with_subelements:
-#     - '{{ psql_db_data | default([]) }}'
-#     - extensions
-#   tags: [ 'postgresql', 'postgres', 'pg_extensions' ]

postgresql/tasks/pgpool-ii.yml

@@ -5,7 +5,8 @@
   tags: [ 'postgresql', 'postgres', 'pgpool' ]
 
 - name: Configure pcp
-  template: src=pcp.conf.j2 dest=/etc/pgpool2/pcp.conf owner=root group=postgres mode=0640
+  #template: src=pcp.conf.j2 dest=/etc/pgpool2/pcp.conf owner=root group=postgres mode=0640
+  shell: pwd=`pg_md5 {{ pcp_pwd }}` ; echo "{{ pgpool_pcp_user }}:${pwd}" > /etc/pgpool2/pcp.conf owner=root group=postgres mode=0640
   tags: [ 'postgresql', 'postgres', 'pgpool', 'pcp_conf', 'pgpool_conf' ]
   
 - name: Install the pgpool configuration file

postgresql/tasks/postgres_pgpool.yml

@@ -6,3 +6,31 @@
   notify: Restart postgresql
   tags: [ 'postgresql', 'postgres', 'pgpool' ]
 
+- name: Add the user that will manage the recovery, if not postgres
+  become: True
+  become_user: postgres
+  postgresql_user: user={{ pgpool_recovery_user }} password={{ pgpool_recovery_user_pwd }} role_attr_flags=REPLICATION port={{ psql_db_port }}
+  when:
+    - ('{{ pgpool_recovery_user }}' != 'postgres')
+    - pgpool_recovery_user_pwd is defined
+  tags: [ 'postgresql', 'postgres', 'pgpool' ]
+
+- name: Give access to the pgpool recovery user, if it is not postgres
+  lineinfile: name=/etc/postgresql/{{ psql_version }}/main/pg_hba.conf regexp="^host {{ item.0.name }} {{ pgpool_recovery_user }} {{ item.1 }}.*$" line="host {{ item.0.name }} {{ pgpool_recovery_user }} {{ item.1 }} md5"
+  with_subelements: 
+    - '{{ psql_db_data | default([]) }}'
+    - allowed_hosts
+  when:
+    - psql_db_data is defined
+    - item.1 is defined
+    - pgpool_recovery_user_pwd is defined
+  notify: Reload postgresql
+  tags: [ 'postgresql', 'postgres', 'pgpool' ]
+
+- name: Install the pgpool recovery and remote restart scripts. They assume that the postgresql hosts can talk to each other
+  template: src={{ item.1 }}.j2 dest={{ item.0.backend_data_directory }}/{{ item.1 }} owner=postgres group=postgres mode=0500
+  with_nested: 
+    - '{{ pgpool_backends | default([]) }}'
+    - [ '{{ pgpool_recovery_stage1_script }}', '{{ pgpool_recovery_stage2_script }}', '{{ pgpool_remote_start_script }}' ]
+  tags: [ 'postgresql', 'postgres', 'pgpool' ]
+

postgresql/templates/pcp.conf.j2

@@ -1 +1 @@
-{{ pgpool_pcp_user }}:{{ '{{ pcp_pwd }}' | hash('md5') }}
+{{ pgpool_pcp_user }}:{{ '{{ pcp_pwd }}' | pg_md5 }}

postgresql/templates/pgpool.conf.j2

@@ -418,13 +418,17 @@ search_primary_node_timeout = 10
 # ONLINE RECOVERY
 #------------------------------------------------------------------------------
 
-recovery_user = 'nobody'
+recovery_user = '{{ pgpool_recovery_user }}'
                                    # Online recovery user
-recovery_password = ''
+{% if pgpool_recovery_user_pwd is defined %}
+recovery_password = '{{ pgpool_recovery_user_pwd | default([]) }}'
                                    # Online recovery password
-recovery_1st_stage_command = ''
+{% else %}
+recovery_password = '{{ pgpool_recovery_user_pwd | default([]) }}'
+{% endif %}
+recovery_1st_stage_command = '{{ pgpool_recovery_stage1_script }}'
                                    # Executes a command in first stage
-recovery_2nd_stage_command = ''
+recovery_2nd_stage_command = '{{ pgpool_recovery_stage2_script }}'
                                    # Executes a command in second stage
 recovery_timeout = 90
                                    # Timeout in seconds to wait for the

postgresql/templates/pgpool_recovery_stage_1.j2

@@ -0,0 +1,10 @@
+#!/bin/bash
+DATA=$1
+RECOVERY_TARGET=$2
+RECOVERY_DATA=$3
+
+psql -c "select pg_start_backup('pgpool-recovery')" postgres
+echo "restore_command = 'scp $HOSTNAME:{{ psql_data_dir }}/archive_log/%f %p'" > {{ psql_data_dir }}/recovery.conf
+tar -C {{ psql_data_dir }}/ -zcf pgsql.tar.gz main
+psql -c 'select pg_stop_backup()' postgres
+scp pgsql.tar.gz $RECOVERY_TARGET:$RECOVERY_DATA

postgresql/templates/pgpool_recovery_stage_2.j2

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Online recovery 2nd stage script
+#
+datadir=$1       # master dabatase cluster
+DEST=$2          # hostname of the DB node to be recovered
+DESTDIR=$3       # database cluster of the DB node to be recovered
+port={{ psql_db_port }}        # PostgreSQL port number
+archdir={{ psql_data_dir }}/archive_log    # archive log directory
+
+# Force to flush current value of sequences to xlog
+psql -p $port -t -c 'SELECT datname FROM pg_database WHERE NOT datistemplate AND datallowconn' template1|
+while read i
+do
+  if [ "$i" != "" ];then
+    psql -p $port -c "SELECT setval(oid, nextval(oid)) FROM pg_class WHERE relkind = 'S'" $i
+  fi
+done
+
+psql -p $port -c "SELECT pgpool_switch_xlog('$archdir')" template1

postgresql/templates/postgresql_remote_start.j2

@@ -0,0 +1,8 @@
+#!/bin/bash
+DEST=$1
+DESTDIR=$2
+
+# Deploy a base backup
+ssh -T $DEST 'cd {{ psql_data_dir }}; tar zxf pgsql.tar.gz' 2>/dev/null 1>/dev/null < /dev/null
+# Startup PostgreSQL server
+ssh -T $DEST sudo /etc/init.d/postgresql start 2>/dev/null 1>/dev/null < /dev/null