library/roles/postgresql: complete the WAL setup needed by pgpool. Fix the pgpool recovery scripts. Fix the pgpool and postgres recovery configuration, users and pgpool functions.

d4science-ghn-cluster/group_vars/postgres_pgpool_test: Add a password for the postgresql postgres user.

postgresql/tasks/configure-access.yml

@@ -75,5 +75,5 @@
   service: name=postgresql state=restarted
   when:
     - postgresql_enabled
-    - ( restart_postgresql | changed )
+    - restart_postgresql.changed
   tags: [ 'postgresql', 'postgres', 'pg_hba', 'pg_conf' ]

postgresql/tasks/manage_pg_db.yml

@@ -4,9 +4,7 @@
   become_user: postgres
   postgresql_user: user={{ item.user }} password={{ item.pwd }} role_attr_flags={{ item.roles }} port={{ psql_db_port }}
   with_items: '{{ psql_db_data }}'
-  when:
-    - psql_db_data is defined
-    - item.roles is defined
+  when: item.roles is defined
   tags: [ 'postgresql', 'postgres', 'pg_db' ]
 
 - name: Add the databases with the correct owner
@@ -14,8 +12,16 @@
   become_user: postgres
   postgresql_db: db={{ item.name }} port={{ psql_db_port }} encoding={{ item.encoding }} owner={{ item.user }} template=template0 state=present
   with_items: '{{ psql_db_data }}'
-  when:
-    - psql_db_data is defined
-    - ( item.createdb is not defined or item.createdb )
+  when: ( item.createdb is not defined or item.createdb )
+  tags: [ 'postgresql', 'postgres', 'pg_db' ]
+
+- name: Only set a db user password. Mostly for the postgresql user
+  become: True
+  become_user: postgres
+  postgresql_user: user={{ item.user }} password={{ item.pwd }} port={{ psql_db_port }}
+  with_items: '{{ psql_db_data }}'
+  when:
+    - item.pwd is defined
+    - item.roles is not defined
   tags: [ 'postgresql', 'postgres', 'pg_db' ]
 

postgresql/tasks/postgres_pgpool.yml

@@ -54,6 +54,15 @@
   notify: Restart postgresql with pgpool config
   tags: [ 'postgresql', 'postgres', 'pg_conf', 'pgpool' ]
 
+- name: Add the pgpool postgres extensions to the template1 dbs
+  become: True
+  become_user: postgres
+  postgresql_ext: name={{ item }} db=template1 port={{ psql_db_port }}
+  with_items:
+    - pgpool_regclass
+    - pgpool_recovery
+  tags: [ 'postgresql', 'postgres', 'pg_extensions' ]
+
 - name: Install the sudoers config that permits the postgres user to restart the service after a recovery
   template: src=postgresql-sudoers.j2 dest=/etc/sudoers.d/postgres-pgpool owner=root group=root mode=0440
   tags: [ 'postgres', 'postgresql', 'sudo', 'pgpool' ]

postgresql/templates/pgpool_recovery_stage_1.j2

@@ -4,28 +4,45 @@
 #
 TS=$(date +%Y-%m-%d_%H-%M-%S)
 MASTER_HOST=$(hostname -f)
+# The variables $1 and $3 are {{ psql_data_dir }}/main
 MASTER_DATA=$1
+MASTER_DATA_BASE_DIR={{ psql_data_dir }}
 RECOVERY_TARGET=$2
 RECOVERY_DATA=$3
+RECOVERY_DATA_BASE_DIR={{ psql_data_dir }}
+
+archive_dir={{ psql_wal_archiving_log_dir }}
+
+# Ensure that postgres is shut down on the target node
+ssh -T $RECOVERY_TARGET sudo /etc/init.d/postgresql stop
 
 # Move the PostgreSQL data directory out of our way.
 ssh -T $RECOVERY_TARGET \
     "[ -d $RECOVERY_DATA ] && mv $RECOVERY_DATA $RECOVERY_DATA.$TS"
 
+# If the archive dir is not empty the backup fails
+ssh -T $RECOVERY_TARGET \
+    "[ -d $archive_dir ] && rm $archive_dir/*"
+
 # We only use archived WAL logs during recoveries, so delete all
 # logs from the last recovery to limit the growth.
-rm $MASTER_DATA/archive_log/*
+rm $archive_dir/*
 
 # With this file present, our archive_command will actually
 # archive WAL files.
-touch $MASTER_DATA/archive_log/backup_in_progress
+touch $archive_dir/backup_in_progress
 
 # Perform a backup of the database.
-ssh -T $RECOVERY_TARGET \
-    "pg_basebackup -h $MASTER_HOST -D $RECOVERY_DATA --xlog"
+# ssh -T $RECOVERY_TARGET \
+#     "pg_basebackup -h $MASTER_HOST -D $archive_dir --xlog"
 
 # Configure the restore_command to use the archive_log WALs we’ll copy
 # over in 2nd_stage.sh.
-echo "restore_command = 'cp $RECOVERY_DATA/archive_log/%f %p'" | \
-    ssh -T $RECOVERY_TARGET "cat > $RECOVERY_DATA/recovery.conf"
-EOF
+# echo "restore_command = 'cp $archive_dir/%f %p'" | ssh -T $RECOVERY_TARGET "cat > $RECOVERY_DATA/recovery.conf"
+
+psql -c "select pg_start_backup('pgpool-recovery')" postgres
+echo "restore_command = 'scp $MASTER_HOST:$archive_dir/%f %p'" > $RECOVERY_DATA/recovery.conf
+tar -C $MASTER_DATA_BASE_DIR -zcf pgsql.tar.gz main
+psql -c 'select pg_stop_backup()' postgres
+scp pgsql.tar.gz $RECOVERY_TARGET:$RECOVERY_DATA_BASE_DIR
+

postgresql/templates/pgpool_recovery_stage_2.j2

@@ -2,11 +2,15 @@
 # Online recovery 2nd stage script
 # Reference: http://michael.stapelberg.de/Artikel/replicated_postgresql_with_pgpool
 #
+MASTER_DATA_BASE_DIR={{ psql_data_dir }}
 MASTER_DATA=$1
 RECOVERY_TARGET=$2
+RECOVERY_DATA_BASE_DIR={{ psql_data_dir }}
 RECOVERY_DATA=$3
 port={{ psql_db_port }}        # PostgreSQL port number
 
+archive_dir={{ psql_wal_archiving_log_dir }}
+
 # Force to flush current value of sequences to xlog
 psql -p $port -t -c 'SELECT datname FROM pg_database WHERE NOT datistemplate AND datallowconn' template1|
 while read i
@@ -19,12 +23,11 @@ done
 # Flush all transactions to disk. Since pgpool stopped all connections,
 # there cannot be any data that does not reside on disk until the
 # to-be-recovered host is back on line.
-psql -p $port -c "SELECT pgpool_switch_xlog('$MASTER_DATA/archive_log')" template1
+psql -p $port -c "SELECT pgpool_switch_xlog('$archive_dir')" template1
 
 # Copy over all archive logs at once.
-rsync -avx --delete $MASTER_DATA/archive_log/ \
-    $RECOVERY_TARGET:$RECOVERY_DATA/archive_log/
+rsync -avx --delete $archive_dir/ \
+    $RECOVERY_TARGET:$archive_dir/
 
 # Delete the flag file to disable WAL archiving again.
-rm $MASTER_DATA/archive_log/backup_in_progress
-EOF
+rm $archive_dir/backup_in_progress

postgresql/templates/pgpool_remote_start.j2

@@ -3,6 +3,6 @@ DEST=$1
 DESTDIR=$2
 
 # Deploy a base backup
-#ssh -T $DEST 'cd {{ psql_data_dir }}; tar zxf pgsql.tar.gz' 2>/dev/null 1>/dev/null < /dev/null
+ssh -T $DEST 'cd {{ psql_data_dir }}; tar zxf pgsql.tar.gz' 2>/dev/null 1>/dev/null < /dev/null
 # Startup PostgreSQL server
 ssh -T $DEST sudo /etc/init.d/postgresql start 2>/dev/null 1>/dev/null < /dev/null