Do not rely on the systemd unit exit status to decide if the service stop failed.

templates/mailman-verified-restart.service.systemd.j2

@@ -6,4 +6,4 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 ExecStart=/usr/local/sbin/mailman-verified-restart
-TimeoutStartSec={{ (mailman_service_stop_timeout | int) + (mailman_weekly_verified_restart_stop_timeout | int) + (mailman_weekly_verified_restart_start_timeout | int) + 30 }}
+TimeoutStartSec={{ (2 * (mailman_service_stop_timeout | int)) + (mailman_weekly_verified_restart_stop_timeout | int) + (2 * (mailman_weekly_verified_restart_start_timeout | int)) + 60 }}

templates/mailman-verified-restart.sh.j2

@@ -3,8 +3,6 @@
 set -uo pipefail
 
 readonly SERVICE='mailman.service'
-readonly MAILMAN='{{ mailman_bindir }}/mailman'
-readonly CONFIG='{{ mailman_conf_dir }}/mailman.cfg'
 readonly LMTP_PORT={{ mailman_lmtp_port | int }}
 readonly SYSTEMD_STOP_TIMEOUT={{ mailman_service_stop_timeout | int }}
 readonly STOP_TIMEOUT={{ mailman_weekly_verified_restart_stop_timeout | int }}
@@ -15,6 +13,27 @@ log()
     printf 'mailman-verified-restart: %s\n' "$*"
 }
 
+service_main_pid()
+{
+    local property
+
+    if ! property="$(systemctl show --property=MainPID "${SERVICE}" 2>/dev/null)"; then
+        return 1
+    fi
+
+    property="${property#MainPID=}"
+    [[ "${property}" =~ ^[0-9]+$ ]] || return 1
+    printf '%s\n' "${property}"
+}
+
+service_main_process_is_running()
+{
+    local main_pid
+
+    main_pid="$(service_main_pid)" || return 1
+    [[ "${main_pid}" -gt 0 ]] && kill -0 "${main_pid}" 2>/dev/null
+}
+
 lmtp_is_listening()
 {
     local sockets
@@ -41,13 +60,13 @@ mailman_is_stopped()
 
     [[ ${listener_status} -eq 1 ]] &&
         ! systemctl is-active --quiet "${SERVICE}" &&
-        ! "${MAILMAN}" -C "${CONFIG}" status >/dev/null 2>&1
+        ! service_main_process_is_running
 }
 
 mailman_is_started()
 {
     systemctl is-active --quiet "${SERVICE}" &&
-        "${MAILMAN}" -C "${CONFIG}" status >/dev/null 2>&1 &&
+        service_main_process_is_running &&
         lmtp_is_listening
 }
 
@@ -66,27 +85,75 @@ wait_for_state()
     "${check_function}"
 }
 
+start_mailman()
+{
+    local start_status
+
+    systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true
+
+    systemctl start --no-block "${SERVICE}"
+    start_status=$?
+    if [[ ${start_status} -ne 0 ]]; then
+        log "WARNING: systemctl start returned ${start_status}; verifying the actual state"
+    fi
+
+    if wait_for_state mailman_is_started "${START_TIMEOUT}"; then
+        return 0
+    fi
+
+    log 'WARNING: Mailman is not fully active; retrying start once'
+    systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true
+    systemctl start --no-block "${SERVICE}" || true
+    wait_for_state mailman_is_started "${START_TIMEOUT}"
+}
+
+restart_completed=0
+ensure_mailman_is_started()
+{
+    local exit_status=$?
+
+    trap - EXIT
+    if [[ ${restart_completed} -eq 0 ]]; then
+        log 'WARNING: restart cycle was interrupted; attempting to leave Mailman running'
+        systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true
+        systemctl start --no-block "${SERVICE}" || true
+    fi
+    exit "${exit_status}"
+}
+trap ensure_mailman_is_started EXIT
+
 log 'stopping Mailman'
 if ! timeout "$((SYSTEMD_STOP_TIMEOUT + 10))" systemctl stop "${SERVICE}"; then
     log 'WARNING: systemctl stop did not complete successfully; verifying the actual state'
 fi
 
-if ! wait_for_state mailman_is_stopped "${STOP_TIMEOUT}"; then
-    log 'ERROR: Mailman did not stop completely within the timeout; start skipped'
-    exit 1
-fi
-log 'Mailman is completely stopped'
+stop_verified=0
+if wait_for_state mailman_is_stopped "${STOP_TIMEOUT}"; then
+    stop_verified=1
+else
+    log 'WARNING: Mailman is not completely stopped; killing remaining processes in the service cgroup'
+    systemctl kill --kill-who=all --signal=TERM "${SERVICE}" >/dev/null 2>&1 || true
+    systemctl kill --kill-who=all --signal=KILL "${SERVICE}" >/dev/null 2>&1 || true
 
-systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true
+    if wait_for_state mailman_is_stopped "${SYSTEMD_STOP_TIMEOUT}"; then
+        stop_verified=1
+    else
+        log 'ERROR: Mailman could not be verified as completely stopped; start will still be attempted'
+    fi
+fi
+if [[ ${stop_verified} -eq 1 ]]; then
+    log 'Mailman is completely stopped'
+fi
 
 log 'starting Mailman'
-if ! systemctl start "${SERVICE}"; then
-    log 'ERROR: systemctl could not start Mailman'
+if ! start_mailman; then
+    log 'ERROR: Mailman did not become active after two start attempts'
     exit 1
 fi
 
-if ! wait_for_state mailman_is_started "${START_TIMEOUT}"; then
-    log 'ERROR: Mailman did not become active within the timeout'
+restart_completed=1
+if [[ ${stop_verified} -eq 0 ]]; then
+    log 'ERROR: Mailman is running, but the preceding stop could not be verified'
     exit 1
 fi