From 6bf6ce47f0361b3cd52cec0dd3944622d19d42db Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Mon, 31 Aug 2026 14:23:55 +0200 Subject: [PATCH] Do not rely on the systemd unit exit status to decide if the service stop failed. --- ...ailman-verified-restart.service.systemd.j2 | 2 +- templates/mailman-verified-restart.sh.j2 | 95 ++++++++++++++++--- 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/templates/mailman-verified-restart.service.systemd.j2 b/templates/mailman-verified-restart.service.systemd.j2 index d77f6d9..1975265 100644 --- a/templates/mailman-verified-restart.service.systemd.j2 +++ b/templates/mailman-verified-restart.service.systemd.j2 @@ -6,4 +6,4 @@ Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/mailman-verified-restart -TimeoutStartSec={{ (mailman_service_stop_timeout | int) + (mailman_weekly_verified_restart_stop_timeout | int) + (mailman_weekly_verified_restart_start_timeout | int) + 30 }} +TimeoutStartSec={{ (2 * (mailman_service_stop_timeout | int)) + (mailman_weekly_verified_restart_stop_timeout | int) + (2 * (mailman_weekly_verified_restart_start_timeout | int)) + 60 }} diff --git a/templates/mailman-verified-restart.sh.j2 b/templates/mailman-verified-restart.sh.j2 index 70daf55..e54c29b 100644 --- a/templates/mailman-verified-restart.sh.j2 +++ b/templates/mailman-verified-restart.sh.j2 @@ -3,8 +3,6 @@ set -uo pipefail readonly SERVICE='mailman.service' -readonly MAILMAN='{{ mailman_bindir }}/mailman' -readonly CONFIG='{{ mailman_conf_dir }}/mailman.cfg' readonly LMTP_PORT={{ mailman_lmtp_port | int }} readonly SYSTEMD_STOP_TIMEOUT={{ mailman_service_stop_timeout | int }} readonly STOP_TIMEOUT={{ mailman_weekly_verified_restart_stop_timeout | int }} @@ -15,6 +13,27 @@ log() printf 'mailman-verified-restart: %s\n' "$*" } +service_main_pid() +{ + local property + + if ! property="$(systemctl show --property=MainPID "${SERVICE}" 2>/dev/null)"; then + return 1 + fi + + property="${property#MainPID=}" + [[ "${property}" =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "${property}" +} + +service_main_process_is_running() +{ + local main_pid + + main_pid="$(service_main_pid)" || return 1 + [[ "${main_pid}" -gt 0 ]] && kill -0 "${main_pid}" 2>/dev/null +} + lmtp_is_listening() { local sockets @@ -41,13 +60,13 @@ mailman_is_stopped() [[ ${listener_status} -eq 1 ]] && ! systemctl is-active --quiet "${SERVICE}" && - ! "${MAILMAN}" -C "${CONFIG}" status >/dev/null 2>&1 + ! service_main_process_is_running } mailman_is_started() { systemctl is-active --quiet "${SERVICE}" && - "${MAILMAN}" -C "${CONFIG}" status >/dev/null 2>&1 && + service_main_process_is_running && lmtp_is_listening } @@ -66,27 +85,75 @@ wait_for_state() "${check_function}" } +start_mailman() +{ + local start_status + + systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true + + systemctl start --no-block "${SERVICE}" + start_status=$? + if [[ ${start_status} -ne 0 ]]; then + log "WARNING: systemctl start returned ${start_status}; verifying the actual state" + fi + + if wait_for_state mailman_is_started "${START_TIMEOUT}"; then + return 0 + fi + + log 'WARNING: Mailman is not fully active; retrying start once' + systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true + systemctl start --no-block "${SERVICE}" || true + wait_for_state mailman_is_started "${START_TIMEOUT}" +} + +restart_completed=0 +ensure_mailman_is_started() +{ + local exit_status=$? + + trap - EXIT + if [[ ${restart_completed} -eq 0 ]]; then + log 'WARNING: restart cycle was interrupted; attempting to leave Mailman running' + systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true + systemctl start --no-block "${SERVICE}" || true + fi + exit "${exit_status}" +} +trap ensure_mailman_is_started EXIT + log 'stopping Mailman' if ! timeout "$((SYSTEMD_STOP_TIMEOUT + 10))" systemctl stop "${SERVICE}"; then log 'WARNING: systemctl stop did not complete successfully; verifying the actual state' fi -if ! wait_for_state mailman_is_stopped "${STOP_TIMEOUT}"; then - log 'ERROR: Mailman did not stop completely within the timeout; start skipped' - exit 1 -fi -log 'Mailman is completely stopped' +stop_verified=0 +if wait_for_state mailman_is_stopped "${STOP_TIMEOUT}"; then + stop_verified=1 +else + log 'WARNING: Mailman is not completely stopped; killing remaining processes in the service cgroup' + systemctl kill --kill-who=all --signal=TERM "${SERVICE}" >/dev/null 2>&1 || true + systemctl kill --kill-who=all --signal=KILL "${SERVICE}" >/dev/null 2>&1 || true -systemctl reset-failed "${SERVICE}" >/dev/null 2>&1 || true + if wait_for_state mailman_is_stopped "${SYSTEMD_STOP_TIMEOUT}"; then + stop_verified=1 + else + log 'ERROR: Mailman could not be verified as completely stopped; start will still be attempted' + fi +fi +if [[ ${stop_verified} -eq 1 ]]; then + log 'Mailman is completely stopped' +fi log 'starting Mailman' -if ! systemctl start "${SERVICE}"; then - log 'ERROR: systemctl could not start Mailman' +if ! start_mailman; then + log 'ERROR: Mailman did not become active after two start attempts' exit 1 fi -if ! wait_for_state mailman_is_started "${START_TIMEOUT}"; then - log 'ERROR: Mailman did not become active within the timeout' +restart_completed=1 +if [[ ${stop_verified} -eq 0 ]]; then + log 'ERROR: Mailman is running, but the preceding stop could not be verified' exit 1 fi