From 137988220bc1adf477fdfe9d0ebe24e01e1c6083 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Mon, 14 Sep 2026 10:44:41 +0200 Subject: [PATCH] See #4577 and #3148. uwsgi workers and hyperkitty archives. --- README.md | 47 ++++++++++++++++++++++++--- defaults/main.yml | 18 +++++++++++ tasks/postorius-hyperkitty.yml | 55 ++++++++++++++++++++++++++++++++ templates/postorius_uwsgi.ini.j2 | 16 ++++++++-- 4 files changed, 129 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e51d26..3254109 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,51 @@ Disable this option before upgrading HyperKitty, django-mailman3, or mailman-hyperkitty, then review whether the compatibility patches are still needed with the new versions. +## uWSGI availability safeguards + +The Mailman web application runs with multiple uWSGI worker processes. A +request that exceeds the configured timeout causes only its worker to be +restarted, while the remaining workers continue to serve Postorius and +HyperKitty. Workers are also recycled after a bounded number of requests or +when their resident memory exceeds the configured threshold: + +```yaml +mailman_postorius_uwsgi_processes: 4 +mailman_postorius_uwsgi_threads: 2 +mailman_postorius_uwsgi_harakiri: 120 +mailman_postorius_uwsgi_max_requests: 1000 +mailman_postorius_uwsgi_reload_on_rss: 512 +``` + +The `harakiri` value should not exceed nginx's `uwsgi_read_timeout`; otherwise +nginx gives up while the blocked worker remains occupied. `reload-on-rss` is +expressed in MiB. Changing these values restarts `mailmansuite-uwsgi` through +the role handler. + ## HyperKitty archive cleanup -Mailman Core does not delete HyperKitty data when a list is removed, and an -`archive_policy` of `never` only prevents future archiving. The optional -cleanup timer removes the corresponding HyperKitty `MailingList`, threads and -messages in both cases: +Mailman Core stores the archive visibility policy and the activation of each +archiver as independent per-list settings. Consequently, setting +`archive_policy` to `never` is not sufficient by itself to stop delivery to +HyperKitty if that archiver remains enabled. The optional reconciliation timer +enforces the safe direction every 15 minutes: it disables HyperKitty for lists +whose policy is `never`, but never enables an archiver and never deletes data: + +```yaml +mailman_enable_hyperkitty_archiver_reconciliation: true +mailman_hyperkitty_archiver_reconciliation_on_active: '5min' +mailman_hyperkitty_archiver_reconciliation_interval: '15min' +``` + +A read-only preview of the current inconsistencies is available with: + +```bash +sudo -u mailman /usr/local/sbin/mailman-hyperkitty-archiver-reconcile --dry-run +``` + +Mailman Core also does not delete HyperKitty data when a list is removed or +when archiving is disabled. The separate cleanup timer removes the +corresponding HyperKitty `MailingList`, threads and messages in both cases: ```yaml mailman_enable_hyperkitty_archive_cleanup: true diff --git a/defaults/main.yml b/defaults/main.yml index 62b94db..41594d1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -63,6 +63,16 @@ mailman_hyperkitty_archive_cleanup_max_deletions: 50 mailman_hyperkitty_archive_cleanup_script: '/usr/local/sbin/hyperkitty-archive-cleanup' mailman_hyperkitty_archive_cleanup_lock: '{{ mailman_lock_dir }}/hyperkitty-archive-cleanup.lock' +# Mailman treats archive_policy and per-list archiver activation as independent +# settings. Periodically enforce the safe, one-way invariant that a list whose +# policy is "never" must not send new messages to HyperKitty. This never enables +# an archiver and does not delete existing archive data. +mailman_enable_hyperkitty_archiver_reconciliation: false +mailman_hyperkitty_archiver_reconciliation_on_active: '5min' +mailman_hyperkitty_archiver_reconciliation_interval: '15min' +mailman_hyperkitty_archiver_reconciliation_script: '/usr/local/sbin/mailman-hyperkitty-archiver-reconcile' +mailman_hyperkitty_archiver_reconciliation_lock: '{{ mailman_lock_dir }}/mailman-hyperkitty-archiver-reconcile.lock' + # Documentation that must be followed to configure the social auth providers # https://django-allauth.readthedocs.io/en/latest/installation.html mailman_use_social_account_providers: False @@ -175,6 +185,14 @@ mailman_postorius_templates_url: '{{ mailman_site_url }}' mailman_postorius_uwsgi_servicename: 'mailmansuite-uwsgi' mailman_postorius_uwsgi_config_file: /etc/mailmansuite-uwsgi.ini mailman_postorius_uwsgi_rundir: /run/mailmansuite +# Keep the HTTP application available when one request blocks on a legacy +# HyperKitty view or a slow backend. Values are intentionally conservative; +# larger installations can override them in group_vars. +mailman_postorius_uwsgi_processes: 2 +mailman_postorius_uwsgi_threads: 2 +mailman_postorius_uwsgi_harakiri: 120 +mailman_postorius_uwsgi_max_requests: 1000 +mailman_postorius_uwsgi_reload_on_rss: 512 # 'systemd_logger,logfile,python36' mailman_postorius_uwsgi_plugins: 'systemd_logger,python36' # 1 is the predefined one, that must be deleted diff --git a/tasks/postorius-hyperkitty.yml b/tasks/postorius-hyperkitty.yml index b9e7b10..983b77c 100644 --- a/tasks/postorius-hyperkitty.yml +++ b/tasks/postorius-hyperkitty.yml @@ -141,6 +141,61 @@ tags: [ 'mailman', 'postorius', 'hyperkitty', 'mailman_conf', 'hyperkitty_archive_cleanup' ] +- name: Reconcile disabled Mailman archives with the HyperKitty archiver + tags: + - mailman + - postorius + - hyperkitty + - mailman_conf + - hyperkitty_archiver_reconciliation + block: + - name: Check whether the HyperKitty archiver reconciliation timer already exists + ansible.builtin.stat: + path: /etc/systemd/system/mailman-hyperkitty-archiver-reconcile.timer + register: mailman_hyperkitty_archiver_reconciliation_timer_before + + - name: Install the HyperKitty archiver reconciliation script + ansible.builtin.template: + src: mailman-hyperkitty-archiver-reconcile.py.j2 + dest: '{{ mailman_hyperkitty_archiver_reconciliation_script }}' + owner: root + group: '{{ mailman_user }}' + mode: '0750' + + - name: Install the HyperKitty archiver reconciliation service + ansible.builtin.template: + src: mailman-hyperkitty-archiver-reconcile.service.systemd.j2 + dest: /etc/systemd/system/mailman-hyperkitty-archiver-reconcile.service + owner: root + group: root + mode: '0644' + register: mailman_hyperkitty_archiver_reconciliation_service_install + + - name: Install the HyperKitty archiver reconciliation timer + ansible.builtin.template: + src: mailman-hyperkitty-archiver-reconcile.timer.systemd.j2 + dest: /etc/systemd/system/mailman-hyperkitty-archiver-reconcile.timer + owner: root + group: root + mode: '0644' + register: mailman_hyperkitty_archiver_reconciliation_timer_install + + - name: Reload systemd after installing the HyperKitty archiver reconciliation units + ansible.builtin.systemd: + daemon_reload: true + when: >- + mailman_hyperkitty_archiver_reconciliation_service_install is changed or + mailman_hyperkitty_archiver_reconciliation_timer_install is changed + + - name: Set the HyperKitty archiver reconciliation timer state + ansible.builtin.systemd: + name: mailman-hyperkitty-archiver-reconcile.timer + state: "{{ mailman_enable_hyperkitty_archiver_reconciliation | bool | ternary('started', 'stopped') }}" + enabled: '{{ mailman_enable_hyperkitty_archiver_reconciliation | bool }}' + when: >- + not ansible_check_mode or + mailman_hyperkitty_archiver_reconciliation_timer_before.stat.exists + - name: Setup the postorius cron jobs block: - name: add a cron job that syncs the mailman core and postorius settings diff --git a/templates/postorius_uwsgi.ini.j2 b/templates/postorius_uwsgi.ini.j2 index b1583e1..ebf8d78 100644 --- a/templates/postorius_uwsgi.ini.j2 +++ b/templates/postorius_uwsgi.ini.j2 @@ -20,8 +20,19 @@ wsgi-file = wsgi.py # Setup default number of processes and threads per process. master = true -process = 2 -threads = 2 +processes = {{ mailman_postorius_uwsgi_processes }} +threads = {{ mailman_postorius_uwsgi_threads }} + +# Kill and respawn a worker whose request exceeds nginx's upstream timeout. +# This prevents a small number of stuck requests from permanently consuming +# every application worker. +harakiri = {{ mailman_postorius_uwsgi_harakiri }} +harakiri-verbose = true + +# Periodic and memory-based recycling bounds the effect of leaks in the legacy +# Python/Django stack. Recycling happens between requests. +max-requests = {{ mailman_postorius_uwsgi_max_requests }} +reload-on-rss = {{ mailman_postorius_uwsgi_reload_on_rss }} {% if not mailman_hyperkitty_qcluster_standalone_service %} # Setup the django_q related worker processes. @@ -43,4 +54,3 @@ log-route = qcluster uwsgi-daemons # Last log and it logs the rest of the stuff. logger = file:{{ mailman_postorius_log_dir }}/uwsgi-error.log -