See #4577 and #3148. uwsgi workers and hyperkitty archives.

README.md

@@ -91,12 +91,51 @@ Disable this option before upgrading HyperKitty, django-mailman3, or
 mailman-hyperkitty, then review whether the compatibility patches are still
 needed with the new versions.
 
+## uWSGI availability safeguards
+
+The Mailman web application runs with multiple uWSGI worker processes. A
+request that exceeds the configured timeout causes only its worker to be
+restarted, while the remaining workers continue to serve Postorius and
+HyperKitty. Workers are also recycled after a bounded number of requests or
+when their resident memory exceeds the configured threshold:
+
+```yaml
+mailman_postorius_uwsgi_processes: 4
+mailman_postorius_uwsgi_threads: 2
+mailman_postorius_uwsgi_harakiri: 120
+mailman_postorius_uwsgi_max_requests: 1000
+mailman_postorius_uwsgi_reload_on_rss: 512
+```
+
+The `harakiri` value should not exceed nginx's `uwsgi_read_timeout`; otherwise
+nginx gives up while the blocked worker remains occupied. `reload-on-rss` is
+expressed in MiB. Changing these values restarts `mailmansuite-uwsgi` through
+the role handler.
+
 ## HyperKitty archive cleanup
 
-Mailman Core does not delete HyperKitty data when a list is removed, and an
-`archive_policy` of `never` only prevents future archiving. The optional
-cleanup timer removes the corresponding HyperKitty `MailingList`, threads and
-messages in both cases:
+Mailman Core stores the archive visibility policy and the activation of each
+archiver as independent per-list settings. Consequently, setting
+`archive_policy` to `never` is not sufficient by itself to stop delivery to
+HyperKitty if that archiver remains enabled. The optional reconciliation timer
+enforces the safe direction every 15 minutes: it disables HyperKitty for lists
+whose policy is `never`, but never enables an archiver and never deletes data:
+
+```yaml
+mailman_enable_hyperkitty_archiver_reconciliation: true
+mailman_hyperkitty_archiver_reconciliation_on_active: '5min'
+mailman_hyperkitty_archiver_reconciliation_interval: '15min'
+```
+
+A read-only preview of the current inconsistencies is available with:
+
+```bash
+sudo -u mailman /usr/local/sbin/mailman-hyperkitty-archiver-reconcile --dry-run
+```
+
+Mailman Core also does not delete HyperKitty data when a list is removed or
+when archiving is disabled. The separate cleanup timer removes the
+corresponding HyperKitty `MailingList`, threads and messages in both cases:
 
 ```yaml
 mailman_enable_hyperkitty_archive_cleanup: true

defaults/main.yml

@@ -63,6 +63,16 @@ mailman_hyperkitty_archive_cleanup_max_deletions: 50
 mailman_hyperkitty_archive_cleanup_script: '/usr/local/sbin/hyperkitty-archive-cleanup'
 mailman_hyperkitty_archive_cleanup_lock: '{{ mailman_lock_dir }}/hyperkitty-archive-cleanup.lock'
 
+# Mailman treats archive_policy and per-list archiver activation as independent
+# settings. Periodically enforce the safe, one-way invariant that a list whose
+# policy is "never" must not send new messages to HyperKitty. This never enables
+# an archiver and does not delete existing archive data.
+mailman_enable_hyperkitty_archiver_reconciliation: false
+mailman_hyperkitty_archiver_reconciliation_on_active: '5min'
+mailman_hyperkitty_archiver_reconciliation_interval: '15min'
+mailman_hyperkitty_archiver_reconciliation_script: '/usr/local/sbin/mailman-hyperkitty-archiver-reconcile'
+mailman_hyperkitty_archiver_reconciliation_lock: '{{ mailman_lock_dir }}/mailman-hyperkitty-archiver-reconcile.lock'
+
 # Documentation that must be followed to configure the social auth providers
 # https://django-allauth.readthedocs.io/en/latest/installation.html
 mailman_use_social_account_providers: False
@@ -175,6 +185,14 @@ mailman_postorius_templates_url: '{{ mailman_site_url }}'
 mailman_postorius_uwsgi_servicename: 'mailmansuite-uwsgi'
 mailman_postorius_uwsgi_config_file: /etc/mailmansuite-uwsgi.ini
 mailman_postorius_uwsgi_rundir: /run/mailmansuite
+# Keep the HTTP application available when one request blocks on a legacy
+# HyperKitty view or a slow backend. Values are intentionally conservative;
+# larger installations can override them in group_vars.
+mailman_postorius_uwsgi_processes: 2
+mailman_postorius_uwsgi_threads: 2
+mailman_postorius_uwsgi_harakiri: 120
+mailman_postorius_uwsgi_max_requests: 1000
+mailman_postorius_uwsgi_reload_on_rss: 512
 # 'systemd_logger,logfile,python36'
 mailman_postorius_uwsgi_plugins: 'systemd_logger,python36'
 # 1 is the predefined one, that must be deleted

tasks/postorius-hyperkitty.yml

@@ -141,6 +141,61 @@
 
   tags: [ 'mailman', 'postorius', 'hyperkitty', 'mailman_conf', 'hyperkitty_archive_cleanup' ]
 
+- name: Reconcile disabled Mailman archives with the HyperKitty archiver
+  tags:
+    - mailman
+    - postorius
+    - hyperkitty
+    - mailman_conf
+    - hyperkitty_archiver_reconciliation
+  block:
+    - name: Check whether the HyperKitty archiver reconciliation timer already exists
+      ansible.builtin.stat:
+        path: /etc/systemd/system/mailman-hyperkitty-archiver-reconcile.timer
+      register: mailman_hyperkitty_archiver_reconciliation_timer_before
+
+    - name: Install the HyperKitty archiver reconciliation script
+      ansible.builtin.template:
+        src: mailman-hyperkitty-archiver-reconcile.py.j2
+        dest: '{{ mailman_hyperkitty_archiver_reconciliation_script }}'
+        owner: root
+        group: '{{ mailman_user }}'
+        mode: '0750'
+
+    - name: Install the HyperKitty archiver reconciliation service
+      ansible.builtin.template:
+        src: mailman-hyperkitty-archiver-reconcile.service.systemd.j2
+        dest: /etc/systemd/system/mailman-hyperkitty-archiver-reconcile.service
+        owner: root
+        group: root
+        mode: '0644'
+      register: mailman_hyperkitty_archiver_reconciliation_service_install
+
+    - name: Install the HyperKitty archiver reconciliation timer
+      ansible.builtin.template:
+        src: mailman-hyperkitty-archiver-reconcile.timer.systemd.j2
+        dest: /etc/systemd/system/mailman-hyperkitty-archiver-reconcile.timer
+        owner: root
+        group: root
+        mode: '0644'
+      register: mailman_hyperkitty_archiver_reconciliation_timer_install
+
+    - name: Reload systemd after installing the HyperKitty archiver reconciliation units
+      ansible.builtin.systemd:
+        daemon_reload: true
+      when: >-
+        mailman_hyperkitty_archiver_reconciliation_service_install is changed or
+        mailman_hyperkitty_archiver_reconciliation_timer_install is changed
+
+    - name: Set the HyperKitty archiver reconciliation timer state
+      ansible.builtin.systemd:
+        name: mailman-hyperkitty-archiver-reconcile.timer
+        state: "{{ mailman_enable_hyperkitty_archiver_reconciliation | bool | ternary('started', 'stopped') }}"
+        enabled: '{{ mailman_enable_hyperkitty_archiver_reconciliation | bool }}'
+      when: >-
+        not ansible_check_mode or
+        mailman_hyperkitty_archiver_reconciliation_timer_before.stat.exists
+
 - name: Setup the postorius cron jobs
   block:
     - name: add a cron job that syncs the mailman core and postorius settings

templates/postorius_uwsgi.ini.j2

@@ -20,8 +20,19 @@ wsgi-file = wsgi.py
 
 # Setup default number of processes and threads per process.
 master = true
-process = 2
-threads = 2
+processes = {{ mailman_postorius_uwsgi_processes }}
+threads = {{ mailman_postorius_uwsgi_threads }}
+
+# Kill and respawn a worker whose request exceeds nginx's upstream timeout.
+# This prevents a small number of stuck requests from permanently consuming
+# every application worker.
+harakiri = {{ mailman_postorius_uwsgi_harakiri }}
+harakiri-verbose = true
+
+# Periodic and memory-based recycling bounds the effect of leaks in the legacy
+# Python/Django stack. Recycling happens between requests.
+max-requests = {{ mailman_postorius_uwsgi_max_requests }}
+reload-on-rss = {{ mailman_postorius_uwsgi_reload_on_rss }}
 
 {% if not mailman_hyperkitty_qcluster_standalone_service %}
 # Setup the django_q related worker processes.
@@ -43,4 +54,3 @@ log-route = qcluster uwsgi-daemons
 
 # Last log and it logs the rest of the stuff.
 logger = file:{{  mailman_postorius_log_dir }}/uwsgi-error.log
-