diff --git a/README.md b/README.md index 3254109..d63c46a 100644 --- a/README.md +++ b/README.md @@ -164,3 +164,49 @@ sudo -u mailman /usr/local/sbin/hyperkitty-archive-cleanup --dry-run The monthly HyperKitty job already configured by this role removes stale entries from the full-text search index. Therefore the weekly cleanup does not run an additional, expensive full index scan. + +## Web signup storm protection + +The optional nginx configuration limits only `POST /accounts/signup/`; all +other Postorius, HyperKitty and API requests use an empty nginx limit key and +are not counted. Both a per-client-IP zone and a site-wide zone are used, since +distributed signup storms can evade a per-IP limit: + +```yaml +mailman_enable_signup_rate_limit: true +mailman_signup_rate_limit_per_ip_rate: '1r/m' +mailman_signup_rate_limit_per_ip_burst: 2 +mailman_signup_rate_limit_global_rate: '5r/m' +mailman_signup_rate_limit_global_burst: 10 +mailman_signup_rate_limit_dry_run: true +``` + +Dry-run mode requires nginx 1.17.1 or newer. It logs requests that would have +been limited at `notice` level but does not reject them. Review those logs +before setting `mailman_signup_rate_limit_dry_run` to `false`; enforced requests +receive HTTP 429. The nginx configuration is validated before it is reloaded. + +## Expired unverified web-account cleanup + +The optional timer removes abandoned django-allauth registrations. An account +is eligible only when it is older than the configured age, has never logged in, +is neither staff nor superuser, belongs to no group, has no social account, has +at least one unverified address but no verified address, and has no confirmation +mail newer than the cutoff: + +```yaml +mailman_enable_unverified_account_cleanup: true +mailman_unverified_account_cleanup_on_calendar: 'Sun *-*-* 03:30:00' +mailman_unverified_account_cleanup_min_age_days: 7 +mailman_unverified_account_cleanup_max_deletions: 50 +``` + +The script repeats all eligibility tests while holding database locks and +deletes the complete batch in one transaction. If the candidate count exceeds +the safety limit, it deletes nothing; it never removes only the first N users. +The timer is not persistent, so a missed run is not started after boot. Preview +the candidates before the first execution with: + +```bash +sudo -u mailman /usr/local/sbin/mailman-unverified-account-cleanup --dry-run +``` diff --git a/defaults/main.yml b/defaults/main.yml index 41594d1..9523bff 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -73,6 +73,28 @@ mailman_hyperkitty_archiver_reconciliation_interval: '15min' mailman_hyperkitty_archiver_reconciliation_script: '/usr/local/sbin/mailman-hyperkitty-archiver-reconcile' mailman_hyperkitty_archiver_reconciliation_lock: '{{ mailman_lock_dir }}/mailman-hyperkitty-archiver-reconcile.lock' +# Protect the django-allauth signup endpoint without limiting normal Postorius +# or HyperKitty traffic. The generated nginx configuration is opt-in and starts +# in observation-only mode by default. +mailman_enable_signup_rate_limit: false +mailman_signup_rate_limit_config_file: '/etc/nginx/conf.d/00-mailman-signup-rate-limit.conf' +mailman_signup_rate_limit_per_ip_rate: '1r/m' +mailman_signup_rate_limit_per_ip_burst: 2 +mailman_signup_rate_limit_global_rate: '5r/m' +mailman_signup_rate_limit_global_burst: 10 +mailman_signup_rate_limit_status: 429 +mailman_signup_rate_limit_dry_run: true + +# Remove old django-allauth accounts that have never been verified or used. +# Deletion is fail-closed: a batch larger than max_deletions is not partially +# processed, and the whole deletion runs in one database transaction. +mailman_enable_unverified_account_cleanup: false +mailman_unverified_account_cleanup_on_calendar: 'Sun *-*-* 03:30:00' +mailman_unverified_account_cleanup_min_age_days: 7 +mailman_unverified_account_cleanup_max_deletions: 50 +mailman_unverified_account_cleanup_script: '/usr/local/sbin/mailman-unverified-account-cleanup' +mailman_unverified_account_cleanup_lock: '{{ mailman_lock_dir }}/mailman-unverified-account-cleanup.lock' + # Documentation that must be followed to configure the social auth providers # https://django-allauth.readthedocs.io/en/latest/installation.html mailman_use_social_account_providers: False diff --git a/handlers/main.yml b/handlers/main.yml index 61b3596..7125ba6 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -17,3 +17,14 @@ sleep: 1 timeout: '{{ mailman_weekly_verified_restart_start_timeout }}' listen: Restart mailman + +- name: Validate nginx configuration after Mailman signup rate-limit change + ansible.builtin.command: /usr/sbin/nginx -t + changed_when: false + listen: Reload nginx after Mailman signup rate-limit change + +- name: Reload nginx after Mailman signup rate-limit change + ansible.builtin.service: + name: nginx + state: reloaded + listen: Reload nginx after Mailman signup rate-limit change diff --git a/tasks/postorius-hyperkitty.yml b/tasks/postorius-hyperkitty.yml index 983b77c..9daa545 100644 --- a/tasks/postorius-hyperkitty.yml +++ b/tasks/postorius-hyperkitty.yml @@ -196,6 +196,84 @@ not ansible_check_mode or mailman_hyperkitty_archiver_reconciliation_timer_before.stat.exists +- name: Configure protection for the Mailman web signup endpoint + tags: + - mailman + - postorius + - mailman_conf + - mailman_signup_rate_limit + block: + - name: Install the Mailman signup nginx rate-limit configuration + ansible.builtin.template: + src: mailman-signup-rate-limit.nginx.conf.j2 + dest: '{{ mailman_signup_rate_limit_config_file }}' + owner: root + group: root + mode: '0644' + when: mailman_enable_signup_rate_limit | bool + notify: Reload nginx after Mailman signup rate-limit change + + - name: Remove the Mailman signup nginx rate-limit configuration + ansible.builtin.file: + path: '{{ mailman_signup_rate_limit_config_file }}' + state: absent + when: not mailman_enable_signup_rate_limit | bool + notify: Reload nginx after Mailman signup rate-limit change + +- name: Configure automatic cleanup of expired unverified web accounts + tags: + - mailman + - postorius + - mailman_conf + - mailman_unverified_account_cleanup + block: + - name: Check whether the unverified-account cleanup timer already exists + ansible.builtin.stat: + path: /etc/systemd/system/mailman-unverified-account-cleanup.timer + register: mailman_unverified_account_cleanup_timer_before + + - name: Install the unverified-account cleanup script + ansible.builtin.template: + src: mailman-unverified-account-cleanup.py.j2 + dest: '{{ mailman_unverified_account_cleanup_script }}' + owner: root + group: '{{ mailman_user }}' + mode: '0750' + + - name: Install the unverified-account cleanup service + ansible.builtin.template: + src: mailman-unverified-account-cleanup.service.systemd.j2 + dest: /etc/systemd/system/mailman-unverified-account-cleanup.service + owner: root + group: root + mode: '0644' + register: mailman_unverified_account_cleanup_service_install + + - name: Install the unverified-account cleanup timer + ansible.builtin.template: + src: mailman-unverified-account-cleanup.timer.systemd.j2 + dest: /etc/systemd/system/mailman-unverified-account-cleanup.timer + owner: root + group: root + mode: '0644' + register: mailman_unverified_account_cleanup_timer_install + + - name: Reload systemd after installing the account cleanup units + ansible.builtin.systemd: + daemon_reload: true + when: >- + mailman_unverified_account_cleanup_service_install is changed or + mailman_unverified_account_cleanup_timer_install is changed + + - name: Set the unverified-account cleanup timer state + ansible.builtin.systemd: + name: mailman-unverified-account-cleanup.timer + state: "{{ mailman_enable_unverified_account_cleanup | bool | ternary('started', 'stopped') }}" + enabled: '{{ mailman_enable_unverified_account_cleanup | bool }}' + when: >- + not ansible_check_mode or + mailman_unverified_account_cleanup_timer_before.stat.exists + - name: Setup the postorius cron jobs block: - name: add a cron job that syncs the mailman core and postorius settings diff --git a/templates/mailman-signup-rate-limit.nginx.conf.j2 b/templates/mailman-signup-rate-limit.nginx.conf.j2 new file mode 100644 index 0000000..165039f --- /dev/null +++ b/templates/mailman-signup-rate-limit.nginx.conf.j2 @@ -0,0 +1,22 @@ +# Managed by Ansible. Limit only attempts to submit the allauth +# signup form; an empty key means every other request bypasses the zones. +map "$request_method:$uri" $mailman_signup_ip_key { + default ""; + "POST:/accounts/signup/" $binary_remote_addr; +} + +map "$request_method:$uri" $mailman_signup_global_key { + default ""; + "POST:/accounts/signup/" "mailman-signup"; +} + +limit_req_zone $mailman_signup_ip_key zone=mailman_signup_ip:1m rate={{ mailman_signup_rate_limit_per_ip_rate }}; +limit_req_zone $mailman_signup_global_key zone=mailman_signup_global:1m rate={{ mailman_signup_rate_limit_global_rate }}; + +limit_req zone=mailman_signup_ip burst={{ mailman_signup_rate_limit_per_ip_burst | int }} nodelay; +limit_req zone=mailman_signup_global burst={{ mailman_signup_rate_limit_global_burst | int }} nodelay; +limit_req_status {{ mailman_signup_rate_limit_status | int }}; +limit_req_log_level notice; +{% if mailman_signup_rate_limit_dry_run | bool %} +limit_req_dry_run on; +{% endif %} diff --git a/templates/mailman-unverified-account-cleanup.py.j2 b/templates/mailman-unverified-account-cleanup.py.j2 new file mode 100644 index 0000000..a4b247c --- /dev/null +++ b/templates/mailman-unverified-account-cleanup.py.j2 @@ -0,0 +1,157 @@ +#!{{ mailman_bindir }}/python3 + +"""Remove expired, unused and unverified django-allauth accounts.""" + +import argparse +from datetime import timedelta +import fcntl +import logging +import os +import sys + + +PROJECT_DIR = {{ mailman_postorius_dir | to_json }} +LOCK_FILE = {{ mailman_unverified_account_cleanup_lock | to_json }} +MIN_AGE_DAYS = {{ mailman_unverified_account_cleanup_min_age_days | int }} +MAX_DELETIONS = {{ mailman_unverified_account_cleanup_max_deletions | int }} + + +logging.basicConfig( + level=logging.INFO, + format='mailman-unverified-account-cleanup: %(levelname)s: %(message)s') +LOG = logging.getLogger(__name__) + + +def eligible_users(User, EmailAddress, EmailConfirmation, SocialAccount, cutoff): + """Build a conservative query that can also be repeated before deletion.""" + from django.db.models import Exists, OuterRef + + verified_addresses = EmailAddress.objects.filter( + user_id=OuterRef('pk'), verified=True) + unverified_addresses = EmailAddress.objects.filter( + user_id=OuterRef('pk'), verified=False) + recent_confirmations = EmailConfirmation.objects.filter( + email_address__user_id=OuterRef('pk'), sent__gte=cutoff) + social_accounts = SocialAccount.objects.filter(user_id=OuterRef('pk')) + group_memberships = User.groups.through.objects.filter(user_id=OuterRef('pk')) + + return User.objects.annotate( + has_verified_address=Exists(verified_addresses), + has_unverified_address=Exists(unverified_addresses), + has_recent_confirmation=Exists(recent_confirmations), + has_social_account=Exists(social_accounts), + has_group=Exists(group_memberships), + ).filter( + date_joined__lt=cutoff, + last_login__isnull=True, + is_staff=False, + is_superuser=False, + has_verified_address=False, + has_unverified_address=True, + has_recent_confirmation=False, + has_social_account=False, + has_group=False, + ) + + +def describe(users, prefix): + for user_id, username, date_joined in users: + LOG.info( + '%s user_id=%s username=%r date_joined=%s', + prefix, user_id, username, date_joined.isoformat()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--dry-run', action='store_true', + help='report candidates without deleting accounts') + arguments = parser.parse_args() + + if MIN_AGE_DAYS < 1: + LOG.error('minimum account age must be at least one day') + return 1 + if MAX_DELETIONS < 1: + LOG.error('maximum deletions must be at least one') + return 1 + + os.chdir(PROJECT_DIR) + sys.path.insert(0, PROJECT_DIR) + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') + + import django + django.setup() + + from allauth.account.models import EmailAddress, EmailConfirmation + from allauth.socialaccount.models import SocialAccount + from django.contrib.auth import get_user_model + from django.db import transaction + from django.utils import timezone + + User = get_user_model() + cutoff = timezone.now() - timedelta(days=MIN_AGE_DAYS) + + try: + with open(LOCK_FILE, 'w') as lock_handle: + try: + fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + LOG.error('another cleanup process holds %s', LOCK_FILE) + return 1 + + candidates = list( + eligible_users( + User, EmailAddress, EmailConfirmation, SocialAccount, cutoff) + .order_by('date_joined', 'pk') + .values_list('pk', 'username', 'date_joined')) + + if arguments.dry_run: + describe(candidates, 'would delete') + if len(candidates) > MAX_DELETIONS: + LOG.warning( + 'a real run would refuse all %d candidates; safety limit is %d', + len(candidates), MAX_DELETIONS) + LOG.info('%d candidates, 0 deleted', len(candidates)) + return 0 + + if len(candidates) > MAX_DELETIONS: + LOG.error( + 'refusing to delete any of %d accounts; safety limit is %d', + len(candidates), MAX_DELETIONS) + return 1 + + candidate_ids = [candidate[0] for candidate in candidates] + if not candidate_ids: + LOG.info('0 candidates, 0 deleted') + return 0 + + with transaction.atomic(): + # Lock the candidate users, then repeat every eligibility test. + # Any deletion error rolls back the complete batch. + locked_ids = list( + User.objects.select_for_update() + .filter(pk__in=candidate_ids) + .values_list('pk', flat=True)) + confirmed = list( + eligible_users( + User, EmailAddress, EmailConfirmation, SocialAccount, cutoff) + .filter(pk__in=locked_ids) + .order_by('date_joined', 'pk') + .values_list('pk', 'username', 'date_joined')) + confirmed_ids = [candidate[0] for candidate in confirmed] + if len(confirmed_ids) != len(candidate_ids): + LOG.warning( + '%d accounts changed during the recheck and will be preserved', + len(candidate_ids) - len(confirmed_ids)) + User.objects.filter(pk__in=confirmed_ids).delete() + + describe(confirmed, 'deleted') + LOG.info('%d candidates, %d deleted', len(candidates), len(confirmed)) + return 0 + except OSError as error: + LOG.error('cannot use cleanup lock %s: %s', LOCK_FILE, error) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/templates/mailman-unverified-account-cleanup.service.systemd.j2 b/templates/mailman-unverified-account-cleanup.service.systemd.j2 new file mode 100644 index 0000000..691bd86 --- /dev/null +++ b/templates/mailman-unverified-account-cleanup.service.systemd.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Remove expired unverified Mailman web accounts +After=postgresql.service +ConditionPathExists={{ mailman_postorius_dir }}/settings.py + +[Service] +Type=oneshot +User={{ mailman_user }} +Group={{ mailman_user }} +WorkingDirectory={{ mailman_postorius_dir }} +ExecStart={{ mailman_unverified_account_cleanup_script }} +TimeoutStartSec=1h +Nice=10 +IOSchedulingClass=idle diff --git a/templates/mailman-unverified-account-cleanup.timer.systemd.j2 b/templates/mailman-unverified-account-cleanup.timer.systemd.j2 new file mode 100644 index 0000000..5fa6394 --- /dev/null +++ b/templates/mailman-unverified-account-cleanup.timer.systemd.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Weekly cleanup of expired unverified Mailman web accounts + +[Timer] +OnCalendar={{ mailman_unverified_account_cleanup_on_calendar }} +AccuracySec=5min +Persistent=false +Unit=mailman-unverified-account-cleanup.service + +[Install] +WantedBy=timers.target