mailman/templates/hyperkitty-archive-cleanup.py.j2

#!{{ mailman_bindir }}/python3

"""Delete HyperKitty archives that Mailman Core no longer archives."""

import argparse
import fcntl
import logging
import os
import sys
from urllib.error import HTTPError


PROJECT_DIR = {{ mailman_postorius_dir | to_json }}
LOCK_FILE = {{ mailman_hyperkitty_archive_cleanup_lock | to_json }}
DELETE_DELETED_LISTS = {{ mailman_hyperkitty_archive_cleanup_deleted_lists | bool | ternary('True', 'False') }}
DELETE_DISABLED_LISTS = {{ mailman_hyperkitty_archive_cleanup_disabled_lists | bool | ternary('True', 'False') }}
MAX_DELETIONS = {{ mailman_hyperkitty_archive_cleanup_max_deletions | int }}
# HyperKitty 1.3.3 itself uses pages of 10 with the legacy Core client.
PAGE_SIZE = 10


logging.basicConfig(
    level=logging.INFO,
    format='hyperkitty-archive-cleanup: %(levelname)s: %(message)s')
LOG = logging.getLogger(__name__)


def normalize_name(value):
    return value.strip().casefold()


def normalize_policy(value):
    if hasattr(value, 'name'):
        value = value.name
    if isinstance(value, int):
        return {0: 'never', 1: 'private', 2: 'public'}.get(value, str(value))
    return str(value).strip().lower()


def get_archive_policy(core_list):
    try:
        value = core_list.archive_policy
    except AttributeError:
        value = core_list.settings['archive_policy']
    return normalize_policy(value)


def load_core_lists(client):
    """Fetch a complete Core snapshot before deciding what may be deleted."""
    result = {}
    page_number = 1

    while True:
        page = list(client.get_list_page(count=PAGE_SIZE, page=page_number))
        if not page:
            break

        previous_count = len(result)
        for core_list in page:
            result[normalize_name(core_list.fqdn_listname)] = get_archive_policy(core_list)

        if len(result) == previous_count:
            raise RuntimeError(
                'Mailman Core pagination returned no new lists on page {}'.format(
                    page_number))
        if len(page) < PAGE_SIZE:
            break
        page_number += 1

    if not result:
        raise RuntimeError(
            'Mailman Core returned no lists; refusing to consider every archive orphaned')
    return result


def recheck_candidate(client, name):
    """Return the current Core policy, or None only for a confirmed 404."""
    try:
        core_list = client.get_list(name)
    except HTTPError as error:
        if error.code == 404:
            return None
        raise
    if core_list is None:
        return None
    return get_archive_policy(core_list)


def find_candidates(core_lists, archive_lists):
    candidates = []
    for archive in archive_lists:
        policy = core_lists.get(normalize_name(archive.name))
        if policy is None and DELETE_DELETED_LISTS:
            candidates.append((archive, 'deleted from Mailman Core'))
        elif policy == 'never' and DELETE_DISABLED_LISTS:
            candidates.append((archive, 'archive policy is never'))
    return candidates


def confirm_candidates(client, candidates):
    confirmed = []
    for archive, original_reason in candidates:
        current_policy = recheck_candidate(client, archive.name)
        if current_policy is None and DELETE_DELETED_LISTS:
            confirmed.append((archive, 'deleted from Mailman Core'))
        elif current_policy == 'never' and DELETE_DISABLED_LISTS:
            confirmed.append((archive, 'archive policy is never'))
        else:
            LOG.warning(
                'skipping %s: Core changed since the initial snapshot (%s)',
                archive.name, original_reason)
    return confirmed


def delete_archives(candidates, dry_run):
    from django.core.cache import cache
    from django.db import transaction
    from django.db.models.signals import post_delete, pre_delete
    from hyperkitty.models import Email, MailingList, Thread, Vote
    from hyperkitty.signals import (
        Email_on_post_delete,
        Email_on_pre_delete,
        Thread_on_post_delete,
        Vote_on_post_delete,
    )

    signal_bindings = (
        (pre_delete, Email_on_pre_delete, Email),
        (post_delete, Email_on_post_delete, Email),
        (post_delete, Thread_on_post_delete, Thread),
        (post_delete, Vote_on_post_delete, Vote),
    )

    if dry_run:
        for archive, reason in candidates:
            LOG.info(
                'would delete %s (%s; %d threads, %d messages)',
                archive.name, reason, archive.threads.count(), archive.emails.count())
        return 0

    disconnected = []
    try:
        # Bulk archive deletion must not enqueue cache rebuilds for objects that
        # cease to exist before django-q executes the tasks (HyperKitty #440).
        for signal, receiver, sender in signal_bindings:
            if signal.disconnect(receiver, sender=sender):
                disconnected.append((signal, receiver, sender))

        deleted = 0
        for archive, reason in candidates:
            with transaction.atomic():
                try:
                    locked_archive = MailingList.objects.select_for_update().get(
                        pk=archive.pk)
                except MailingList.DoesNotExist:
                    LOG.warning('archive %s disappeared before deletion', archive.name)
                    continue

                thread_count = locked_archive.threads.count()
                message_count = locked_archive.emails.count()
                # Match HyperKitty's administrative delete view: deleting
                # threads explicitly also works around backend FK constraints.
                locked_archive.threads.all().delete()
                locked_archive.delete()
                deleted += 1
                LOG.info(
                    'deleted %s (%s; %d threads, %d messages)',
                    archive.name, reason, thread_count, message_count)
    finally:
        for signal, receiver, sender in disconnected:
            signal.connect(receiver, sender=sender)

    if deleted:
        cache.clear()
    return deleted


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--dry-run', action='store_true',
        help='report candidates without deleting archives')
    arguments = parser.parse_args()

    os.chdir(PROJECT_DIR)
    sys.path.insert(0, PROJECT_DIR)
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')

    import django
    django.setup()

    from django_mailman3.lib.mailman import get_mailman_client
    from hyperkitty.models import MailingList
    from mailmanclient import MailmanConnectionError

    try:
        with open(LOCK_FILE, 'w') as lock_handle:
            try:
                fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
            except OSError:
                LOG.error('another cleanup process holds %s', LOCK_FILE)
                return 1

            try:
                client = get_mailman_client()
                core_lists = load_core_lists(client)
                archives = list(MailingList.objects.all().order_by('name'))
                candidates = find_candidates(core_lists, archives)
                candidates = confirm_candidates(client, candidates)
            except (HTTPError, MailmanConnectionError, RuntimeError) as error:
                LOG.error('cannot obtain a reliable Mailman Core view: %s', error)
                return 1

            if MAX_DELETIONS > 0 and len(candidates) > MAX_DELETIONS:
                LOG.error(
                    'refusing to delete %d archives; safety limit is %d',
                    len(candidates), MAX_DELETIONS)
                for archive, reason in candidates:
                    LOG.error('candidate %s (%s)', archive.name, reason)
                return 1

            deleted = delete_archives(candidates, arguments.dry_run)
            LOG.info(
                '%d Core lists, %d HyperKitty archives, %d candidates, %d deleted',
                len(core_lists), len(archives), len(candidates), deleted)
            return 0
    except OSError as error:
        LOG.error('cannot use cleanup lock %s: %s', LOCK_FILE, error)
        return 1


if __name__ == '__main__':
    sys.exit(main())