diff --git a/tasks/postorius-hyperkitty.yml b/tasks/postorius-hyperkitty.yml
index 43e4ac8..2ed8677 100644
--- a/tasks/postorius-hyperkitty.yml
+++ b/tasks/postorius-hyperkitty.yml
@@ -45,6 +45,7 @@
         owner: root
         group: '{{ mailman_user }}'
         mode: '0750'
+      register: mailman_postorius_site_config_helper_install
       when: mailman_manage_postorius_site | bool
       tags: [mailman_postorius_site]
 
@@ -60,7 +61,11 @@
         'CHANGED:' in mailman_postorius_site_config_result.stdout or
         'WOULD CHANGE:' in mailman_postorius_site_config_result.stdout
       check_mode: false
-      when: mailman_manage_postorius_site | bool
+      when:
+        - mailman_manage_postorius_site | bool
+        - >-
+          not ansible_check_mode or
+          mailman_postorius_site_config_helper_install is not changed
       tags: [mailman_postorius_site]
 
     - name: Install the hyperkitty configuration file