defaults/main.yml

@@ -149,36 +149,10 @@ nginx_robots_disallow_everything: false
 nginx_robots_enable_crawl_delay: false
 nginx_robots_crawl_delay: 10
 nginx_robots_disallowed_useragent_list:
-  - name: SemrushBot
-    uri: /
-  - name: SemrushBot-SA
-    uri: /
-  - name: Yandex
-    uri: /
-  - name: YandexBot
-    uri: /
-#   - name: ClaudeBot
-#     uri: /
-#   - name: Claude-Web
-#     uri: /
-#   - name: Bytedance
-#     uri: /
-#   - name: Bytespider
-#     uri: /
-#   - name: FacebookBot
-#     uri: /
-#   - name: ChatGPT-User
-#     uri: /
-#   - name: openai
-#     uri: /
-#   - name: openai.com
-#     uri: /
-#   - name: GPTBot
-#     uri: /
-#   - name: DotBot
-#     uri: /
-#   - name: PetalBot
-#     uri: /
+  - SemrushBot
+  - SemrushBot-SA
+  - Yandex
+  - YandexBot
 
 nginx_robots_disallowed_uris: false
 nginx_robots_disallowed_uris_list: []

tasks/basic-auth.yml

@@ -1,28 +1,25 @@
 ---
-- name: basic-auth | Manage the basic auth username and password on deb systems
-  when:
-    - ansible_distribution_file_variety == "Debian"
-    - ansible_distribution_version is version_compare('18.04', '>=')
-    - nginx_basic_auth
-  tags: nginx
-  block:
-    - name: basic-auth |Install the python passlib library on deb based distributions
+- block:
+    - name: Install the python passlib library on deb based distributions
+      apt: pkg=python-passlib state=present cache_valid_time=3600
+      when:
+        - ansible_distribution_file_variety == "Debian"
+        - ansible_distribution_version is version_compare('18.04', '<=')
+
+    - name: Install the python passlib library on deb based distributions
       apt: pkg=python3-passlib state=present cache_valid_time=3600
+      when:
+        - ansible_distribution_file_variety == "Debian"
+        - ansible_distribution_version is version_compare('18.04', '>=')
 
-    - name: basic-auth |Create the htpasswd file needed by the basic auth
-      htpasswd: path={{ item.file | default ('/etc/nginx/htpasswd') }} name={{ item.name }} password={{ item.pwd }} state={{ item.state | default('present') }} crypt_scheme={{ item.crypt | default('sha256_crypt') }}
-      with_items: '{{ nginx_basic_auth_users }}'
-
-- name: basic-auth | Manage the basic auth username and password on EL systems
-  when:
-    - ansible_distribution_file_variety == "RedHat"
-    - nginx_basic_auth
-  tags: nginx
-  block:
-    - name: basic-auth |Install the python passlib library on RH based distributions
+    - name: Install the python passlib library on RH based distributions
       yum: pkg=python-passlib state=present
+      when: ansible_distribution_file_variety == "RedHat"
 
-    - name: basic-auth |Create the htpasswd file needed by the basic auth
+    - name: Create the htpasswd file needed by the basic auth
       htpasswd: path={{ item.file | default ('/etc/nginx/htpasswd') }} name={{ item.name }} password={{ item.pwd }} state={{ item.state | default('present') }} crypt_scheme={{ item.crypt | default('sha256_crypt') }}
       with_items: '{{ nginx_basic_auth_users }}'
 
+  when: nginx_basic_auth
+  tags: nginx
+

templates/nginx-cors.conf.j2

@@ -1,10 +1,10 @@
 {% if nginx_cors_extended_rules %}
 if ($request_method = 'OPTIONS') {
 {% if nginx_cors_limit_origin %}
-#     proxy_hide_header Access-Control-Allow-Origin;
+    proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_cors_acl_origin | default("$http_origin") }}';
 {% else %}
-#     proxy_hide_header Access-Control-Allow-Origin;
+    proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_access_control_allow_origin_src | default("*") }}';
 {% endif %}
     add_header 'Access-Control-Allow-Credentials' 'true';
@@ -23,10 +23,10 @@ if ($request_method = 'OPTIONS') {
 }
 if ($request_method = 'POST') {
 {% if nginx_cors_limit_origin %}
-#     proxy_hide_header Access-Control-Allow-Origin;
+    proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_cors_acl_origin | default("$http_origin") }}';
 {% else %}
-#     proxy_hide_header Access-Control-Allow-Origin;
+    proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_access_control_allow_origin_src | default("*") }}';
 {% endif %}
     add_header 'Access-Control-Allow-Credentials' 'true';
@@ -36,10 +36,10 @@ if ($request_method = 'POST') {
 }
 if ($request_method = 'GET') {
 {% if nginx_cors_limit_origin %}
-#     proxy_hide_header Access-Control-Allow-Origin;
+    proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_cors_acl_origin | default("$http_origin") }}';
 {% else %}
-#     proxy_hide_header Access-Control-Allow-Origin;
+    proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_access_control_allow_origin_src | default("*") }}';
 {% endif %}
     add_header 'Access-Control-Allow-Credentials' 'true';

templates/nginx-virthost.j2

@@ -101,9 +101,6 @@ server {
         access_log off;
     }
     location = /robots.txt {
-{% if nginx_install_robots_txt %}
-        root {{ nginx_webroot }};
-{% endif %}
         allow all;
         log_not_found off;
         access_log off;
@@ -251,9 +248,6 @@ server {
         access_log off;
     }
     location = /robots.txt {
-{% if nginx_install_robots_txt %}
-        root {{ nginx_webroot }};
-{% endif %}
         allow all;
         log_not_found off;
         access_log off;

templates/robots.txt.j2

@@ -3,11 +3,10 @@ User-Agent: *
 Disallow: /
 {% else %}
 {% for ua in nginx_robots_disallowed_useragent_list %}
-User-agent: {{ ua.name }}
-Disallow: {{ ua.uri | default('/') }}
+User-agent: {{ ua }}
+Disallow: /
 
 {% endfor %}
-
 {% if nginx_robots_disallowed_uris %}
 User-agent: *
 {% for uri in nginx_robots_disallowed_uris_list %}
@@ -19,5 +18,4 @@ Disallow: {{ uri }}
 User-Agent: *
 Crawl-Delay: {{ nginx_robots_crawl_delay }}
 {% endif %}
-{% endif %}
-
+{% endif %}