Remove some not allowed proxy_hide_header.

Reviewed-on: #3

defaults/main.yml

@@ -149,10 +149,36 @@ nginx_robots_disallow_everything: false
 nginx_robots_enable_crawl_delay: false
 nginx_robots_crawl_delay: 10
 nginx_robots_disallowed_useragent_list:
-  - SemrushBot
+  - name: SemrushBot
-  - SemrushBot-SA
+    uri: /
-  - Yandex
+  - name: SemrushBot-SA
-  - YandexBot
+    uri: /
+  - name: Yandex
+    uri: /
+  - name: YandexBot
+    uri: /
+#   - name: ClaudeBot
+#     uri: /
+#   - name: Claude-Web
+#     uri: /
+#   - name: Bytedance
+#     uri: /
+#   - name: Bytespider
+#     uri: /
+#   - name: FacebookBot
+#     uri: /
+#   - name: ChatGPT-User
+#     uri: /
+#   - name: openai
+#     uri: /
+#   - name: openai.com
+#     uri: /
+#   - name: GPTBot
+#     uri: /
+#   - name: DotBot
+#     uri: /
+#   - name: PetalBot
+#     uri: /
 
 nginx_robots_disallowed_uris: false
 nginx_robots_disallowed_uris_list: []

templates/nginx-cors.conf.j2

@@ -1,10 +1,10 @@
 {% if nginx_cors_extended_rules %}
 if ($request_method = 'OPTIONS') {
 {% if nginx_cors_limit_origin %}
-    proxy_hide_header Access-Control-Allow-Origin;
+#     proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_cors_acl_origin | default("$http_origin") }}';
 {% else %}
-    proxy_hide_header Access-Control-Allow-Origin;
+#     proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_access_control_allow_origin_src | default("*") }}';
 {% endif %}
     add_header 'Access-Control-Allow-Credentials' 'true';
@@ -23,10 +23,10 @@ if ($request_method = 'OPTIONS') {
 }
 if ($request_method = 'POST') {
 {% if nginx_cors_limit_origin %}
-    proxy_hide_header Access-Control-Allow-Origin;
+#     proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_cors_acl_origin | default("$http_origin") }}';
 {% else %}
-    proxy_hide_header Access-Control-Allow-Origin;
+#     proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_access_control_allow_origin_src | default("*") }}';
 {% endif %}
     add_header 'Access-Control-Allow-Credentials' 'true';
@@ -36,10 +36,10 @@ if ($request_method = 'POST') {
 }
 if ($request_method = 'GET') {
 {% if nginx_cors_limit_origin %}
-    proxy_hide_header Access-Control-Allow-Origin;
+#     proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_cors_acl_origin | default("$http_origin") }}';
 {% else %}
-    proxy_hide_header Access-Control-Allow-Origin;
+#     proxy_hide_header Access-Control-Allow-Origin;
     add_header 'Access-Control-Allow-Origin' '{{ nginx_access_control_allow_origin_src | default("*") }}';
 {% endif %}
     add_header 'Access-Control-Allow-Credentials' 'true';

templates/nginx-virthost.j2

@@ -101,6 +101,9 @@ server {
         access_log off;
     }
     location = /robots.txt {
+{% if nginx_install_robots_txt %}
+        root {{ nginx_webroot }};
+{% endif %}
         allow all;
         log_not_found off;
         access_log off;
@@ -248,6 +251,9 @@ server {
         access_log off;
     }
     location = /robots.txt {
+{% if nginx_install_robots_txt %}
+        root {{ nginx_webroot }};
+{% endif %}
         allow all;
         log_not_found off;
         access_log off;

templates/robots.txt.j2

@@ -3,10 +3,11 @@ User-Agent: *
 Disallow: /
 {% else %}
 {% for ua in nginx_robots_disallowed_useragent_list %}
-User-agent: {{ ua }}
+User-agent: {{ ua.name }}
-Disallow: /
+Disallow: {{ ua.uri | default('/') }}
 
 {% endfor %}
+
 {% if nginx_robots_disallowed_uris %}
 User-agent: *
 {% for uri in nginx_robots_disallowed_uris_list %}
@@ -18,4 +19,5 @@ Disallow: {{ uri }}
 User-Agent: *
 Crawl-Delay: {{ nginx_robots_crawl_delay }}
 {% endif %}
-{% endif %}
+{% endif %}
+