Increase nfiles in the systemd unit.

defaults/main.yml

@@ -10,4 +10,5 @@ prometheus_confdir: '/opt/prometheus/conf'
 prometheus_cmd: '{{ prometheus_dist_dir }}/{{ prometheus_dir }}/prometheus'
 prometheus_loglevel: info
 prometheus_http_port: 9090
-prometheus_opts: '--storage.tsdb.retention=120d'
+prometheus_opts: '--storage.tsdb.retention.time=120d'
+# --storage.tsdb.retention.size=512GB

templates/prometheus.service.j2

@@ -7,6 +7,8 @@ After=network.target
 Type=simple
 User={{ prometheus_user }}
 Group={{ prometheus_user }}
+LimitCORE=infinity
+LimitNOFILE=65536
 EnvironmentFile=/etc/default/prometheus
 ExecStart={{ prometheus_cmd }} $PROMETHEUS_STARTUP_OPTS
 ExecReload=/bin/kill -HUP $MAINPID