openstack-infrastructure-terraform/s2i2s/project-setup/haproxy.tf

#
# HAPROXY L7 behind the main Octavia load balancer
#

# Server group for anti-affinity (VMs on different hosts)
resource "openstack_compute_servergroup_v2" "main_haproxy_l7" {
  name     = "main_haproxy_l7"
  policies = ["anti-affinity"]
}

# Security group for traffic from the Octavia LB to HAProxy.
# The main LB uses the OVN provider, not amphora: OVN does not SNAT the traffic to
# the amphora VIP, so the HAProxy L7 servers see the original client source IP.
# Only the health monitor probes and the hairpinned traffic come from the private
# subnet. Therefore the public listeners (80, 443) must accept any source.
resource "openstack_networking_secgroup_v2" "main_lb_to_haproxy_l7" {
  name                 = "traffic_from_main_lb_to_haproxy_l7"
  delete_default_rules = "true"
  description          = "Traffic coming from the main L4 lb (OVN provider, client IP is preserved) directed to the haproxy l7 servers"
}

resource "openstack_networking_secgroup_rule_v2" "haproxy_l7_1_peer" {
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "Peer traffic from haproxy l7 1 to l7 2"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 10000
  port_range_max    = 10000
  remote_ip_prefix  = local.basic_services_ip.haproxy_l7_1_cidr
}

resource "openstack_networking_secgroup_rule_v2" "haproxy_l7_2_peer" {
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "Peer traffic from haproxy l7 2 to l7 1"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 10000
  port_range_max    = 10000
  remote_ip_prefix  = local.basic_services_ip.haproxy_l7_2_cidr
}

resource "openstack_networking_secgroup_rule_v2" "octavia_to_haproxy_l7_80" {
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "HTTP traffic to HAPROXY l7 port 80, through the OVN based main lb. The source is the client IP"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 80
  port_range_max    = 80
  remote_ip_prefix  = "0.0.0.0/0"
}

resource "openstack_networking_secgroup_rule_v2" "octavia_to_haproxy_l7_443" {
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "HTTPS traffic to HAPROXY l7 port 443, through the OVN based main lb. The source is the client IP"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 443
  port_range_max    = 443
  remote_ip_prefix  = "0.0.0.0/0"
}

# Git over SSH: the traffic arrives from the main load balancer, that keeps the
# public port 22, and reaches HAPROXY on this port. Like 80 and 443, the source
# is the real client address, so the rule cannot be restricted
resource "openstack_networking_secgroup_rule_v2" "octavia_to_haproxy_l7_git_ssh" {
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "Git over SSH, through the OVN based main lb. The source is the client IP"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = local.git_ssh_data.haproxy_bind_port
  port_range_max    = local.git_ssh_data.haproxy_bind_port
  remote_ip_prefix  = "0.0.0.0/0"
}

# HAPROXY stats, port 8880.
# The OVN provider does not support allowed_cidrs on the listener, so this security
# group is the only place where the stats can be restricted. Since OVN preserves the
# client source IP, we can filter on the real client addresses instead of opening the
# port to everybody. The private subnet is still needed for the health monitor probes.
resource "openstack_networking_secgroup_rule_v2" "octavia_to_haproxy_l7_8880_healthcheck" {
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "Health monitor probes to the HAPROXY l7 stats port 8880"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8880
  port_range_max    = 8880
  remote_ip_prefix  = local.main_private_subnet.cidr
}

resource "openstack_networking_secgroup_rule_v2" "haproxy_l7_8880_stats_sources" {
  for_each = {
    d4s_vpn_1        = local.ssh_sources.d4s_vpn_1_cidr
    d4s_vpn_2        = local.ssh_sources.d4s_vpn_2_cidr
    s2i2s_vpn_1      = local.ssh_sources.s2i2s_vpn_1_cidr
    s2i2s_vpn_2      = local.ssh_sources.s2i2s_vpn_2_cidr
    infrascience_net = local.ssh_sources.infrascience_net_cidr
    # Prometheus scrapes the HAPROXY stats: it reaches them over the private network,
    # and through the load balancer public IP via its own floating IP
    prometheus_private = local.basic_services_ip.prometheus_cidr
    prometheus_public  = "${openstack_networking_floatingip_v2.prometheus_server_ip.address}/32"
  }
  security_group_id = openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  description       = "HAPROXY l7 stats port 8880 from ${each.key}"
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8880
  port_range_max    = 8880
  remote_ip_prefix  = each.value
}

# Ports in the main private network for HAProxy instances
resource "openstack_networking_port_v2" "main_haproxy_l7_port" {
  count          = local.haproxy_l7_data.vm_count
  name           = format("%s-%02d-port", local.haproxy_l7_data.name, count.index + 1)
  admin_state_up = true
  network_id     = local.main_private_network_id
  security_group_ids = [
    openstack_networking_secgroup_v2.default.id,
    openstack_networking_secgroup_v2.main_lb_to_haproxy_l7.id
  ]
  fixed_ip {
    subnet_id  = local.main_private_subnet_id
    ip_address = local.main_haproxy_l7_ip[count.index]
  }
}

# HAProxy L7 instances
resource "openstack_compute_instance_v2" "main_haproxy_l7" {
  count                   = local.haproxy_l7_data.vm_count
  name                    = format("%s-%02d", local.haproxy_l7_data.name, count.index + 1)
  availability_zone_hints = local.availability_zones_names.availability_zone_no_gpu
  flavor_name             = local.haproxy_l7_data.flavor
  key_pair                = module.ssh_settings.ssh_key_name
  scheduler_hints {
    group = openstack_compute_servergroup_v2.main_haproxy_l7.id
  }

  block_device {
    uuid                  = local.ubuntu_2404.uuid
    source_type           = "image"
    volume_size           = 10
    boot_index            = 0
    destination_type      = "volume"
    delete_on_termination = false
  }

  network {
    port = openstack_networking_port_v2.main_haproxy_l7_port[count.index].id
  }

  user_data = file("${local.ubuntu2404_data_file}")
  # Do not replace the instance when the ssh key changes
  lifecycle {
    ignore_changes = [
      key_pair, user_data, network
    ]
  }
}