Upcloud: Add possibility to setup cluster using nodes with no public IPs (#11696)

* terraform upcloud: Added possibility to set up nodes with only private IPs

* terraform upcloud: add support for gateway in private zone

* terraform upcloud: split LB proxy protocol config per backend

* terraform upcloud: fix flexible plans

* terraform upcloud: Removed overview of cluster setup

---------

Co-authored-by: davidumea <david.andersson@elastisys.com>

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -53,6 +53,44 @@ locals {
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
+
+  master_ip = {
+    for instance in upcloud_server.master :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+  worker_ip = {
+    for instance in upcloud_server.worker :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+
+  bastion_ip = {
+    for instance in upcloud_server.bastion :
+      instance.hostname => {
+        for nic in instance.network_interface :
+          nic.type => nic.ip_address
+        if nic.ip_address != null
+      }
+  }
+
+  node_user_data = {
+    for name, machine in var.machines :
+      name => <<EOF
+%{ if ( length(machine.dns_servers != null ? machine.dns_servers : [] ) > 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}
+#!/bin/bash
+echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf
+
+systemctl restart systemd-resolved
+%{ endif ~}
+EOF
+  }
 }
 
 resource "upcloud_network" "private" {
@@ -62,6 +100,9 @@ resource "upcloud_network" "private" {
   ip_network {
     address            = var.private_network_cidr
     dhcp_default_route = var.router_enable
+    # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.
+    #       See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562
+    # dhcp_dns           = length(var.private_network_dns) > 0 ? var.private_network_dns : null
     dhcp               = true
     family             = "IPv4"
   }
@@ -89,8 +130,8 @@ resource "upcloud_server" "master" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
+  cpu          = each.value.cpu
+  mem          = each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -99,9 +140,12 @@ resource "upcloud_server" "master" {
     size    = each.value.disk_size
   }
 
-  # Public network interface
-  network_interface {
-    type = "public"
+  dynamic "network_interface" {
+    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
+
+    content {
+      type = "public"
+    }
   }
 
   # Private network interface
@@ -136,6 +180,9 @@ resource "upcloud_server" "master" {
     keys            = var.ssh_public_keys
     create_password = false
   }
+
+  metadata = local.node_user_data[each.key] != "" ? true : null
+  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
 }
 
 resource "upcloud_server" "worker" {
@@ -147,8 +194,8 @@ resource "upcloud_server" "worker" {
 
   hostname     = "${local.resource-prefix}${each.key}"
   plan         = each.value.plan
-  cpu          = each.value.plan == null ? null : each.value.cpu
-  mem          = each.value.plan == null ? null : each.value.mem
+  cpu          = each.value.cpu
+  mem          = each.value.mem
   zone         = var.zone
   server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
 
@@ -158,9 +205,12 @@ resource "upcloud_server" "worker" {
     size    = each.value.disk_size
   }
 
-  # Public network interface
-  network_interface {
-    type = "public"
+  dynamic "network_interface" {
+    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
+
+    content {
+      type = "public"
+    }
   }
 
   # Private network interface
@@ -195,6 +245,63 @@ resource "upcloud_server" "worker" {
     keys            = var.ssh_public_keys
     create_password = false
   }
+
+  metadata = local.node_user_data[each.key] != "" ? true : null
+  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
+}
+
+resource "upcloud_server" "bastion" {
+  for_each = {
+    for name, machine in var.machines :
+    name => machine
+    if machine.node_type == "bastion"
+  }
+
+  hostname     = "${local.resource-prefix}${each.key}"
+  plan         = each.value.plan
+  cpu          = each.value.cpu
+  mem          = each.value.mem
+  zone         = var.zone
+  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
+
+
+  template {
+    storage = var.template_name
+    size    = each.value.disk_size
+  }
+
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
+
+  # Private network interface
+  network_interface {
+    type    = "public"
+  }
+
+  firewall = var.firewall_enabled
+
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+      disk_key_name => disk
+      # Only add the disk if it matches the node name in the start of its name
+      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
+
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
 
 resource "upcloud_firewall_rules" "master" {
@@ -543,6 +650,53 @@ resource "upcloud_firewall_rules" "k8s" {
   }
 }
 
+resource "upcloud_firewall_rules" "bastion" {
+  for_each  = upcloud_server.bastion
+  server_id = each.value.id
+
+  dynamic "firewall_rule" {
+    for_each = var.bastion_allowed_remote_ips
+
+    content {
+      action                 = "accept"
+      comment                = "Allow bastion SSH access from this network"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic "firewall_rule" {
+    for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []
+
+    content {
+      action                 = "drop"
+      comment                = "Drop bastion SSH access from other networks"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_in ? "drop" : "accept"
+    direction = "in"
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_out ? "drop" : "accept"
+    direction = "out"
+  }
+}
+
 resource "upcloud_loadbalancer" "lb" {
   count             = var.loadbalancer_enabled ? 1 : 0
   configured_status = "started"
@@ -583,7 +737,7 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
   loadbalancer = upcloud_loadbalancer.lb[0].id
   name         = "lb-backend-${each.key}"
   properties {
-    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
+    outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : ""
   }
 }
 
@@ -622,7 +776,7 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
 
   backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
   name         = "${local.resource-prefix}${each.key}"
-  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
+  ip           = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private
   port         = each.value.port
   weight       = 100
   max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
@@ -662,7 +816,7 @@ resource "upcloud_router" "router" {
 resource "upcloud_gateway" "gateway" {
   for_each = var.router_enable ? var.gateways : {}
   name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.zone
+  zone = var.private_cloud ? var.public_zone : var.zone
 
   features = each.value.features
   plan = each.value.plan

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -1,22 +1,13 @@
-
 output "master_ip" {
-  value = {
-    for instance in upcloud_server.master :
-    instance.hostname => {
-      "public_ip" : instance.network_interface[0].ip_address
-      "private_ip" : instance.network_interface[1].ip_address
-    }
-  }
+  value = local.master_ip
 }
 
 output "worker_ip" {
-  value = {
-    for instance in upcloud_server.worker :
-    instance.hostname => {
-      "public_ip" : instance.network_interface[0].ip_address
-      "private_ip" : instance.network_interface[1].ip_address
-    }
-  }
+  value = local.worker_ip
+}
+
+output "bastion_ip" {
+  value = local.bastion_ip
 }
 
 output "loadbalancer_domain" {

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -20,15 +20,21 @@ variable "username" {}
 
 variable "private_network_cidr" {}
 
+variable "dns_servers" {}
+
+variable "use_public_ips" {}
+
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type = string
     plan      = string
-    cpu       = string
-    mem       = string
+    cpu       = optional(number)
+    mem       = optional(number)
     disk_size = number
     server_group : string
+    force_public_ip : optional(bool, false)
+    dns_servers : optional(set(string))
     additional_disks = map(object({
       size = number
       tier = string
@@ -58,6 +64,13 @@ variable "k8s_allowed_remote_ips" {
   }))
 }
 
+variable "bastion_allowed_remote_ips" {
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+}
+
 variable "master_allowed_ports" {
   type = list(object({
     protocol       = string
@@ -94,10 +107,6 @@ variable "loadbalancer_plan" {
   type = string
 }
 
-variable "loadbalancer_outbound_proxy_protocol" {
-  type = string
-}
-
 variable "loadbalancer_legacy_network" {
   type = bool
   default = false
@@ -107,6 +116,7 @@ variable "loadbalancers" {
   description = "Load balancers"
 
   type = map(object({
+    proxy_protocol          = bool
     port                    = number
     target_port             = number
     allow_internal_frontend = optional(bool)