Merge pull request #1338 from lennart/feature/sys-upgrade-disable-eviction-flag

adds system_upgrade_enable_eviction flag

Author: mysticaltech

init.tf

@@ -218,7 +218,8 @@ resource "null_resource" "kustomization" {
     content = templatefile(
       "${path.module}/templates/plans.yaml.tpl",
       {
-        channel = var.initial_k3s_channel
+        channel          = var.initial_k3s_channel
+        disable_eviction = !var.system_upgrade_enable_eviction
     })
     destination = "/var/post_install/plans.yaml"
   }

kube.tf.example

@@ -556,6 +556,12 @@ module "kube-hetzner" {
   # The default is "true" (in HA setup i.e. at least 3 control plane nodes & 2 agents, just keep it enabled since it works flawlessly).
   # automatically_upgrade_k3s = false
 
+  # During k3s via system-upgrade-manager pods are evicted by default.
+  # On small clusters this can lead to hanging upgrades and indefinitely unschedulable nodes,
+  # in that case, set this to false to immediately delete pods before upgrading.
+  # NOTE: Turning this flag off might lead to downtimes of services (which may be acceptable for your use case)
+  # system_upgrade_enable_eviction = false
+
   # The default is "true" (in HA setup it works wonderfully well, with automatic roll-back to the previous snapshot in case of an issue).
   # IMPORTANT! For non-HA clusters i.e. when the number of control-plane nodes is < 3, you have to turn it off.
   # automatically_upgrade_os = false

templates/plans.yaml.tpl

@@ -26,6 +26,7 @@ spec:
     args: ["prepare", "k3s-server"]
   drain:
     force: true
+    disableEviction: ${disable_eviction}
     skipWaitForDeleteTimeout: 60
   upgrade:
     image: rancher/k3s-upgrade

variables.tf

@@ -508,6 +508,12 @@ variable "initial_k3s_channel" {
   }
 }
 
+variable "system_upgrade_enable_eviction" {
+  type        = bool
+  default     = true
+  description = "Whether to directly delete pods during system upgrade (k3s) or evict them. Defaults to true. Disable this on small clusters to avoid system upgrades hanging since pods resisting eviction keep node unschedulable forever. NOTE: turning this off, introduces potential downtime of services of the upgraded nodes."
+}
+
 variable "automatically_upgrade_k3s" {
   type        = bool
   default     = true