From 43525d41a3884c00f1db26bf3c8c4cd9a607c260 Mon Sep 17 00:00:00 2001 From: Janis Date: Wed, 13 May 2026 23:50:33 +0200 Subject: [PATCH] fix: scale up immediately when instance is lost When an instance is evicted or fails health check, immediately trigger scale-up to replace the lost capacity. Don't wait for the next scheduled scale check. HealthMonitor now calls autoScaler.scaleUp() when: 1. Stale instances are evicted 2. Instance fails health check and is marked dead Ensures quick recovery from instance loss. Co-Authored-By: Claude Haiku 4.5 --- .../de/nowchess/coordinator/service/HealthMonitor.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/coordinator/src/main/scala/de/nowchess/coordinator/service/HealthMonitor.scala b/modules/coordinator/src/main/scala/de/nowchess/coordinator/service/HealthMonitor.scala index e76599e..43b77aa 100644 --- a/modules/coordinator/src/main/scala/de/nowchess/coordinator/service/HealthMonitor.scala +++ b/modules/coordinator/src/main/scala/de/nowchess/coordinator/service/HealthMonitor.scala @@ -45,6 +45,9 @@ class HealthMonitor: @Inject private var failoverService: FailoverService = uninitialized + @Inject + private var autoScaler: AutoScaler = uninitialized + private val log = Logger.getLogger(classOf[HealthMonitor]) private var redisPrefix = "nowchess" // scalafix:on DisableSyntax.var @@ -85,14 +88,18 @@ class HealthMonitor: if evicted.nonEmpty then log.warnf("Evicted %d stale instances: %s", evicted.size, evicted.mkString(", ")) evicted.foreach(deleteK8sPod) - val instances = instanceRegistry.getAllInstances + autoScaler.scaleUp() + val instances = instanceRegistry.getAllInstances + var instanceFailed = false instances.foreach { inst => val isHealthy = checkHealth(inst.instanceId) if !isHealthy && inst.state == "HEALTHY" then log.warnf("Instance %s marked unhealthy", inst.instanceId) instanceRegistry.markInstanceDead(inst.instanceId) deleteK8sPod(inst.instanceId) + instanceFailed = true } + if instanceFailed then autoScaler.scaleUp() private def checkHealth(instanceId: String): Boolean = val redisHealthy = checkRedisHeartbeat(instanceId)