feat: implement periodic scaling checks and enhance instance management in AutoScaler

modules/coordinator/src/main/scala/de/nowchess/coordinator/service/AutoScaler.scala

@@ -8,6 +8,7 @@ import de.nowchess.coordinator.config.CoordinatorConfig
 import io.fabric8.kubernetes.api.model.GenericKubernetesResource
 import io.fabric8.kubernetes.client.KubernetesClient
 import io.micrometer.core.instrument.{Gauge, MeterRegistry}
+import io.quarkus.scheduler.Scheduled
 import org.jboss.logging.Logger
 
 import java.util.concurrent.atomic.AtomicReference
@@ -25,6 +26,12 @@ class AutoScaler:
   @Inject
   private var instanceRegistry: InstanceRegistry = uninitialized
 
+  @Inject
+  private var loadBalancer: LoadBalancer = uninitialized
+
+  @Inject
+  private var failoverService: FailoverService = uninitialized
+
   @Inject
   private var meterRegistry: MeterRegistry = uninitialized
   // scalafix:on DisableSyntax.var
@@ -51,6 +58,11 @@ class AutoScaler:
     meterRegistry.counter("nowchess.coordinator.scale.failures", "direction", "down").increment(0)
     ()
 
+  @Scheduled(every = "10s")
+  def periodicScaleCheck(): Unit =
+    try checkAndScale
+    catch case ex: Exception => log.warnf(ex, "Auto-scale check failed")
+
   // scalafix:off DisableSyntax.asInstanceOf
   private def rolloutSpec(rollout: GenericKubernetesResource): Option[java.util.Map[String, AnyRef]] =
     Option(rollout.get[AnyRef]("spec")).collect { case m: java.util.Map[?, ?] =>
@@ -105,6 +117,7 @@ class AutoScaler:
                       currentReplicas,
                       currentReplicas + 1,
                     )
+                    loadBalancer.rebalance
                   else log.infof("Already at max replicas %d for %s", maxReplicas, config.k8sRolloutName)
                 case _ => ()
             }
@@ -116,6 +129,17 @@ class AutoScaler:
 
   def scaleDown(): Unit =
     log.info("Scaling down Argo Rollout")
+    val underloadedInstance = instanceRegistry.getAllInstances
+      .filter(_.state == "HEALTHY")
+      .minByOption(_.subscriptionCount)
+
+    underloadedInstance.foreach { inst =>
+      log.infof("Draining instance %s before scale-down", inst.instanceId)
+      failoverService
+        .onInstanceStreamDropped(inst.instanceId)
+        .subscribe()
+        .`with`(
+          _ =>
             kubeClientOpt match
               case None =>
                 log.warn("Kubernetes client not available, cannot scale")
@@ -155,4 +179,11 @@ class AutoScaler:
                 catch
                   case ex: Exception =>
                     meterRegistry.counter("nowchess.coordinator.scale.failures", "direction", "down").increment()
-            log.warnf(ex, "Failed to scale down %s", config.k8sRolloutName)
+                    log.warnf(ex, "Failed to scale down %s", config.k8sRolloutName),
+          ex =>
+            meterRegistry.counter("nowchess.coordinator.scale.failures", "direction", "down").increment()
+            log.warnf(ex, "Failed to drain instance %s before scale-down", inst.instanceId),
+        )
+    }
+
+    if underloadedInstance.isEmpty then log.warn("No healthy instances found for scale-down")

modules/coordinator/src/main/scala/de/nowchess/coordinator/service/InstanceRegistry.scala

@@ -76,6 +76,10 @@ class InstanceRegistry:
       .onItem()
       .transformToUni { value =>
         try
+          if value == null then
+            log.debugf("Instance %s metadata missing from Redis (may have expired)", instanceId)
+            Uni.createFrom().item(())
+          else
             val metadata = mapper.readValue(value, classOf[InstanceMetadata])
             val isNew    = !instances.containsKey(instanceId)
             instances.put(instanceId, metadata)

modules/core/src/main/scala/de/nowchess/chess/engine/GameEngine.scala

@@ -60,8 +60,8 @@ class GameEngine(
   @SuppressWarnings(Array("DisableSyntax.var"))
   private var pendingTakebackRequest: Option[Color] = initialTakebackRequest
 
-  meterRegistry.foreach { reg =>
   GameEngine.activeGamesCount.incrementAndGet()
+  meterRegistry.foreach { reg =>
     reg.counter("nowchess.games.started").increment()
   }
   private def gamesCompletedCounter(result: String): Counter =

modules/core/src/main/scala/de/nowchess/chess/service/InstanceHeartbeatService.scala

@@ -199,7 +199,7 @@ class InstanceHeartbeatService:
         val json = mapper.writeValueAsString(metadata)
         reactiveRedis
           .value(classOf[String])
-          .setex(key, 5L, json)
+          .setex(key, 15L, json)
           .subscribe()
           .`with`(
             _ => redisHeartbeatPending.set(false),