feat(coordinator): add configurable coordinator settings and enhance WebSocket connection handling
Build & Test (NowChessSystems) TeamCity build failed

This commit is contained in:
2026-04-26 22:33:16 +02:00
parent 64d5afa4d1
commit 6b59e68e04
27 changed files with 873 additions and 617 deletions
@@ -0,0 +1,27 @@
{
"reflection": [
{ "type": "scala.Tuple1[]" },
{ "type": "scala.Tuple2[]" },
{ "type": "scala.Tuple3[]" },
{ "type": "scala.Tuple4[]" },
{ "type": "scala.Tuple5[]" },
{ "type": "scala.Tuple6[]" },
{ "type": "scala.Tuple7[]" },
{ "type": "scala.Tuple8[]" },
{ "type": "scala.Tuple9[]" },
{ "type": "scala.Tuple10[]" },
{ "type": "scala.Tuple11[]" },
{ "type": "scala.Tuple12[]" },
{ "type": "scala.Tuple13[]" },
{ "type": "scala.Tuple14[]" },
{ "type": "scala.Tuple15[]" },
{ "type": "scala.Tuple16[]" },
{ "type": "scala.Tuple17[]" },
{ "type": "scala.Tuple18[]" },
{ "type": "scala.Tuple19[]" },
{ "type": "scala.Tuple20[]" },
{ "type": "scala.Tuple21[]" },
{ "type": "scala.Tuple22[]" },
{ "type": "com.fasterxml.jackson.module.scala.introspect.PropertyDescriptor[]" }
]
}
+3
View File
@@ -63,12 +63,15 @@ dependencies {
implementation(enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}"))
implementation("io.quarkus:quarkus-rest")
implementation("io.quarkus:quarkus-rest-jackson")
implementation("io.quarkus:quarkus-grpc")
implementation("io.quarkus:quarkus-arc")
implementation("io.quarkus:quarkus-config-yaml")
implementation("io.quarkus:quarkus-smallrye-health")
implementation("io.quarkus:quarkus-smallrye-openapi")
implementation("io.quarkus:quarkus-rest-client")
implementation("io.quarkus:quarkus-rest-client-jackson")
implementation("com.fasterxml.jackson.module:jackson-module-scala_3:${versions["JACKSON_SCALA"]!!}")
implementation("org.redisson:redisson:${versions["REDISSON"]!!}")
implementation("io.fabric8:kubernetes-client:6.13.0")
@@ -0,0 +1,27 @@
{
"reflection": [
{ "type": "scala.Tuple1[]" },
{ "type": "scala.Tuple2[]" },
{ "type": "scala.Tuple3[]" },
{ "type": "scala.Tuple4[]" },
{ "type": "scala.Tuple5[]" },
{ "type": "scala.Tuple6[]" },
{ "type": "scala.Tuple7[]" },
{ "type": "scala.Tuple8[]" },
{ "type": "scala.Tuple9[]" },
{ "type": "scala.Tuple10[]" },
{ "type": "scala.Tuple11[]" },
{ "type": "scala.Tuple12[]" },
{ "type": "scala.Tuple13[]" },
{ "type": "scala.Tuple14[]" },
{ "type": "scala.Tuple15[]" },
{ "type": "scala.Tuple16[]" },
{ "type": "scala.Tuple17[]" },
{ "type": "scala.Tuple18[]" },
{ "type": "scala.Tuple19[]" },
{ "type": "scala.Tuple20[]" },
{ "type": "scala.Tuple21[]" },
{ "type": "scala.Tuple22[]" },
{ "type": "com.fasterxml.jackson.module.scala.introspect.PropertyDescriptor[]" }
]
}
@@ -1,10 +1,22 @@
quarkus:
application.name: nowchess-coordinator
http.port: 8086
grpc.server.port: 9086
config.yaml.enabled: true
rest-client.connection-timeout: 5000
rest-client.read-timeout: 10000
application:
name: nowchess-coordinator
http:
port: 8086
grpc:
server:
port: 9086
rest-client:
connection-timeout: 5000
read-timeout: 10000
smallrye-openapi:
info-title: NowChess Coordinator Service
info-version: 1.0.0
info-description: Coordination endpoints for instance health, balancing, failover, and scaling
path: /openapi
swagger-ui:
always-include: true
path: /swagger-ui
nowchess:
redis:
@@ -18,7 +30,7 @@ nowchess:
rebalance-interval: 30s
rebalance-min-interval: 60s
heartbeat-ttl: 5s
stream-heartbeat-interval: 200ms
stream-heartbeat-interval: PT0.2S
cache-eviction-interval: 10m
game-idle-threshold: 45m
auto-scale-enabled: false
@@ -34,6 +46,5 @@ nowchess:
# dev profile
"%dev":
quarkus:
log.level: DEBUG
log.category:
"de.nowchess": DEBUG
log:
level: DEBUG
@@ -5,51 +5,51 @@ import io.smallrye.config.WithName
import java.time.Duration
@ConfigMapping(prefix = "nowchess.coordinator")
class CoordinatorConfig:
trait CoordinatorConfig:
@WithName("max-games-per-core")
def maxGamesPerCore: Int = ???
def maxGamesPerCore: Int
@WithName("max-deviation-percent")
def maxDeviationPercent: Int = ???
def maxDeviationPercent: Int
@WithName("rebalance-interval")
def rebalanceInterval: Duration = ???
def rebalanceInterval: Duration
@WithName("rebalance-min-interval")
def rebalanceMinInterval: Duration = ???
def rebalanceMinInterval: Duration
@WithName("heartbeat-ttl")
def heartbeatTtl: Duration = ???
def heartbeatTtl: Duration
@WithName("stream-heartbeat-interval")
def streamHeartbeatInterval: Duration = ???
def streamHeartbeatInterval: Duration
@WithName("cache-eviction-interval")
def cacheEvictionInterval: Duration = ???
def cacheEvictionInterval: Duration
@WithName("game-idle-threshold")
def gameIdleThreshold: Duration = ???
def gameIdleThreshold: Duration
@WithName("auto-scale-enabled")
def autoScaleEnabled: Boolean = ???
def autoScaleEnabled: Boolean
@WithName("scale-up-threshold")
def scaleUpThreshold: Double = ???
def scaleUpThreshold: Double
@WithName("scale-down-threshold")
def scaleDownThreshold: Double = ???
def scaleDownThreshold: Double
@WithName("scale-min-replicas")
def scaleMinReplicas: Int = ???
def scaleMinReplicas: Int
@WithName("scale-max-replicas")
def scaleMaxReplicas: Int = ???
def scaleMaxReplicas: Int
@WithName("k8s-namespace")
def k8sNamespace: String = ???
def k8sNamespace: String
@WithName("k8s-rollout-name")
def k8sRolloutName: String = ???
def k8sRolloutName: String
@WithName("k8s-rollout-label-selector")
def k8sRolloutLabelSelector: String = ???
def k8sRolloutLabelSelector: String
@@ -0,0 +1,18 @@
package de.nowchess.coordinator.config
import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.quarkus.jackson.ObjectMapperCustomizer
import jakarta.inject.Singleton
@Singleton
class JacksonConfig extends ObjectMapperCustomizer:
def customize(mapper: ObjectMapper): Unit =
mapper.registerModule(new DefaultScalaModule() {
override def version(): Version =
// scalafix:off DisableSyntax.null
new Version(2, 21, 1, null, "com.fasterxml.jackson.module", "jackson-module-scala")
// scalafix:on DisableSyntax.null
})
@@ -0,0 +1,14 @@
package de.nowchess.coordinator.config
import de.nowchess.coordinator.dto.InstanceMetadata
import de.nowchess.coordinator.resource.MetricsDto
import io.quarkus.runtime.annotations.RegisterForReflection
@RegisterForReflection(
targets = Array(
classOf[InstanceMetadata],
classOf[MetricsDto],
),
)
class NativeReflectionConfig
@@ -24,6 +24,7 @@ nowchess:
prefix: nowchess
coordinator:
enabled: ${NOWCHESS_COORDINATOR_ENABLED:false}
host: localhost
grpc-port: 9086
stream-heartbeat-interval: 200ms
@@ -99,6 +100,7 @@ nowchess:
prefix: ${REDIS_PREFIX:nowchess}
coordinator:
enabled: ${NOWCHESS_COORDINATOR_ENABLED:true}
host: ${COORDINATOR_SERVICE_HOST:localhost}
grpc-port: ${COORDINATOR_SERVICE_GRPC_PORT:9086}
stream-heartbeat-interval: 200ms
@@ -3,6 +3,6 @@ package de.nowchess.chess.redis
sealed trait C2sMessage
object C2sMessage:
case object Connected extends C2sMessage
case class Move(uci: String) extends C2sMessage
case object Ping extends C2sMessage
case object Connected extends C2sMessage
case class Move(uci: String, playerId: Option[String] = None) extends C2sMessage
case object Ping extends C2sMessage
@@ -1,7 +1,9 @@
package de.nowchess.chess.redis
import com.fasterxml.jackson.databind.ObjectMapper
import de.nowchess.api.board.Color
import de.nowchess.api.dto.GameFullEventDto
import de.nowchess.api.game.GameMode
import de.nowchess.chess.config.RedisConfig
import de.nowchess.chess.grpc.IoGrpcClientWrapper
import de.nowchess.chess.observer.Observer
@@ -82,10 +84,10 @@ class GameRedisSubscriberManager:
private def handleC2sMessage(gameId: String, msg: String): Unit =
parseC2sMessage(msg) match
case Some(C2sMessage.Connected) => handleConnected(gameId)
case Some(C2sMessage.Move(uci)) => handleMove(gameId, uci)
case Some(C2sMessage.Ping) => ()
case None => ()
case Some(C2sMessage.Connected) => handleConnected(gameId)
case Some(C2sMessage.Move(uci, playerId)) => handleMove(gameId, uci, playerId)
case Some(C2sMessage.Ping) => ()
case None => ()
private def handleConnected(gameId: String): Unit =
registry.get(gameId).foreach { entry =>
@@ -94,16 +96,30 @@ class GameRedisSubscriberManager:
redisson.getTopic(s2cTopicName(gameId)).publish(json)
}
private def handleMove(gameId: String, uci: String): Unit =
private def handleMove(gameId: String, uci: String, playerId: Option[String]): Unit =
registry.get(gameId).foreach { entry =>
entry.engine.processUserInput(uci)
entry.mode match
case GameMode.Open => entry.engine.processUserInput(uci)
case GameMode.Authenticated =>
playerId match
case None => ()
case Some(pid) =>
val turn = entry.engine.context.turn
val authorised =
(entry.white.id.value == pid && turn == Color.White) ||
(entry.black.id.value == pid && turn == Color.Black)
if authorised then entry.engine.processUserInput(uci)
}
private def parseC2sMessage(msg: String): Option[C2sMessage] =
Try(objectMapper.readTree(msg)).toOption.flatMap { node =>
Option(node.get("type")).map(_.asText()).flatMap {
case "CONNECTED" => Some(C2sMessage.Connected)
case "MOVE" => Option(node.get("uci")).map(u => C2sMessage.Move(u.asText()))
case "MOVE" =>
Option(node.get("uci")).map { u =>
val pid = Option(node.get("playerId")).map(_.asText()).filter(_.nonEmpty)
C2sMessage.Move(u.asText(), pid)
}
case "PING" => Some(C2sMessage.Ping)
case _ => None
}
@@ -33,6 +33,9 @@ class InstanceHeartbeatService:
@ConfigProperty(name = "quarkus.grpc.server.port", defaultValue = "9000")
private var grpcPort: Int = 0
@ConfigProperty(name = "nowchess.coordinator.enabled", defaultValue = "true")
private var coordinatorEnabled: Boolean = true
private var coordinatorStub: CoordinatorServiceStub = uninitialized
private val log = Logger.getLogger(classOf[InstanceHeartbeatService])
@@ -45,20 +48,36 @@ class InstanceHeartbeatService:
private var redisHeartbeatExecutor = Executors.newScheduledThreadPool(1)
private var subscriptionCount = 0
private var localCacheSize = 0
private var serviceActive = false
private var shuttingDown = false
def onStart(@Observes event: StartupEvent): Unit =
if !coordinatorEnabled then
log.info("Coordinator support disabled via config; skipping heartbeat service startup")
return
try
shuttingDown = false
generateInstanceId()
initializeHeartbeatStream()
scheduleHeartbeats()
serviceActive = true
log.infof("Instance heartbeat service started with ID: %s", instanceId)
catch
case ex: Exception =>
serviceActive = false
log.errorf(ex, "Failed to start instance heartbeat service")
def onShutdown(@Observes event: ShutdownEvent): Unit =
shuttingDown = true
if !serviceActive then
log.info("Instance heartbeat service stopped")
return
try
cleanup()
serviceActive = false
log.info("Instance heartbeat service stopped")
catch
case ex: Exception =>
@@ -74,12 +93,16 @@ class InstanceHeartbeatService:
localCacheSize = count
def addGameSubscription(gameId: String): Unit =
if !coordinatorEnabled then return
val setKey = s"$redisPrefix:instance:$instanceId:games"
val gameSet = redissonClient.getSet[String](setKey)
gameSet.add(gameId)
subscriptionCount += 1
def removeGameSubscription(gameId: String): Unit =
if !coordinatorEnabled then return
val setKey = s"$redisPrefix:instance:$instanceId:games"
val gameSet = redissonClient.getSet[String](setKey)
gameSet.remove(gameId)
@@ -103,7 +126,8 @@ class InstanceHeartbeatService:
override def onError(t: Throwable): Unit =
log.warnf(t, "Heartbeat stream error")
streamObserver = None
heartbeatExecutor.schedule((() => initializeHeartbeatStream()): Runnable, 5, TimeUnit.SECONDS)
if !shuttingDown then
heartbeatExecutor.schedule((() => initializeHeartbeatStream()): Runnable, 5, TimeUnit.SECONDS)
override def onCompleted: Unit =
log.info("Heartbeat stream completed")
@@ -182,11 +206,12 @@ class InstanceHeartbeatService:
streamObserver.foreach(_.onCompleted())
streamObserver = None
val key = s"$redisPrefix:instances:$instanceId"
redissonClient.getBucket[String](key).delete()
if instanceId.nonEmpty then
val key = s"$redisPrefix:instances:$instanceId"
redissonClient.getBucket[String](key).delete()
val setKey = s"$redisPrefix:instance:$instanceId:games"
redissonClient.getSet[String](setKey).delete()
val setKey = s"$redisPrefix:instance:$instanceId:games"
redissonClient.getSet[String](setKey).delete()
heartbeatExecutor.shutdown()
redisHeartbeatExecutor.shutdown()
@@ -12,6 +12,8 @@ quarkus:
url: http://localhost:8085
nowchess:
coordinator:
enabled: false
redis:
host: localhost
port: 6379
@@ -0,0 +1,17 @@
package de.nowchess.chess.config
import jakarta.annotation.Priority
import jakarta.enterprise.context.ApplicationScoped
import jakarta.enterprise.inject.Alternative
import jakarta.enterprise.inject.Produces
import org.mockito.Mockito
import org.redisson.api.RedissonClient
@Alternative
@Priority(1)
@ApplicationScoped
class MockRedissonProducer:
@Produces
@ApplicationScoped
def produceRedissonClient(): RedissonClient =
Mockito.mock(classOf[RedissonClient], Mockito.RETURNS_DEEP_STUBS)
@@ -0,0 +1,27 @@
{
"reflection": [
{ "type": "scala.Tuple1[]" },
{ "type": "scala.Tuple2[]" },
{ "type": "scala.Tuple3[]" },
{ "type": "scala.Tuple4[]" },
{ "type": "scala.Tuple5[]" },
{ "type": "scala.Tuple6[]" },
{ "type": "scala.Tuple7[]" },
{ "type": "scala.Tuple8[]" },
{ "type": "scala.Tuple9[]" },
{ "type": "scala.Tuple10[]" },
{ "type": "scala.Tuple11[]" },
{ "type": "scala.Tuple12[]" },
{ "type": "scala.Tuple13[]" },
{ "type": "scala.Tuple14[]" },
{ "type": "scala.Tuple15[]" },
{ "type": "scala.Tuple16[]" },
{ "type": "scala.Tuple17[]" },
{ "type": "scala.Tuple18[]" },
{ "type": "scala.Tuple19[]" },
{ "type": "scala.Tuple20[]" },
{ "type": "scala.Tuple21[]" },
{ "type": "scala.Tuple22[]" },
{ "type": "com.fasterxml.jackson.module.scala.introspect.PropertyDescriptor[]" }
]
}
@@ -0,0 +1,18 @@
package de.nowchess.store.config
import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.quarkus.jackson.ObjectMapperCustomizer
import jakarta.inject.Singleton
@Singleton
class JacksonConfig extends ObjectMapperCustomizer:
def customize(mapper: ObjectMapper): Unit =
mapper.registerModule(new DefaultScalaModule() {
override def version(): Version =
// scalafix:off DisableSyntax.null
new Version(2, 21, 1, null, "com.fasterxml.jackson.module", "jackson-module-scala")
// scalafix:on DisableSyntax.null
})
@@ -0,0 +1,14 @@
package de.nowchess.store.config
import de.nowchess.store.domain.GameRecord
import de.nowchess.store.redis.GameWritebackEventDto
import io.quarkus.runtime.annotations.RegisterForReflection
@RegisterForReflection(
targets = Array(
classOf[GameRecord],
classOf[GameWritebackEventDto],
),
)
class NativeReflectionConfig
+3
View File
@@ -48,9 +48,12 @@ dependencies {
implementation(enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}"))
implementation("io.quarkus:quarkus-websockets-next")
implementation("io.quarkus:quarkus-jackson")
implementation("io.quarkus:quarkus-arc")
implementation("io.quarkus:quarkus-smallrye-jwt")
implementation("io.quarkus:quarkus-config-yaml")
implementation("io.quarkus:quarkus-smallrye-health")
implementation("com.fasterxml.jackson.module:jackson-module-scala_3:${versions["JACKSON_SCALA"]!!}")
implementation("org.redisson:redisson:${versions["REDISSON"]!!}")
testImplementation(platform("org.junit:junit-bom:${versions["JUNIT_BOM"]!!}"))
@@ -3,6 +3,9 @@ quarkus:
port: 8084
application:
name: nowchess-ws
swagger-ui:
always-include: true
path: /swagger-ui
grpc:
server:
use-separate-server: false
@@ -19,6 +22,12 @@ nowchess:
host: localhost
port: 6379
prefix: nowchess
mp:
jwt:
verify:
publickey:
location: keys/public.pem
issuer: nowchess
"%deployed":
nowchess:
@@ -26,3 +35,9 @@ nowchess:
host: ${REDIS_HOST}
port: ${REDIS_PORT:6379}
prefix: ${REDIS_PREFIX:nowchess}
mp:
jwt:
verify:
publickey:
location: ${JWT_PUBLIC_KEY_PATH:keys/public.pem}
issuer: nowchess
@@ -0,0 +1,9 @@
-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxDsnsCAl0vQx7Vu9CLDZ
g0SG05NgUzu9T+3DTEaHGq60T2uriO8BenwyvsF3BnDqTbKf4voohZ1DNfzdbT1J
Fj8B62FrDmxcO+sp1/b5HUCJP6y2uSRCmzOHe5k7Pk1IEi72FgBpKXSRkFibRlVf
634g7mgsPZAQ9PJEsv4Qvm05T9L6+Gmq6N3bMVLKRXs4RhDhaFbYH9GtUg1eI0yH
YjGyRfqzW/nqVMstOLHt8CuPouq4p7eMzeDH3YHkxPm4GG5foCXMOd2DZrW0SCcr
7dhFeNVWzQ2m53eOhBzNQX+v3pgjVStsePhBRt2LyGfwkNzmqDgqWsMzSHRMY+cn
WQIDAQAB
-----END PUBLIC KEY-----
@@ -0,0 +1,18 @@
package de.nowchess.ws.config
import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import io.quarkus.jackson.ObjectMapperCustomizer
import jakarta.inject.Singleton
@Singleton
class JacksonConfig extends ObjectMapperCustomizer:
def customize(mapper: ObjectMapper): Unit =
mapper.registerModule(new DefaultScalaModule() {
override def version(): Version =
// scalafix:off DisableSyntax.null
new Version(2, 21, 1, null, "com.fasterxml.jackson.module", "jackson-module-scala")
// scalafix:on DisableSyntax.null
})
@@ -0,0 +1,12 @@
package de.nowchess.ws.config
import de.nowchess.ws.resource.ConnectionMeta
import io.quarkus.runtime.annotations.RegisterForReflection
@RegisterForReflection(
targets = Array(
classOf[ConnectionMeta],
),
)
class NativeReflectionConfig
@@ -0,0 +1,7 @@
package de.nowchess.ws.resource
final case class ConnectionMeta(
gameId: String,
listenerId: Int,
playerId: Option[String],
)
@@ -2,10 +2,13 @@ package de.nowchess.ws.resource
import de.nowchess.ws.config.RedisConfig
import io.quarkus.websockets.next.*
import io.smallrye.jwt.auth.principal.JWTParser
import jakarta.inject.Inject
import org.redisson.api.listener.MessageListener
import org.redisson.api.RedissonClient
import scala.compiletime.uninitialized
import scala.util.Try
import java.util.concurrent.ConcurrentHashMap
@WebSocket(path = "/api/board/game/{gameId}/ws")
@@ -17,9 +20,12 @@ class GameWebSocketResource:
@Inject
var redisConfig: RedisConfig = uninitialized
@Inject
var jwtParser: JWTParser = uninitialized
// scalafix:on DisableSyntax.var
private val listenerIds = new ConcurrentHashMap[String, (String, Int)]()
private val connections = new ConcurrentHashMap[String, ConnectionMeta]()
private def s2cTopic(gameId: String): String =
s"${redisConfig.prefix}:game:$gameId:s2c"
@@ -28,27 +34,41 @@ class GameWebSocketResource:
s"${redisConfig.prefix}:game:$gameId:c2s"
@OnOpen
def onOpen(connection: WebSocketConnection): Unit =
val gameId = connection.pathParam("gameId")
val topic = redisson.getTopic(s2cTopic(gameId))
def onOpen(connection: WebSocketConnection, handshake: HandshakeRequest): Unit =
val gameId = connection.pathParam("gameId")
val playerId = Option(handshake.header("Authorization"))
.filter(_.nonEmpty)
.flatMap(token => Try(jwtParser.parse(token)).toOption)
.map(_.getSubject)
val topic = redisson.getTopic(s2cTopic(gameId))
val listenerId = topic.addListener(
classOf[String],
new MessageListener[String]:
def onMessage(channel: CharSequence, msg: String): Unit =
connection.sendText(msg).subscribe().`with`(_ => (), _ => ()),
)
listenerIds.put(connection.id(), (gameId, listenerId))
val connectedMsg = s"""{"type":"CONNECTED","gameId":"$gameId"}"""
connections.put(connection.id(), ConnectionMeta(gameId, listenerId, playerId))
val connectedMsg = playerId match
case Some(pid) => s"""{"type":"CONNECTED","gameId":"$gameId","playerId":"$pid"}"""
case None => s"""{"type":"CONNECTED","gameId":"$gameId"}"""
redisson.getTopic(c2sTopic(gameId)).publish(connectedMsg)
@OnTextMessage
def onTextMessage(connection: WebSocketConnection, message: String): Unit =
Option(listenerIds.get(connection.id())).foreach { case (gameId, _) =>
redisson.getTopic(c2sTopic(gameId)).publish(message)
Option(connections.get(connection.id())).foreach { meta =>
val enriched = meta.playerId match
case Some(pid) => injectPlayerId(message, pid)
case None => message
redisson.getTopic(c2sTopic(meta.gameId)).publish(enriched)
}
@OnClose
def onClose(connection: WebSocketConnection): Unit =
Option(listenerIds.remove(connection.id())).foreach { case (gameId, listenerId) =>
redisson.getTopic(s2cTopic(gameId)).removeListener(listenerId)
Option(connections.remove(connection.id())).foreach { meta =>
redisson.getTopic(s2cTopic(meta.gameId)).removeListener(meta.listenerId)
}
private def injectPlayerId(msg: String, pid: String): String =
val trimmed = msg.trim
if trimmed.endsWith("}") then trimmed.dropRight(1) + s""","playerId":"$pid"}"""
else msg