From 23f814ae2919912ed5976fe19ad72aa912b9d4bf Mon Sep 17 00:00:00 2001
From: Janis Eccarius <eccariusjanis@gmail.com>
Date: Wed, 17 Jun 2026 00:03:06 +0200
Subject: [PATCH] fix(spark): add headless service so executors can reach
 driver

Spark executors crashed immediately (exit 1) because the driver
advertised its pod name as the RPC hostname, which is not DNS-resolvable
in-cluster. Added headless Service `spark-live-dashboard-driver` and
pinned spark.driver.host/port + spark.blockManager.port so workers can
connect back to the driver. Stops the JVM fork storm that saturated all
three nodes at ~100% CPU.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 spark-analytics/base/streaming.yaml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/spark-analytics/base/streaming.yaml b/spark-analytics/base/streaming.yaml
index 0d3b726..2176d2a 100755
--- a/spark-analytics/base/streaming.yaml
+++ b/spark-analytics/base/streaming.yaml
@@ -1,3 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-live-dashboard-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-live-dashboard
+  ports:
+    - name: driver
+      port: 37007
+      targetPort: 37007
+    - name: blockmanager
+      port: 37777
+      targetPort: 37777
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -35,6 +54,12 @@ spec:
             - spark.executor.memory=512m
             - --conf
             - spark.driver.memory=512m
+            - --conf
+            - spark.driver.host=spark-live-dashboard-driver
+            - --conf
+            - spark.driver.port=37007
+            - --conf
+            - spark.blockManager.port=37777
             - --class
             - de.nowchess.analytics.LiveDashboardJob
             - local:///app/analytics.jar