From 1b75466b3eff116f827b971b586ef4e2287c0edd Mon Sep 17 00:00:00 2001 From: Janis Eccarius Date: Wed, 17 Jun 2026 00:44:16 +0200 Subject: [PATCH] fix(spark): cap executor cores to prevent job starvation Spark standalone gives executors all worker cores by default. LiveDashboard was consuming all 8 cores (2 workers x 4 cores). - streaming: spark.executor.cores=1 (uses 1 core total) - batch jobs: spark.executor.cores=2 (uses 4 cores total, 2 per executor) Leaves headroom for concurrent streaming + batch. Co-Authored-By: Claude Sonnet 4.6 --- spark-analytics/base/cronjobs.yaml | 8 ++++++++ spark-analytics/base/streaming.yaml | 2 ++ 2 files changed, 10 insertions(+) diff --git a/spark-analytics/base/cronjobs.yaml b/spark-analytics/base/cronjobs.yaml index 92832a8..12a65b6 100755 --- a/spark-analytics/base/cronjobs.yaml +++ b/spark-analytics/base/cronjobs.yaml @@ -33,6 +33,8 @@ spec: - --conf - spark.executor.instances=2 - --conf + - spark.executor.cores=2 + - --conf - spark.executor.memory=1g - --conf - spark.driver.memory=512m @@ -107,6 +109,8 @@ spec: - --conf - spark.executor.instances=2 - --conf + - spark.executor.cores=2 + - --conf - spark.executor.memory=1g - --conf - spark.driver.memory=512m @@ -180,6 +184,8 @@ spec: - --conf - spark.executor.instances=2 - --conf + - spark.executor.cores=2 + - --conf - spark.executor.memory=1g - --conf - spark.driver.memory=512m @@ -254,6 +260,8 @@ spec: - --conf - spark.executor.instances=2 - --conf + - spark.executor.cores=2 + - --conf - spark.executor.memory=1g - --conf - spark.driver.memory=512m diff --git a/spark-analytics/base/streaming.yaml b/spark-analytics/base/streaming.yaml index 041cada..f98f08e 100755 --- a/spark-analytics/base/streaming.yaml +++ b/spark-analytics/base/streaming.yaml @@ -53,6 +53,8 @@ spec: - --conf - spark.executor.instances=1 - --conf + - spark.executor.cores=1 + - --conf - spark.executor.memory=512m - --conf - spark.driver.memory=512m