From d3feb65eb3baa6a420ecea700659a68288f3f169 Mon Sep 17 00:00:00 2001
From: Janis Eccarius <eccariusjanis@gmail.com>
Date: Wed, 17 Jun 2026 00:41:49 +0200
Subject: [PATCH] fix(spark): increase worker cores to 4 to allow concurrent
 jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With SPARK_WORKER_CORES=1 and 2 workers the cluster has 2 total cores.
LiveDashboard holds both — batch jobs wait indefinitely. 4 cores per
worker gives 8 total, enough to run streaming + batch concurrently.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 spark-cluster/base/workers.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spark-cluster/base/workers.yaml b/spark-cluster/base/workers.yaml
index cbccfee..9f54b41 100755
--- a/spark-cluster/base/workers.yaml
+++ b/spark-cluster/base/workers.yaml
@@ -25,7 +25,7 @@ spec:
             - spark://spark-master:7077
           env:
             - name: SPARK_WORKER_CORES
-              value: "1"
+              value: "4"
             - name: SPARK_WORKER_MEMORY
               value: "512m"
           ports: