NowChess/GitOps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(spark): add headless Services + driver.host/port so executors can reach driver pods

Browse Source
This commit is contained in:
Janis Eccarius
2026-06-17 01:29:35 +02:00
parent 27920182cb
commit fe75a75a2e
2 changed files with 144 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spark-analytics/base/cronjobs.yaml
+112 -8
View File
@@ -1,3 +1,79 @@
apiVersion: v1
kind: Service
metadata:
name: spark-opening-book-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-opening-book
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: v1
kind: Service
metadata:
name: spark-player-stats-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-player-stats
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: v1
kind: Service
metadata:
name: spark-player-clustering-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-player-clustering
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: v1
kind: Service
metadata:
name: spark-player-graph-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-player-graph
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: batch/v1 apiVersion: batch/v1
kind: CronJob kind: CronJob
metadata: metadata:
@@ -15,6 +91,7 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app: spark-opening-book
app.kubernetes.io/name: spark-analytics app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess app.kubernetes.io/part-of: nowchess
spec: spec:
@@ -32,13 +109,19 @@ spec:
- --master - --master
- spark://spark-master:7077 - spark://spark-master:7077
- --conf - --conf
- spark.executor.instances=2 - spark.executor.instances=1
- --conf - --conf
- spark.executor.cores=2 - spark.executor.cores=1
- --conf - --conf
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-opening-book-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.OpeningBookJob - de.nowchess.analytics.OpeningBookJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -92,6 +175,7 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app: spark-player-stats
app.kubernetes.io/name: spark-analytics app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess app.kubernetes.io/part-of: nowchess
spec: spec:
@@ -109,13 +193,19 @@ spec:
- --master - --master
- spark://spark-master:7077 - spark://spark-master:7077
- --conf - --conf
- spark.executor.instances=2 - spark.executor.instances=1
- --conf - --conf
- spark.executor.cores=2 - spark.executor.cores=1
- --conf - --conf
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-player-stats-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.PlayerStatsJob - de.nowchess.analytics.PlayerStatsJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -168,6 +258,7 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app: spark-player-clustering
app.kubernetes.io/name: spark-analytics app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess app.kubernetes.io/part-of: nowchess
spec: spec:
@@ -185,13 +276,19 @@ spec:
- --master - --master
- spark://spark-master:7077 - spark://spark-master:7077
- --conf - --conf
- spark.executor.instances=2 - spark.executor.instances=1
- --conf - --conf
- spark.executor.cores=2 - spark.executor.cores=1
- --conf - --conf
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-player-clustering-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.PlayerClusteringJob - de.nowchess.analytics.PlayerClusteringJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -245,6 +342,7 @@ spec:
template: template:
metadata: metadata:
labels: labels:
app: spark-player-graph
app.kubernetes.io/name: spark-analytics app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess app.kubernetes.io/part-of: nowchess
spec: spec:
@@ -262,13 +360,19 @@ spec:
- --master - --master
- spark://spark-master:7077 - spark://spark-master:7077
- --conf - --conf
- spark.executor.instances=2 - spark.executor.instances=1
- --conf - --conf
- spark.executor.cores=2 - spark.executor.cores=1
- --conf - --conf
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-player-graph-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.PlayerGraphJob - de.nowchess.analytics.PlayerGraphJob
- local:///app/analytics.jar - local:///app/analytics.jar
spark-analytics/eu-central-1-prod/kustomization.yaml
+32
View File
@@ -33,6 +33,12 @@ patches:
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-opening-book-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.OpeningBookJob - de.nowchess.analytics.OpeningBookJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -65,6 +71,12 @@ patches:
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-player-stats-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.PlayerStatsJob - de.nowchess.analytics.PlayerStatsJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -96,6 +108,12 @@ patches:
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-player-clustering-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.PlayerClusteringJob - de.nowchess.analytics.PlayerClusteringJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -128,6 +146,12 @@ patches:
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-player-graph-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class - --class
- de.nowchess.analytics.PlayerGraphJob - de.nowchess.analytics.PlayerGraphJob
- local:///app/analytics.jar - local:///app/analytics.jar
@@ -152,9 +176,17 @@ patches:
- --conf - --conf
- spark.executor.instances=1 - spark.executor.instances=1
- --conf - --conf
- spark.executor.cores=1
- --conf
- spark.executor.memory=512m - spark.executor.memory=512m
- --conf - --conf
- spark.driver.memory=512m - spark.driver.memory=512m
- --conf
- spark.driver.host=spark-live-dashboard-driver
- --conf
- spark.driver.port=37007
- --conf
- spark.blockManager.port=37777
- --class - --class
- de.nowchess.analytics.LiveDashboardJob - de.nowchess.analytics.LiveDashboardJob
- local:///app/analytics.jar - local:///app/analytics.jar