NowChess/GitOps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: co-locate analytics driver pods with spark-worker for RWO PVC access

Browse Source 
do-block-storage only supports ReadWriteOnce. Executor tasks failed with
`Mkdirs failed` because the executor ran on the spark-worker node which
had no mount of the spark-analytics-output PVC.

- Mount spark-analytics-output PVC on spark-worker pod at /spark-output
- Add podAffinity to all 11 analytics CronJob driver pods to schedule on
  the same node as spark-worker, ensuring the RWO PVC is accessible by
  both driver and executor

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Janis Eccarius
2026-06-21 16:06:51 +02:00
parent 283c6728d4
commit f2d13efcd7
2 changed files with 82 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spark-analytics/base/cronjobs.yaml
+77
View File
@@ -230,6 +230,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -314,6 +321,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -397,6 +411,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -481,6 +502,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -564,6 +592,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -647,6 +682,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -730,6 +772,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -813,6 +862,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -896,6 +952,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -978,6 +1041,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
@@ -1061,6 +1131,13 @@ spec:
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets: