NowChess/GitOps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1b75466b3eff116f827b971b586ef4e2287c0edd
GitOps/spark-analytics/base/cronjobs.yaml
T
Janis Eccarius 1b75466b3e fix(spark): cap executor cores to prevent job starvation 
Spark standalone gives executors all worker cores by default.
LiveDashboard was consuming all 8 cores (2 workers x 4 cores).
- streaming: spark.executor.cores=1 (uses 1 core total)
- batch jobs: spark.executor.cores=2 (uses 4 cores total, 2 per executor)
Leaves headroom for concurrent streaming + batch.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-17 00:44:16 +02:00

302 lines
9.3 KiB
YAML
Executable File
Raw Blame History

apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-opening-book
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 2 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.cores=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.OpeningBookJob
- local:///app/analytics.jar
- /spark-output/opening-book
- "10"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-stats
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 3 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.cores=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.PlayerStatsJob
- local:///app/analytics.jar
- /spark-output/player-stats
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-clustering
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 4 * * 0"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.cores=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.PlayerClusteringJob
- local:///app/analytics.jar
- /spark-output/player-clusters
- "4"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-graph
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 5 * * 0"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.cores=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.PlayerGraphJob
- local:///app/analytics.jar
- /spark-output/player-graph
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
Reference in New Issue View Git Blame Copy Permalink