NowChess/GitOps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7b95f16361764280e5fc9f4675fdfdc3b61708bf
GitOps/spark-analytics/base/cronjobs.yaml
T
Janis Eccarius ab95b4f101 fix(spark): replace shared spark-events PVC with emptyDir to fix Multi-Attach error 
local-path provisioner is RWO-only; attaching a single PVC to master, two
workers, history-server, and streaming pod on separate nodes causes
Multi-Attach errors. Each pod now gets its own ephemeral emptyDir for
/spark-events. Event logging confs removed from CronJobs and streaming
Deployment (logs are pod-local and do not persist across restarts).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 20:33:48 +02:00

286 lines
8.9 KiB
YAML
Executable File
Raw Blame History

apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-opening-book
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 2 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.OpeningBookJob
- local:///app/analytics.jar
- /spark-output/opening-book
- "10"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-stats
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 3 * * *"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.PlayerStatsJob
- local:///app/analytics.jar
- /spark-output/player-stats
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-clustering
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 4 * * 0"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.PlayerClusteringJob
- local:///app/analytics.jar
- /spark-output/player-clusters
- "4"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-player-graph
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 5 * * 0"
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=2
- --conf
- spark.executor.memory=1g
- --conf
- spark.driver.memory=512m
- --class
- de.nowchess.analytics.PlayerGraphJob
- local:///app/analytics.jar
- /spark-output/player-graph
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 500m
memory: 768Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
Reference in New Issue View Git Blame Copy Permalink