Janis Eccarius
a3eb172916
Wire five Spark jobs (OpeningBook, PlayerStats, PlayerClustering, PlayerGraph, LiveDashboard) to Kubernetes via native Spark-on-K8s mode. - spark-analytics/base: ServiceAccount, RBAC Role/RoleBinding (driver executor pod management), two PVCs (10Gi output, 2Gi checkpoint), four CronJobs for batch analytics, one Deployment for the Structured Streaming live dashboard - spark-analytics/eu-central-1-prod: namespace overlay pinning image to 0.1.0 with strategic merge patches that update both the container image field and the spark.kubernetes.container.image conf arg - eu-central-1/argo-apps/spark-analytics: ArgoCD Application pointing at the prod overlay with automated prune+selfHeal sync Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
207 lines
7.5 KiB
YAML
Executable File
207 lines
7.5 KiB
YAML
Executable File
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
kind: Kustomization
|
|
namespace: nowchess
|
|
resources:
|
|
- ../base
|
|
images:
|
|
- name: ghcr.io/now-chess/now-chess-systems/analytics
|
|
newTag: 0.1.0
|
|
patches:
|
|
# Update the spark.kubernetes.container.image conf argument in each CronJob
|
|
# and the Deployment so executor pods pull the same pinned tag as the driver.
|
|
- patch: |-
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: spark-opening-book
|
|
spec:
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: spark-driver
|
|
command:
|
|
- /opt/spark/bin/spark-submit
|
|
- --master
|
|
- k8s://https://kubernetes.default.svc:443
|
|
- --deploy-mode
|
|
- client
|
|
- --conf
|
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
|
- --conf
|
|
- spark.kubernetes.namespace=nowchess
|
|
- --conf
|
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
|
- --conf
|
|
- spark.kubernetes.executor.deleteOnTermination=true
|
|
- --conf
|
|
- spark.executor.instances=2
|
|
- --conf
|
|
- spark.executor.memory=1g
|
|
- --conf
|
|
- spark.driver.memory=512m
|
|
- --class
|
|
- de.nowchess.analytics.OpeningBookJob
|
|
- local:///app/analytics.jar
|
|
- /spark-output/opening-book
|
|
- "10"
|
|
target:
|
|
kind: CronJob
|
|
name: spark-opening-book
|
|
- patch: |-
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: spark-player-stats
|
|
spec:
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: spark-driver
|
|
command:
|
|
- /opt/spark/bin/spark-submit
|
|
- --master
|
|
- k8s://https://kubernetes.default.svc:443
|
|
- --deploy-mode
|
|
- client
|
|
- --conf
|
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
|
- --conf
|
|
- spark.kubernetes.namespace=nowchess
|
|
- --conf
|
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
|
- --conf
|
|
- spark.kubernetes.executor.deleteOnTermination=true
|
|
- --conf
|
|
- spark.executor.instances=2
|
|
- --conf
|
|
- spark.executor.memory=1g
|
|
- --conf
|
|
- spark.driver.memory=512m
|
|
- --class
|
|
- de.nowchess.analytics.PlayerStatsJob
|
|
- local:///app/analytics.jar
|
|
- /spark-output/player-stats
|
|
target:
|
|
kind: CronJob
|
|
name: spark-player-stats
|
|
- patch: |-
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: spark-player-clustering
|
|
spec:
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: spark-driver
|
|
command:
|
|
- /opt/spark/bin/spark-submit
|
|
- --master
|
|
- k8s://https://kubernetes.default.svc:443
|
|
- --deploy-mode
|
|
- client
|
|
- --conf
|
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
|
- --conf
|
|
- spark.kubernetes.namespace=nowchess
|
|
- --conf
|
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
|
- --conf
|
|
- spark.kubernetes.executor.deleteOnTermination=true
|
|
- --conf
|
|
- spark.executor.instances=2
|
|
- --conf
|
|
- spark.executor.memory=1g
|
|
- --conf
|
|
- spark.driver.memory=512m
|
|
- --class
|
|
- de.nowchess.analytics.PlayerClusteringJob
|
|
- local:///app/analytics.jar
|
|
- /spark-output/player-clusters
|
|
- "4"
|
|
target:
|
|
kind: CronJob
|
|
name: spark-player-clustering
|
|
- patch: |-
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: spark-player-graph
|
|
spec:
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: spark-driver
|
|
command:
|
|
- /opt/spark/bin/spark-submit
|
|
- --master
|
|
- k8s://https://kubernetes.default.svc:443
|
|
- --deploy-mode
|
|
- client
|
|
- --conf
|
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
|
- --conf
|
|
- spark.kubernetes.namespace=nowchess
|
|
- --conf
|
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
|
- --conf
|
|
- spark.kubernetes.executor.deleteOnTermination=true
|
|
- --conf
|
|
- spark.executor.instances=2
|
|
- --conf
|
|
- spark.executor.memory=1g
|
|
- --conf
|
|
- spark.driver.memory=512m
|
|
- --class
|
|
- de.nowchess.analytics.PlayerGraphJob
|
|
- local:///app/analytics.jar
|
|
- /spark-output/player-graph
|
|
target:
|
|
kind: CronJob
|
|
name: spark-player-graph
|
|
- patch: |-
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: spark-live-dashboard
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: spark-driver
|
|
command:
|
|
- /opt/spark/bin/spark-submit
|
|
- --master
|
|
- k8s://https://kubernetes.default.svc:443
|
|
- --deploy-mode
|
|
- client
|
|
- --conf
|
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
|
- --conf
|
|
- spark.kubernetes.namespace=nowchess
|
|
- --conf
|
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
|
- --conf
|
|
- spark.kubernetes.executor.deleteOnTermination=true
|
|
- --conf
|
|
- spark.executor.instances=1
|
|
- --conf
|
|
- spark.executor.memory=512m
|
|
- --conf
|
|
- spark.driver.memory=512m
|
|
- --class
|
|
- de.nowchess.analytics.LiveDashboardJob
|
|
- local:///app/analytics.jar
|
|
- /spark-checkpoint
|
|
target:
|
|
kind: Deployment
|
|
name: spark-live-dashboard
|