feat(nci): add Spark analytics Kubernetes manifests
Wire five Spark jobs (OpeningBook, PlayerStats, PlayerClustering, PlayerGraph, LiveDashboard) to Kubernetes via native Spark-on-K8s mode. - spark-analytics/base: ServiceAccount, RBAC Role/RoleBinding (driver executor pod management), two PVCs (10Gi output, 2Gi checkpoint), four CronJobs for batch analytics, one Deployment for the Structured Streaming live dashboard - spark-analytics/eu-central-1-prod: namespace overlay pinning image to 0.1.0 with strategic merge patches that update both the container image field and the spark.kubernetes.container.image conf arg - eu-central-1/argo-apps/spark-analytics: ArgoCD Application pointing at the prod overlay with automated prune+selfHeal sync Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Janis Eccarius
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: spark-analytics
|
||||||
|
namespace: argocd
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
destination:
|
||||||
|
namespace: nowchess
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
sources:
|
||||||
|
- repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
|
||||||
|
path: ./spark-analytics/eu-central-1-prod
|
||||||
|
targetRevision: main
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
Executable
+329
@@ -0,0 +1,329 @@
|
|||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-opening-book
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
schedule: "0 2 * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
serviceAccountName: spark-analytics
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-pull-secret
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
# spark.kubernetes.container.image must match the analytics image tag set in the overlay
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.OpeningBookJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/opening-book
|
||||||
|
- "10"
|
||||||
|
env:
|
||||||
|
- name: NOWCHESS_JDBC_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_URL
|
||||||
|
- name: NOWCHESS_DB_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_USER
|
||||||
|
- name: NOWCHESS_DB_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_PASSWORD
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 768Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: spark-output
|
||||||
|
mountPath: /spark-output
|
||||||
|
volumes:
|
||||||
|
- name: spark-output
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: spark-analytics-output
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-player-stats
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
schedule: "0 3 * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
serviceAccountName: spark-analytics
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-pull-secret
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
# spark.kubernetes.container.image must match the analytics image tag set in the overlay
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.PlayerStatsJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/player-stats
|
||||||
|
env:
|
||||||
|
- name: NOWCHESS_JDBC_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_URL
|
||||||
|
- name: NOWCHESS_DB_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_USER
|
||||||
|
- name: NOWCHESS_DB_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_PASSWORD
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 768Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: spark-output
|
||||||
|
mountPath: /spark-output
|
||||||
|
volumes:
|
||||||
|
- name: spark-output
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: spark-analytics-output
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-player-clustering
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
schedule: "0 4 * * 0"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
serviceAccountName: spark-analytics
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-pull-secret
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
# spark.kubernetes.container.image must match the analytics image tag set in the overlay
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.PlayerClusteringJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/player-clusters
|
||||||
|
- "4"
|
||||||
|
env:
|
||||||
|
- name: NOWCHESS_JDBC_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_URL
|
||||||
|
- name: NOWCHESS_DB_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_USER
|
||||||
|
- name: NOWCHESS_DB_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_PASSWORD
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 768Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: spark-output
|
||||||
|
mountPath: /spark-output
|
||||||
|
volumes:
|
||||||
|
- name: spark-output
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: spark-analytics-output
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-player-graph
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
schedule: "0 5 * * 0"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
serviceAccountName: spark-analytics
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-pull-secret
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
# spark.kubernetes.container.image must match the analytics image tag set in the overlay
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.PlayerGraphJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/player-graph
|
||||||
|
env:
|
||||||
|
- name: NOWCHESS_JDBC_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_URL
|
||||||
|
- name: NOWCHESS_DB_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_USER
|
||||||
|
- name: NOWCHESS_DB_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_PASSWORD
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 768Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: spark-output
|
||||||
|
mountPath: /spark-output
|
||||||
|
volumes:
|
||||||
|
- name: spark-output
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: spark-analytics-output
|
||||||
Executable
+9
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- serviceaccount.yaml
|
||||||
|
- rbac.yaml
|
||||||
|
- pvc.yaml
|
||||||
|
- cronjobs.yaml
|
||||||
|
- streaming.yaml
|
||||||
Executable
+27
@@ -0,0 +1,27 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: spark-analytics-output
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: spark-streaming-checkpoint
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 2Gi
|
||||||
Executable
+32
@@ -0,0 +1,32 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: spark-analytics-driver
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods"]
|
||||||
|
verbs: ["create", "get", "list", "watch", "delete", "patch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["services"]
|
||||||
|
verbs: ["create", "get", "delete"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["configmaps"]
|
||||||
|
verbs: ["create", "get", "list", "delete"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: spark-analytics-driver
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: spark-analytics
|
||||||
|
roleRef:
|
||||||
|
kind: Role
|
||||||
|
name: spark-analytics-driver
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
Executable
+7
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: spark-analytics
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
Executable
+80
@@ -0,0 +1,80 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: spark-live-dashboard
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: spark-live-dashboard
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: spark-live-dashboard
|
||||||
|
app.kubernetes.io/name: spark-analytics
|
||||||
|
app.kubernetes.io/part-of: nowchess
|
||||||
|
spec:
|
||||||
|
serviceAccountName: spark-analytics
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: ghcr-pull-secret
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
# spark.kubernetes.container.image must match the analytics image tag set in the overlay
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=1
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=512m
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.LiveDashboardJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-checkpoint
|
||||||
|
env:
|
||||||
|
- name: NOWCHESS_JDBC_URL
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_URL
|
||||||
|
- name: NOWCHESS_DB_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_USER
|
||||||
|
- name: NOWCHESS_DB_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: ncs-db-secrets
|
||||||
|
key: STORE_DB_PASSWORD
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 768Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: spark-checkpoint
|
||||||
|
mountPath: /spark-checkpoint
|
||||||
|
volumes:
|
||||||
|
- name: spark-checkpoint
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: spark-streaming-checkpoint
|
||||||
+206
@@ -0,0 +1,206 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: nowchess
|
||||||
|
resources:
|
||||||
|
- ../base
|
||||||
|
images:
|
||||||
|
- name: ghcr.io/now-chess/now-chess-systems/analytics
|
||||||
|
newTag: 0.1.0
|
||||||
|
patches:
|
||||||
|
# Update the spark.kubernetes.container.image conf argument in each CronJob
|
||||||
|
# and the Deployment so executor pods pull the same pinned tag as the driver.
|
||||||
|
- patch: |-
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-opening-book
|
||||||
|
spec:
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.OpeningBookJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/opening-book
|
||||||
|
- "10"
|
||||||
|
target:
|
||||||
|
kind: CronJob
|
||||||
|
name: spark-opening-book
|
||||||
|
- patch: |-
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-player-stats
|
||||||
|
spec:
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.PlayerStatsJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/player-stats
|
||||||
|
target:
|
||||||
|
kind: CronJob
|
||||||
|
name: spark-player-stats
|
||||||
|
- patch: |-
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-player-clustering
|
||||||
|
spec:
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.PlayerClusteringJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/player-clusters
|
||||||
|
- "4"
|
||||||
|
target:
|
||||||
|
kind: CronJob
|
||||||
|
name: spark-player-clustering
|
||||||
|
- patch: |-
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: spark-player-graph
|
||||||
|
spec:
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=2
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=1g
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.PlayerGraphJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-output/player-graph
|
||||||
|
target:
|
||||||
|
kind: CronJob
|
||||||
|
name: spark-player-graph
|
||||||
|
- patch: |-
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: spark-live-dashboard
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: spark-driver
|
||||||
|
command:
|
||||||
|
- /opt/spark/bin/spark-submit
|
||||||
|
- --master
|
||||||
|
- k8s://https://kubernetes.default.svc:443
|
||||||
|
- --deploy-mode
|
||||||
|
- client
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.namespace=nowchess
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||||
|
- --conf
|
||||||
|
- spark.kubernetes.executor.deleteOnTermination=true
|
||||||
|
- --conf
|
||||||
|
- spark.executor.instances=1
|
||||||
|
- --conf
|
||||||
|
- spark.executor.memory=512m
|
||||||
|
- --conf
|
||||||
|
- spark.driver.memory=512m
|
||||||
|
- --class
|
||||||
|
- de.nowchess.analytics.LiveDashboardJob
|
||||||
|
- local:///app/analytics.jar
|
||||||
|
- /spark-checkpoint
|
||||||
|
target:
|
||||||
|
kind: Deployment
|
||||||
|
name: spark-live-dashboard
|
||||||
Reference in New Issue
Block a user