feat(nci): add Spark analytics Kubernetes manifests
Wire five Spark jobs (OpeningBook, PlayerStats, PlayerClustering, PlayerGraph, LiveDashboard) to Kubernetes via native Spark-on-K8s mode. - spark-analytics/base: ServiceAccount, RBAC Role/RoleBinding (driver executor pod management), two PVCs (10Gi output, 2Gi checkpoint), four CronJobs for batch analytics, one Deployment for the Structured Streaming live dashboard - spark-analytics/eu-central-1-prod: namespace overlay pinning image to 0.1.0 with strategic merge patches that update both the container image field and the spark.kubernetes.container.image conf arg - eu-central-1/argo-apps/spark-analytics: ArgoCD Application pointing at the prod overlay with automated prune+selfHeal sync Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Executable
+80
@@ -0,0 +1,80 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: spark-live-dashboard
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: spark-live-dashboard
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: spark-live-dashboard
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
serviceAccountName: spark-analytics
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull-secret
|
||||
containers:
|
||||
- name: spark-driver
|
||||
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- k8s://https://kubernetes.default.svc:443
|
||||
- --deploy-mode
|
||||
- client
|
||||
- --conf
|
||||
# spark.kubernetes.container.image must match the analytics image tag set in the overlay
|
||||
- spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
- --conf
|
||||
- spark.kubernetes.namespace=nowchess
|
||||
- --conf
|
||||
- spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics
|
||||
- --conf
|
||||
- spark.kubernetes.executor.deleteOnTermination=true
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --class
|
||||
- de.nowchess.analytics.LiveDashboardJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-checkpoint
|
||||
env:
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: spark-checkpoint
|
||||
mountPath: /spark-checkpoint
|
||||
volumes:
|
||||
- name: spark-checkpoint
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-streaming-checkpoint
|
||||
Reference in New Issue
Block a user