feat(spark): add staging overlay, scale down prod
- spark-cluster/eu-central-1-staging: namespace nowchess-staging, ingress patched to spark-st.* / spark-history-st.* with TLS + cert-manager - spark-analytics/eu-central-1-staging: namespace nowchess-staging, image pinned to 0.2.0, all CronJob + live-dashboard patches carried over - eu-central-1/argo-apps: spark-cluster-staging + spark-analytics-staging ArgoCD apps pointing at new staging overlays - spark-cluster/eu-central-1-prod: scale spark-master and spark-worker to 0 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: spark-analytics-staging
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
destination:
|
||||
namespace: nowchess-staging
|
||||
server: https://kubernetes.default.svc
|
||||
sources:
|
||||
- repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
|
||||
path: ./spark-analytics/eu-central-1-staging
|
||||
targetRevision: main
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: spark-cluster-staging
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
destination:
|
||||
namespace: nowchess-staging
|
||||
server: https://kubernetes.default.svc
|
||||
sources:
|
||||
- repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
|
||||
path: ./spark-cluster/eu-central-1-staging
|
||||
targetRevision: main
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -0,0 +1,194 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: nowchess-staging
|
||||
resources:
|
||||
- ../base
|
||||
images:
|
||||
- name: ghcr.io/now-chess/now-chess-systems/analytics
|
||||
newTag: 0.2.0
|
||||
patches:
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-opening-book
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.cores=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.driver.host=spark-opening-book-driver
|
||||
- --conf
|
||||
- spark.driver.port=44167
|
||||
- --conf
|
||||
- spark.blockManager.port=44777
|
||||
- --class
|
||||
- de.nowchess.analytics.OpeningBookJob
|
||||
- file:///app/analytics.jar
|
||||
- /spark-output/opening-book
|
||||
- "10"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-opening-book
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-player-stats
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.cores=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.driver.host=spark-player-stats-driver
|
||||
- --conf
|
||||
- spark.driver.port=44167
|
||||
- --conf
|
||||
- spark.blockManager.port=44777
|
||||
- --class
|
||||
- de.nowchess.analytics.PlayerStatsJob
|
||||
- file:///app/analytics.jar
|
||||
- /spark-output/player-stats
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-player-stats
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-player-clustering
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.cores=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.driver.host=spark-player-clustering-driver
|
||||
- --conf
|
||||
- spark.driver.port=44167
|
||||
- --conf
|
||||
- spark.blockManager.port=44777
|
||||
- --class
|
||||
- de.nowchess.analytics.PlayerClusteringJob
|
||||
- file:///app/analytics.jar
|
||||
- /spark-output/player-clusters
|
||||
- "4"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-player-clustering
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-player-graph
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.cores=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.driver.host=spark-player-graph-driver
|
||||
- --conf
|
||||
- spark.driver.port=44167
|
||||
- --conf
|
||||
- spark.blockManager.port=44777
|
||||
- --class
|
||||
- de.nowchess.analytics.PlayerGraphJob
|
||||
- file:///app/analytics.jar
|
||||
- /spark-output/player-graph
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-player-graph
|
||||
- patch: |-
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: spark-live-dashboard
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.cores=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.driver.host=spark-live-dashboard-driver
|
||||
- --conf
|
||||
- spark.driver.port=37007
|
||||
- --conf
|
||||
- spark.blockManager.port=37777
|
||||
- --class
|
||||
- de.nowchess.analytics.LiveDashboardJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-checkpoint
|
||||
target:
|
||||
kind: Deployment
|
||||
name: spark-live-dashboard
|
||||
@@ -3,3 +3,24 @@ kind: Kustomization
|
||||
namespace: nowchess
|
||||
resources:
|
||||
- ../base
|
||||
patches:
|
||||
- patch: |-
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: spark-master
|
||||
spec:
|
||||
replicas: 0
|
||||
target:
|
||||
kind: Deployment
|
||||
name: spark-master
|
||||
- patch: |-
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: spark-worker
|
||||
spec:
|
||||
replicas: 0
|
||||
target:
|
||||
kind: Deployment
|
||||
name: spark-worker
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: nowchess-staging
|
||||
resources:
|
||||
- ../base
|
||||
patches:
|
||||
- patch: |-
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: spark-master-ingress
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
||||
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
|
||||
spec:
|
||||
rules:
|
||||
- host: spark-st.nowchess.janis-eccarius.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: spark-master
|
||||
port:
|
||||
number: 8080
|
||||
tls:
|
||||
- hosts:
|
||||
- spark-st.nowchess.janis-eccarius.de
|
||||
secretName: spark-cluster-staging-cert
|
||||
target:
|
||||
kind: Ingress
|
||||
name: spark-master-ingress
|
||||
- patch: |-
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: spark-history-ingress
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
||||
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
|
||||
spec:
|
||||
rules:
|
||||
- host: spark-history-st.nowchess.janis-eccarius.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: spark-history-server-svc
|
||||
port:
|
||||
number: 18080
|
||||
tls:
|
||||
- hosts:
|
||||
- spark-history-st.nowchess.janis-eccarius.de
|
||||
secretName: spark-history-staging-cert
|
||||
target:
|
||||
kind: Ingress
|
||||
name: spark-history-ingress
|
||||
Reference in New Issue
Block a user