feat(nci-11): add Kubernetes manifests for nowchess-analysis service (#392)
Co-authored-by: Janis Eccarius <eccariusjanis@gmail.com> Reviewed-on: #392
This commit was merged in pull request #392.
This commit is contained in:
Executable
+321
@@ -0,0 +1,321 @@
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-opening-book
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
schedule: "0 2 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
serviceAccountName: spark-analytics
|
||||
restartPolicy: OnFailure
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull-secret
|
||||
containers:
|
||||
- name: spark-driver
|
||||
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=2
|
||||
- --conf
|
||||
- spark.executor.memory=1g
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.eventLog.enabled=true
|
||||
- --conf
|
||||
- spark.eventLog.dir=/spark-events
|
||||
- --class
|
||||
- de.nowchess.analytics.OpeningBookJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-output/opening-book
|
||||
- "10"
|
||||
env:
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: spark-output
|
||||
mountPath: /spark-output
|
||||
- name: spark-events
|
||||
mountPath: /spark-events
|
||||
volumes:
|
||||
- name: spark-output
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-analytics-output
|
||||
- name: spark-events
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-events
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-player-stats
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
schedule: "0 3 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
serviceAccountName: spark-analytics
|
||||
restartPolicy: OnFailure
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull-secret
|
||||
containers:
|
||||
- name: spark-driver
|
||||
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=2
|
||||
- --conf
|
||||
- spark.executor.memory=1g
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.eventLog.enabled=true
|
||||
- --conf
|
||||
- spark.eventLog.dir=/spark-events
|
||||
- --class
|
||||
- de.nowchess.analytics.PlayerStatsJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-output/player-stats
|
||||
env:
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: spark-output
|
||||
mountPath: /spark-output
|
||||
- name: spark-events
|
||||
mountPath: /spark-events
|
||||
volumes:
|
||||
- name: spark-output
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-analytics-output
|
||||
- name: spark-events
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-events
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-player-clustering
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
schedule: "0 4 * * 0"
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
serviceAccountName: spark-analytics
|
||||
restartPolicy: OnFailure
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull-secret
|
||||
containers:
|
||||
- name: spark-driver
|
||||
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=2
|
||||
- --conf
|
||||
- spark.executor.memory=1g
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.eventLog.enabled=true
|
||||
- --conf
|
||||
- spark.eventLog.dir=/spark-events
|
||||
- --class
|
||||
- de.nowchess.analytics.PlayerClusteringJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-output/player-clusters
|
||||
- "4"
|
||||
env:
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: spark-output
|
||||
mountPath: /spark-output
|
||||
- name: spark-events
|
||||
mountPath: /spark-events
|
||||
volumes:
|
||||
- name: spark-output
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-analytics-output
|
||||
- name: spark-events
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-events
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-player-graph
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
schedule: "0 5 * * 0"
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
serviceAccountName: spark-analytics
|
||||
restartPolicy: OnFailure
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull-secret
|
||||
containers:
|
||||
- name: spark-driver
|
||||
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=2
|
||||
- --conf
|
||||
- spark.executor.memory=1g
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.eventLog.enabled=true
|
||||
- --conf
|
||||
- spark.eventLog.dir=/spark-events
|
||||
- --class
|
||||
- de.nowchess.analytics.PlayerGraphJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-output/player-graph
|
||||
env:
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: spark-output
|
||||
mountPath: /spark-output
|
||||
- name: spark-events
|
||||
mountPath: /spark-events
|
||||
volumes:
|
||||
- name: spark-output
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-analytics-output
|
||||
- name: spark-events
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-events
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- serviceaccount.yaml
|
||||
- rbac.yaml
|
||||
- pvc.yaml
|
||||
- cronjobs.yaml
|
||||
- streaming.yaml
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: spark-analytics-output
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: spark-streaming-checkpoint
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: spark-analytics-driver
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["create", "get", "list", "watch", "delete", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["create", "get", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["create", "get", "list", "delete"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: spark-analytics-driver
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: spark-analytics
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: spark-analytics-driver
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: spark-analytics
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: spark-live-dashboard
|
||||
labels:
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: spark-live-dashboard
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: spark-live-dashboard
|
||||
app.kubernetes.io/name: spark-analytics
|
||||
app.kubernetes.io/part-of: nowchess
|
||||
spec:
|
||||
serviceAccountName: spark-analytics
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull-secret
|
||||
containers:
|
||||
- name: spark-driver
|
||||
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
|
||||
command:
|
||||
- /opt/spark/bin/spark-submit
|
||||
- --master
|
||||
- spark://spark-master:7077
|
||||
- --conf
|
||||
- spark.executor.instances=1
|
||||
- --conf
|
||||
- spark.executor.memory=512m
|
||||
- --conf
|
||||
- spark.driver.memory=512m
|
||||
- --conf
|
||||
- spark.eventLog.enabled=true
|
||||
- --conf
|
||||
- spark.eventLog.dir=/spark-events
|
||||
- --class
|
||||
- de.nowchess.analytics.LiveDashboardJob
|
||||
- local:///app/analytics.jar
|
||||
- /spark-checkpoint
|
||||
env:
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: spark-checkpoint
|
||||
mountPath: /spark-checkpoint
|
||||
- name: spark-events
|
||||
mountPath: /spark-events
|
||||
volumes:
|
||||
- name: spark-checkpoint
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-streaming-checkpoint
|
||||
- name: spark-events
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-events
|
||||
Reference in New Issue
Block a user