diff --git a/eu-central-1/argo-apps/spark-analytics/spark-analytics.yaml b/eu-central-1/argo-apps/spark-analytics/spark-analytics.yaml new file mode 100755 index 0000000..1809544 --- /dev/null +++ b/eu-central-1/argo-apps/spark-analytics/spark-analytics.yaml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: spark-analytics + namespace: argocd +spec: + project: default + destination: + namespace: nowchess + server: https://kubernetes.default.svc + sources: + - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git + path: ./spark-analytics/eu-central-1-prod + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/spark-analytics/base/cronjobs.yaml b/spark-analytics/base/cronjobs.yaml new file mode 100755 index 0000000..460e39d --- /dev/null +++ b/spark-analytics/base/cronjobs.yaml @@ -0,0 +1,329 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-opening-book + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + # spark.kubernetes.container.image must match the analytics image tag set in the overlay + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.OpeningBookJob + - local:///app/analytics.jar + - /spark-output/opening-book + - "10" + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 500m + memory: 768Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-player-stats + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 3 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + # spark.kubernetes.container.image must match the analytics image tag set in the overlay + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.PlayerStatsJob + - local:///app/analytics.jar + - /spark-output/player-stats + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 500m + memory: 768Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-player-clustering + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 4 * * 0" + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + # spark.kubernetes.container.image must match the analytics image tag set in the overlay + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.PlayerClusteringJob + - local:///app/analytics.jar + - /spark-output/player-clusters + - "4" + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 500m + memory: 768Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-player-graph + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 5 * * 0" + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + # spark.kubernetes.container.image must match the analytics image tag set in the overlay + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.PlayerGraphJob + - local:///app/analytics.jar + - /spark-output/player-graph + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 500m + memory: 768Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output diff --git a/spark-analytics/base/kustomization.yaml b/spark-analytics/base/kustomization.yaml new file mode 100755 index 0000000..1a2aa6b --- /dev/null +++ b/spark-analytics/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceaccount.yaml + - rbac.yaml + - pvc.yaml + - cronjobs.yaml + - streaming.yaml diff --git a/spark-analytics/base/pvc.yaml b/spark-analytics/base/pvc.yaml new file mode 100755 index 0000000..307214c --- /dev/null +++ b/spark-analytics/base/pvc.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: spark-analytics-output + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: spark-streaming-checkpoint + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/spark-analytics/base/rbac.yaml b/spark-analytics/base/rbac.yaml new file mode 100755 index 0000000..ecc046a --- /dev/null +++ b/spark-analytics/base/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: spark-analytics-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete", "patch"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark-analytics-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +subjects: + - kind: ServiceAccount + name: spark-analytics +roleRef: + kind: Role + name: spark-analytics-driver + apiGroup: rbac.authorization.k8s.io diff --git a/spark-analytics/base/serviceaccount.yaml b/spark-analytics/base/serviceaccount.yaml new file mode 100755 index 0000000..7632108 --- /dev/null +++ b/spark-analytics/base/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark-analytics + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess diff --git a/spark-analytics/base/streaming.yaml b/spark-analytics/base/streaming.yaml new file mode 100755 index 0000000..5b964fb --- /dev/null +++ b/spark-analytics/base/streaming.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spark-live-dashboard + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + replicas: 1 + selector: + matchLabels: + app: spark-live-dashboard + template: + metadata: + labels: + app: spark-live-dashboard + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + # spark.kubernetes.container.image must match the analytics image tag set in the overlay + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:latest + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.LiveDashboardJob + - local:///app/analytics.jar + - /spark-checkpoint + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 500m + memory: 768Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-checkpoint + mountPath: /spark-checkpoint + volumes: + - name: spark-checkpoint + persistentVolumeClaim: + claimName: spark-streaming-checkpoint diff --git a/spark-analytics/eu-central-1-prod/kustomization.yaml b/spark-analytics/eu-central-1-prod/kustomization.yaml new file mode 100755 index 0000000..e03aea4 --- /dev/null +++ b/spark-analytics/eu-central-1-prod/kustomization.yaml @@ -0,0 +1,206 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: nowchess +resources: + - ../base +images: + - name: ghcr.io/now-chess/now-chess-systems/analytics + newTag: 0.1.0 +patches: + # Update the spark.kubernetes.container.image conf argument in each CronJob + # and the Deployment so executor pods pull the same pinned tag as the driver. + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-opening-book + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0 + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.OpeningBookJob + - local:///app/analytics.jar + - /spark-output/opening-book + - "10" + target: + kind: CronJob + name: spark-opening-book + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-player-stats + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0 + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.PlayerStatsJob + - local:///app/analytics.jar + - /spark-output/player-stats + target: + kind: CronJob + name: spark-player-stats + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-player-clustering + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0 + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.PlayerClusteringJob + - local:///app/analytics.jar + - /spark-output/player-clusters + - "4" + target: + kind: CronJob + name: spark-player-clustering + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-player-graph + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0 + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=2 + - --conf + - spark.executor.memory=1g + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.PlayerGraphJob + - local:///app/analytics.jar + - /spark-output/player-graph + target: + kind: CronJob + name: spark-player-graph + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: spark-live-dashboard + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - k8s://https://kubernetes.default.svc:443 + - --deploy-mode + - client + - --conf + - spark.kubernetes.container.image=ghcr.io/now-chess/now-chess-systems/analytics:0.1.0 + - --conf + - spark.kubernetes.namespace=nowchess + - --conf + - spark.kubernetes.authenticate.driver.serviceAccountName=spark-analytics + - --conf + - spark.kubernetes.executor.deleteOnTermination=true + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --class + - de.nowchess.analytics.LiveDashboardJob + - local:///app/analytics.jar + - /spark-checkpoint + target: + kind: Deployment + name: spark-live-dashboard