diff --git a/eu-central-1/argo-apps/spark-analytics/spark-analytics-staging.yaml b/eu-central-1/argo-apps/spark-analytics/spark-analytics-staging.yaml new file mode 100644 index 0000000..80677a6 --- /dev/null +++ b/eu-central-1/argo-apps/spark-analytics/spark-analytics-staging.yaml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: spark-analytics-staging + namespace: argocd +spec: + project: default + destination: + namespace: nowchess-staging + server: https://kubernetes.default.svc + sources: + - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git + path: ./spark-analytics/eu-central-1-staging + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/eu-central-1/argo-apps/spark-cluster/spark-cluster-staging.yaml b/eu-central-1/argo-apps/spark-cluster/spark-cluster-staging.yaml new file mode 100644 index 0000000..b25d7d4 --- /dev/null +++ b/eu-central-1/argo-apps/spark-cluster/spark-cluster-staging.yaml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: spark-cluster-staging + namespace: argocd +spec: + project: default + destination: + namespace: nowchess-staging + server: https://kubernetes.default.svc + sources: + - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git + path: ./spark-cluster/eu-central-1-staging + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/spark-analytics/eu-central-1-staging/kustomization.yaml b/spark-analytics/eu-central-1-staging/kustomization.yaml new file mode 100644 index 0000000..fa8ee6e --- /dev/null +++ b/spark-analytics/eu-central-1-staging/kustomization.yaml @@ -0,0 +1,194 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: nowchess-staging +resources: + - ../base +images: + - name: ghcr.io/now-chess/now-chess-systems/analytics + newTag: 0.2.0 +patches: + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-opening-book + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-opening-book-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.OpeningBookJob + - file:///app/analytics.jar + - /spark-output/opening-book + - "10" + target: + kind: CronJob + name: spark-opening-book + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-player-stats + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-stats-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.PlayerStatsJob + - file:///app/analytics.jar + - /spark-output/player-stats + target: + kind: CronJob + name: spark-player-stats + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-player-clustering + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-clustering-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.PlayerClusteringJob + - file:///app/analytics.jar + - /spark-output/player-clusters + - "4" + target: + kind: CronJob + name: spark-player-clustering + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-player-graph + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-graph-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.PlayerGraphJob + - file:///app/analytics.jar + - /spark-output/player-graph + target: + kind: CronJob + name: spark-player-graph + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: spark-live-dashboard + spec: + template: + spec: + containers: + - name: spark-driver + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-live-dashboard-driver + - --conf + - spark.driver.port=37007 + - --conf + - spark.blockManager.port=37777 + - --class + - de.nowchess.analytics.LiveDashboardJob + - local:///app/analytics.jar + - /spark-checkpoint + target: + kind: Deployment + name: spark-live-dashboard diff --git a/spark-cluster/eu-central-1-prod/kustomization.yaml b/spark-cluster/eu-central-1-prod/kustomization.yaml index a5cd04c..76e3f11 100755 --- a/spark-cluster/eu-central-1-prod/kustomization.yaml +++ b/spark-cluster/eu-central-1-prod/kustomization.yaml @@ -3,3 +3,24 @@ kind: Kustomization namespace: nowchess resources: - ../base +patches: + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: spark-master + spec: + replicas: 0 + target: + kind: Deployment + name: spark-master + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: spark-worker + spec: + replicas: 0 + target: + kind: Deployment + name: spark-worker diff --git a/spark-cluster/eu-central-1-staging/kustomization.yaml b/spark-cluster/eu-central-1-staging/kustomization.yaml new file mode 100644 index 0000000..1e9d95d --- /dev/null +++ b/spark-cluster/eu-central-1-staging/kustomization.yaml @@ -0,0 +1,62 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: nowchess-staging +resources: + - ../base +patches: + - patch: |- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: spark-master-ingress + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + spec: + rules: + - host: spark-st.nowchess.janis-eccarius.de + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: spark-master + port: + number: 8080 + tls: + - hosts: + - spark-st.nowchess.janis-eccarius.de + secretName: spark-cluster-staging-cert + target: + kind: Ingress + name: spark-master-ingress + - patch: |- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: spark-history-ingress + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + spec: + rules: + - host: spark-history-st.nowchess.janis-eccarius.de + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: spark-history-server-svc + port: + number: 18080 + tls: + - hosts: + - spark-history-st.nowchess.janis-eccarius.de + secretName: spark-history-staging-cert + target: + kind: Ingress + name: spark-history-ingress