diff --git a/spark-analytics/base/cronjobs.yaml b/spark-analytics/base/cronjobs.yaml index e4e247b..bd9aefb 100755 --- a/spark-analytics/base/cronjobs.yaml +++ b/spark-analytics/base/cronjobs.yaml @@ -1,3 +1,79 @@ +apiVersion: v1 +kind: Service +metadata: + name: spark-opening-book-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-opening-book + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-player-stats-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-player-stats + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-player-clustering-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-player-clustering + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-player-graph-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-player-graph + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- apiVersion: batch/v1 kind: CronJob metadata: @@ -15,6 +91,7 @@ spec: template: metadata: labels: + app: spark-opening-book app.kubernetes.io/name: spark-analytics app.kubernetes.io/part-of: nowchess spec: @@ -32,13 +109,19 @@ spec: - --master - spark://spark-master:7077 - --conf - - spark.executor.instances=2 + - spark.executor.instances=1 - --conf - - spark.executor.cores=2 + - spark.executor.cores=1 - --conf - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-opening-book-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.OpeningBookJob - local:///app/analytics.jar @@ -92,6 +175,7 @@ spec: template: metadata: labels: + app: spark-player-stats app.kubernetes.io/name: spark-analytics app.kubernetes.io/part-of: nowchess spec: @@ -109,13 +193,19 @@ spec: - --master - spark://spark-master:7077 - --conf - - spark.executor.instances=2 + - spark.executor.instances=1 - --conf - - spark.executor.cores=2 + - spark.executor.cores=1 - --conf - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-stats-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.PlayerStatsJob - local:///app/analytics.jar @@ -168,6 +258,7 @@ spec: template: metadata: labels: + app: spark-player-clustering app.kubernetes.io/name: spark-analytics app.kubernetes.io/part-of: nowchess spec: @@ -185,13 +276,19 @@ spec: - --master - spark://spark-master:7077 - --conf - - spark.executor.instances=2 + - spark.executor.instances=1 - --conf - - spark.executor.cores=2 + - spark.executor.cores=1 - --conf - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-clustering-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.PlayerClusteringJob - local:///app/analytics.jar @@ -245,6 +342,7 @@ spec: template: metadata: labels: + app: spark-player-graph app.kubernetes.io/name: spark-analytics app.kubernetes.io/part-of: nowchess spec: @@ -262,13 +360,19 @@ spec: - --master - spark://spark-master:7077 - --conf - - spark.executor.instances=2 + - spark.executor.instances=1 - --conf - - spark.executor.cores=2 + - spark.executor.cores=1 - --conf - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-graph-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.PlayerGraphJob - local:///app/analytics.jar diff --git a/spark-analytics/eu-central-1-prod/kustomization.yaml b/spark-analytics/eu-central-1-prod/kustomization.yaml index c326dcb..146ed63 100755 --- a/spark-analytics/eu-central-1-prod/kustomization.yaml +++ b/spark-analytics/eu-central-1-prod/kustomization.yaml @@ -33,6 +33,12 @@ patches: - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-opening-book-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.OpeningBookJob - local:///app/analytics.jar @@ -65,6 +71,12 @@ patches: - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-stats-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.PlayerStatsJob - local:///app/analytics.jar @@ -96,6 +108,12 @@ patches: - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-clustering-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.PlayerClusteringJob - local:///app/analytics.jar @@ -128,6 +146,12 @@ patches: - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-player-graph-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 - --class - de.nowchess.analytics.PlayerGraphJob - local:///app/analytics.jar @@ -152,9 +176,17 @@ patches: - --conf - spark.executor.instances=1 - --conf + - spark.executor.cores=1 + - --conf - spark.executor.memory=512m - --conf - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-live-dashboard-driver + - --conf + - spark.driver.port=37007 + - --conf + - spark.blockManager.port=37777 - --class - de.nowchess.analytics.LiveDashboardJob - local:///app/analytics.jar