diff --git a/spark-cluster/base/history-server.yaml b/spark-cluster/base/history-server.yaml index c14633f..1c7e7cc 100755 --- a/spark-cluster/base/history-server.yaml +++ b/spark-cluster/base/history-server.yaml @@ -18,10 +18,9 @@ spec: spec: containers: - name: history-server - # Same image version the Bitnami chart deploys for master/workers. - image: docker.io/bitnami/spark:3.5.2 + image: apache/spark:3.5.4-scala2.13-java17-ubuntu command: - - /opt/bitnami/spark/bin/spark-class + - /opt/spark/bin/spark-class - org.apache.spark.deploy.history.HistoryServer env: - name: SPARK_HISTORY_OPTS diff --git a/spark-cluster/base/ingress.yaml b/spark-cluster/base/ingress.yaml index 008cc10..2fa23be 100755 --- a/spark-cluster/base/ingress.yaml +++ b/spark-cluster/base/ingress.yaml @@ -15,9 +15,9 @@ spec: pathType: Prefix backend: service: - name: spark-master-svc + name: spark-master port: - number: 80 + number: 8080 --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -26,11 +26,6 @@ metadata: labels: app.kubernetes.io/name: spark-cluster app.kubernetes.io/part-of: nowchess - annotations: - # NOTE: The Bitnami spark chart 9.2.10 does not include a standalone history - # server deployment or service. This Ingress is a placeholder. To activate - # history server access, deploy a separate spark-history-server Deployment - # and Service named spark-history-server-svc on port 18080. spec: ingressClassName: traefik rules: diff --git a/spark-cluster/base/kustomization.yaml b/spark-cluster/base/kustomization.yaml index 3bce769..bf969d5 100755 --- a/spark-cluster/base/kustomization.yaml +++ b/spark-cluster/base/kustomization.yaml @@ -3,58 +3,7 @@ kind: Kustomization resources: - pvc.yaml + - master.yaml + - workers.yaml - history-server.yaml - ingress.yaml - -helmCharts: - - name: spark - repo: https://charts.bitnami.com/bitnami - version: "9.2.10" - releaseName: spark - namespace: nowchess - valuesInline: - master: - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - extraVolumes: - - name: spark-events - persistentVolumeClaim: - claimName: spark-events - extraVolumeMounts: - - name: spark-events - mountPath: /spark-events - worker: - replicaCount: 2 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - extraVolumes: - - name: spark-events - persistentVolumeClaim: - claimName: spark-events - extraVolumeMounts: - - name: spark-events - mountPath: /spark-events - historyServer: - enabled: true - eventsDir: /spark-events - extraVolumes: - - name: spark-events - persistentVolumeClaim: - claimName: spark-events - extraVolumeMounts: - - name: spark-events - mountPath: /spark-events - ingress: - enabled: false - service: - type: ClusterIP diff --git a/spark-cluster/base/master.yaml b/spark-cluster/base/master.yaml new file mode 100755 index 0000000..79dc0f1 --- /dev/null +++ b/spark-cluster/base/master.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spark-master + labels: + app.kubernetes.io/name: spark-master + app.kubernetes.io/part-of: nowchess +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: spark-master + template: + metadata: + labels: + app.kubernetes.io/name: spark-master + app.kubernetes.io/part-of: nowchess + spec: + containers: + - name: spark-master + image: apache/spark:3.5.4-scala2.13-java17-ubuntu + command: + - /opt/spark/bin/spark-class + - org.apache.spark.deploy.master.Master + ports: + - name: cluster + containerPort: 7077 + - name: webui + containerPort: 8080 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + volumeMounts: + - name: spark-events + mountPath: /spark-events + volumes: + - name: spark-events + persistentVolumeClaim: + claimName: spark-events +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-master + labels: + app.kubernetes.io/name: spark-master + app.kubernetes.io/part-of: nowchess +spec: + selector: + app.kubernetes.io/name: spark-master + ports: + - name: cluster + port: 7077 + targetPort: cluster + - name: webui + port: 8080 + targetPort: webui + type: ClusterIP diff --git a/spark-cluster/base/workers.yaml b/spark-cluster/base/workers.yaml new file mode 100755 index 0000000..7cf572e --- /dev/null +++ b/spark-cluster/base/workers.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spark-worker + labels: + app.kubernetes.io/name: spark-worker + app.kubernetes.io/part-of: nowchess +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: spark-worker + template: + metadata: + labels: + app.kubernetes.io/name: spark-worker + app.kubernetes.io/part-of: nowchess + spec: + containers: + - name: spark-worker + image: apache/spark:3.5.4-scala2.13-java17-ubuntu + command: + - /opt/spark/bin/spark-class + - org.apache.spark.deploy.worker.Worker + - spark://spark-master:7077 + env: + - name: SPARK_WORKER_CORES + value: "1" + - name: SPARK_WORKER_MEMORY + value: "1g" + ports: + - name: webui + containerPort: 8081 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + volumeMounts: + - name: spark-events + mountPath: /spark-events + volumes: + - name: spark-events + persistentVolumeClaim: + claimName: spark-events