feat(spark): replace Bitnami chart with Apache Spark manifests

Bitnami Spark chart requires a Broadcom commercial license. Replace with
plain Deployments using apache/spark:3.5.4-scala2.13-java17-ubuntu
(Apache 2.0, same image used by the analytics jobs):
- spark-master: Deployment + ClusterIP Service (ports 7077/8080)
- spark-worker: Deployment, 2 replicas, connects to spark-master:7077
- spark-history-server: updated to apache/spark image
Ingress backend corrected to spark-master:8080.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

spark-cluster/base/history-server.yaml

@@ -18,10 +18,9 @@ spec:
     spec:
       containers:
         - name: history-server
-          # Same image version the Bitnami chart deploys for master/workers.
+          image: apache/spark:3.5.4-scala2.13-java17-ubuntu
-          image: docker.io/bitnami/spark:3.5.2
           command:
-            - /opt/bitnami/spark/bin/spark-class
+            - /opt/spark/bin/spark-class
             - org.apache.spark.deploy.history.HistoryServer
           env:
             - name: SPARK_HISTORY_OPTS

spark-cluster/base/ingress.yaml

@@ -15,9 +15,9 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: spark-master-svc
+                name: spark-master
                 port:
-                  number: 80
+                  number: 8080
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -26,11 +26,6 @@ metadata:
   labels:
     app.kubernetes.io/name: spark-cluster
     app.kubernetes.io/part-of: nowchess
-  annotations:
-    # NOTE: The Bitnami spark chart 9.2.10 does not include a standalone history
-    # server deployment or service. This Ingress is a placeholder. To activate
-    # history server access, deploy a separate spark-history-server Deployment
-    # and Service named spark-history-server-svc on port 18080.
 spec:
   ingressClassName: traefik
   rules:

spark-cluster/base/kustomization.yaml

@@ -3,58 +3,7 @@ kind: Kustomization
 
 resources:
   - pvc.yaml
+  - master.yaml
+  - workers.yaml
   - history-server.yaml
   - ingress.yaml
-
-helmCharts:
-  - name: spark
-    repo: https://charts.bitnami.com/bitnami
-    version: "9.2.10"
-    releaseName: spark
-    namespace: nowchess
-    valuesInline:
-      master:
-        resources:
-          requests:
-            cpu: 500m
-            memory: 1Gi
-          limits:
-            cpu: "1"
-            memory: 2Gi
-        extraVolumes:
-          - name: spark-events
-            persistentVolumeClaim:
-              claimName: spark-events
-        extraVolumeMounts:
-          - name: spark-events
-            mountPath: /spark-events
-      worker:
-        replicaCount: 2
-        resources:
-          requests:
-            cpu: 500m
-            memory: 1Gi
-          limits:
-            cpu: "1"
-            memory: 2Gi
-        extraVolumes:
-          - name: spark-events
-            persistentVolumeClaim:
-              claimName: spark-events
-        extraVolumeMounts:
-          - name: spark-events
-            mountPath: /spark-events
-      historyServer:
-        enabled: true
-        eventsDir: /spark-events
-        extraVolumes:
-          - name: spark-events
-            persistentVolumeClaim:
-              claimName: spark-events
-        extraVolumeMounts:
-          - name: spark-events
-            mountPath: /spark-events
-      ingress:
-        enabled: false
-      service:
-        type: ClusterIP

spark-cluster/base/master.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-master
+  labels:
+    app.kubernetes.io/name: spark-master
+    app.kubernetes.io/part-of: nowchess
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: spark-master
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: spark-master
+        app.kubernetes.io/part-of: nowchess
+    spec:
+      containers:
+        - name: spark-master
+          image: apache/spark:3.5.4-scala2.13-java17-ubuntu
+          command:
+            - /opt/spark/bin/spark-class
+            - org.apache.spark.deploy.master.Master
+          ports:
+            - name: cluster
+              containerPort: 7077
+            - name: webui
+              containerPort: 8080
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "1"
+              memory: 2Gi
+          volumeMounts:
+            - name: spark-events
+              mountPath: /spark-events
+      volumes:
+        - name: spark-events
+          persistentVolumeClaim:
+            claimName: spark-events
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-master
+  labels:
+    app.kubernetes.io/name: spark-master
+    app.kubernetes.io/part-of: nowchess
+spec:
+  selector:
+    app.kubernetes.io/name: spark-master
+  ports:
+    - name: cluster
+      port: 7077
+      targetPort: cluster
+    - name: webui
+      port: 8080
+      targetPort: webui
+  type: ClusterIP

spark-cluster/base/workers.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-worker
+  labels:
+    app.kubernetes.io/name: spark-worker
+    app.kubernetes.io/part-of: nowchess
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: spark-worker
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: spark-worker
+        app.kubernetes.io/part-of: nowchess
+    spec:
+      containers:
+        - name: spark-worker
+          image: apache/spark:3.5.4-scala2.13-java17-ubuntu
+          command:
+            - /opt/spark/bin/spark-class
+            - org.apache.spark.deploy.worker.Worker
+            - spark://spark-master:7077
+          env:
+            - name: SPARK_WORKER_CORES
+              value: "1"
+            - name: SPARK_WORKER_MEMORY
+              value: "1g"
+          ports:
+            - name: webui
+              containerPort: 8081
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "1"
+              memory: 2Gi
+          volumeMounts:
+            - name: spark-events
+              mountPath: /spark-events
+      volumes:
+        - name: spark-events
+          persistentVolumeClaim:
+            claimName: spark-events