chore(kargo): update image tag official-bots to 0.36.0 on eu-central-1-prod

This reverts commit af3310297b.

.claude/agent-memory/gitops-k8s-specialist/MEMORY.md

@@ -0,0 +1 @@
+- [Spark cluster deployment](project-spark-cluster.md) — chart 9.2.10 service names, history server gap, hostname pattern, standalone mode migration

.claude/agent-memory/gitops-k8s-specialist/project-spark-cluster.md

@@ -0,0 +1,22 @@
+---
+name: project-spark-cluster
+description: Spark cluster deployment facts — chart version, service names, history server gap, hostname pattern
+metadata:
+  type: project
+---
+
+Bitnami Spark chart 9.2.10 (Spark 3.5.2) is pinned in `spark-cluster/base/kustomization.yaml`.
+
+**Chart service names (rendered with releaseName=spark):**
+- `spark-master-svc` — master Web UI on port 80 (targetPort: http/8080), cluster port 7077
+- `spark-headless` — headless service for all nodes
+
+**History server gap:** Chart 9.2.10 has NO standalone history server Deployment/Service. The `historyServer` chart key does not exist in this version. The `spark-history-ingress` in `spark-cluster/base/ingress.yaml` is a placeholder pointing to `spark-history-server-svc:18080` — this service does not exist until a separate Deployment is added.
+
+**Why:** The Bitnami chart 9.2.10 only renders master + worker StatefulSets. A real history server requires either a newer chart version that adds it, or a hand-rolled Deployment.
+
+**Hostnames:** `*.nowchess.janis-eccarius.de` — master UI at `spark.nowchess.janis-eccarius.de`, history at `spark-history.nowchess.janis-eccarius.de`. Ingress class: `traefik` (k3d default, different from production overlays which use `nginx`).
+
+**Spark standalone mode:** All analytics CronJobs and the streaming Deployment use `spark://spark-master:7077` (not `k8s://https://kubernetes.default.svc:443`). Event logging enabled to `/spark-events` PVC (`spark-events`, RWO/local-path compatible).
+
+**How to apply:** When working on Spark-related NCI tickets, check `spark-cluster/base/` for chart version and service names before writing ingress or spark-submit commands. If history server is needed, a new Deployment + Service named `spark-history-server-svc` on port 18080 must be added to the base.

eu-central-1/argo-apps/kargo/kargo.yaml

@@ -11,7 +11,7 @@ spec:
   sources:
     - repoURL: ghcr.io/akuity/kargo-charts
       chart: kargo
-      targetRevision: 1.10.7
+      targetRevision: 1.10.8
       helm:
         valueFiles:
           - $values/kargo/eu-central-1/values.yaml

eu-central-1/argo-apps/sealed-secrets/sealed-secrets.yaml

@@ -9,9 +9,9 @@ spec:
     namespace: kube-system
     server: https://kubernetes.default.svc
   sources:
-    - repoURL: https://bitnami-labs.github.io/sealed-secrets
+    - repoURL: https://charts.bitnami.com/bitnami
       chart: sealed-secrets
-      targetRevision: 2.16.1
+      targetRevision: 2.5.19
       helm:
         valueFiles:
           - $values/sealed-secrets/eu-central-1/values.yaml

eu-central-1/argo-apps/spark-analytics/spark-analytics-staging.yaml

@@ -0,0 +1,18 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: spark-analytics-staging
+  namespace: argocd
+spec:
+  project: default
+  destination:
+    namespace: nowchess-staging
+    server: https://kubernetes.default.svc
+  sources:
+    - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
+      path: ./spark-analytics/eu-central-1-staging
+      targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

eu-central-1/argo-apps/spark-analytics/spark-analytics.yaml

@@ -12,4 +12,7 @@ spec:
     - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
       path: ./spark-analytics/eu-central-1-prod
       targetRevision: main
-  syncPolicy: {}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

eu-central-1/argo-apps/spark-cluster/spark-cluster-staging.yaml

@@ -0,0 +1,18 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: spark-cluster-staging
+  namespace: argocd
+spec:
+  project: default
+  destination:
+    namespace: nowchess-staging
+    server: https://kubernetes.default.svc
+  sources:
+    - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
+      path: ./spark-cluster/eu-central-1-staging
+      targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

eu-central-1/argo-apps/spark-cluster/spark-cluster.yaml

@@ -12,4 +12,7 @@ spec:
     - repoURL: git@git.janis-eccarius.de:NowChess/Gitops.git
       path: ./spark-cluster/eu-central-1-prod
       targetRevision: main
-  syncPolicy: {}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

nowchess/base/nowchess-rollouts.yaml

@@ -440,7 +440,19 @@ spec:
                   name: ncs-internal-secret
                   key: INTERNAL_SECRET
             - name: ACCOUNT_SERVICE_URL
-              value: http://nowchess-account:8083
+              value: http://nowchess-account-active:8083
+            - name: TOURNAMENT_SERVICE_URL
+              value: http://nowchess-tournament-active:8088
+            - name: TOURNAMENT_SELF_URL
+              value: ""
+            - name: MP_JWT_VERIFY_PUBLICKEY_LOCATION
+              value: /secrets/jwt/public.pem
+            - name: MP_JWT_VERIFY_ISSUER
+              value: nowchess
+          volumeMounts:
+            - name: jwt-keys
+              mountPath: /secrets/jwt
+              readOnly: true
           ports:
             - containerPort: 8088
               protocol: TCP
@@ -463,6 +475,10 @@ spec:
             limits:
               cpu: "500m"
               memory: "256Mi"
+      volumes:
+        - name: jwt-keys
+          secret:
+            secretName: ncs-jwt-keys
   rollbackWindow:
     revisions: 3
   strategy:
@@ -875,6 +891,10 @@ spec:
                 secretKeyRef:
                   name: ncs-db-secrets
                   key: ACCOUNT_DB_URL
+            - name: TOURNAMENT_SELF_URL
+              value: ""
+            - name: TOURNAMENT_EXTERNAL_SERVERS
+              value: ""
             - name: MP_JWT_VERIFY_PUBLICKEY_LOCATION
               value: /secrets/jwt/public.pem
           volumeMounts:

nowchess/eu-central-1-prod/ingress.yaml

@@ -8,6 +8,8 @@ metadata:
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
     nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
     nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
 spec:
   ingressClassName: nginx
   rules:
@@ -56,6 +58,20 @@ spec:
                 name: nowchess-tournament-active
                 port:
                   number: 8088
+          - path: /api/bots
+            pathType: Prefix
+            backend:
+              service:
+                name: nowchess-official-bots-active
+                port:
+                  number: 8088
+          - path: /api/analysis
+            pathType: Prefix
+            backend:
+              service:
+                name: nowchess-analysis-active
+                port:
+                  number: 8087
           - path: /api
             pathType: Prefix
             backend:

nowchess/eu-central-1-prod/kustomization.yaml

@@ -33,7 +33,7 @@ patches:
     name: nowchess-frontend-env-config
 images:
 - name: ghcr.io/now-chess/now-chess-systems/account
-  newTag: 0.24.0
+  newTag: 0.26.0
 - name: ghcr.io/now-chess/now-chess-systems/analysis
   newTag: 0.2.0
 - name: ghcr.io/now-chess/now-chess-systems/bot-platform
@@ -43,19 +43,19 @@ images:
 - name: ghcr.io/now-chess/now-chess-systems/core
   newTag: 0.51.0
 - name: ghcr.io/now-chess/now-chess-systems/frontend
-  newTag: 0.3.0
+  newTag: 0.6.3
 - name: ghcr.io/now-chess/now-chess-systems/io
   newTag: 0.23.0
 - name: ghcr.io/now-chess/now-chess-systems/official-bots
-  newTag: 0.17.0
+  newTag: 0.36.0
 - name: ghcr.io/now-chess/now-chess-systems/rule
   newTag: 0.18.0
 - name: ghcr.io/now-chess/now-chess-systems/store
   newTag: 0.24.0
 - name: ghcr.io/now-chess/now-chess-systems/tournament
-  newTag: 0.3.0
+  newTag: 0.9.0
 - name: ghcr.io/now-chess/now-chess-systems/ws
-  newTag: 0.16.0
+  newTag: 0.17.0
 configurations:
 - rollout-transform.yaml
 openapi:

nowchess/eu-central-1-staging/ingress.yaml

@@ -7,6 +7,9 @@ metadata:
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
     nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
 spec:
   ingressClassName: nginx
   rules:
@@ -55,6 +58,20 @@ spec:
                 name: nowchess-tournament-active
                 port:
                   number: 8088
+          - path: /api/bots
+            pathType: Prefix
+            backend:
+              service:
+                name: nowchess-official-bots-active
+                port:
+                  number: 8088
+          - path: /api/analysis
+            pathType: Prefix
+            backend:
+              service:
+                name: nowchess-analysis-active
+                port:
+                  number: 8087
           - path: /api
             pathType: Prefix
             backend:

nowchess/eu-central-1-staging/kustomization.yaml

@@ -93,9 +93,24 @@ patches:
       name: nowchess-core
     spec:
       replicas: 1
+- patch: |-
+    apiVersion: argoproj.io/v1alpha1
+    kind: Rollout
+    metadata:
+      name: nowchess-official-bots
+    spec:
+      template:
+        spec:
+          containers:
+            - name: nowchess-official-bots
+              env:
+                - name: QUARKUS_LOG_CATEGORY__IO_SMALLRYE_JWT__LEVEL
+                  value: DEBUG
+                - name: QUARKUS_LOG_CATEGORY__IO_QUARKUS_SECURITY__LEVEL
+                  value: DEBUG
 images:
 - name: ghcr.io/now-chess/now-chess-systems/account
-  newTag: 0.24.0
+  newTag: 0.26.0
 - name: ghcr.io/now-chess/now-chess-systems/analysis
   newTag: 0.2.0
 - name: ghcr.io/now-chess/now-chess-systems/bot-platform
@@ -103,21 +118,21 @@ images:
 - name: ghcr.io/now-chess/now-chess-systems/coordinator
   newTag: 0.32.0
 - name: ghcr.io/now-chess/now-chess-systems/core
-  newTag: 0.51.0
+  newTag: 0.53.0
 - name: ghcr.io/now-chess/now-chess-systems/frontend
-  newTag: 0.3.0
+  newTag: 0.6.3
 - name: ghcr.io/now-chess/now-chess-systems/io
   newTag: 0.23.0
 - name: ghcr.io/now-chess/now-chess-systems/official-bots
-  newTag: 0.18.0
+  newTag: 0.40.0
 - name: ghcr.io/now-chess/now-chess-systems/rule
   newTag: 0.18.0
 - name: ghcr.io/now-chess/now-chess-systems/store
   newTag: 0.24.0
 - name: ghcr.io/now-chess/now-chess-systems/tournament
-  newTag: 0.3.0
+  newTag: 0.9.0
 - name: ghcr.io/now-chess/now-chess-systems/ws
-  newTag: 0.16.0
+  newTag: 0.17.0
 configurations:
 - rollout-transform.yaml
 openapi:

nowchess/htwg-1-prod/ingress.yaml

@@ -53,6 +53,20 @@ spec:
                 name: nowchess-tournament-active
                 port:
                   number: 8088
+          - path: /api/bots
+            pathType: Prefix
+            backend:
+              service:
+                name: nowchess-official-bots-active
+                port:
+                  number: 8088
+          - path: /api/analysis
+            pathType: Prefix
+            backend:
+              service:
+                name: nowchess-analysis-active
+                port:
+                  number: 8087
           - path: /api
             pathType: Prefix
             backend:

nowchess/htwg-1-prod/kustomization.yaml

@@ -12,10 +12,11 @@ patches:
       name: nowchess-env-config
     data:
       QUARKUS_PROFILE: deployed
-      CORS_ORIGINS: http://141.37.74.142
+      CORS_ORIGINS: http://141.37.123.122
       NOWCHESS_COORDINATOR_ENABLED: "false"
       REDIS_PREFIX: nowchess
-      QUARKUS_LOG_LEVEL: "WARN"
+      QUARKUS_LOG_LEVEL: "INFO"
+      OTEL_SDK_DISABLED: "true"
   target:
     kind: ConfigMap
     name: nowchess-env-config
@@ -25,14 +26,55 @@ patches:
     metadata:
       name: nowchess-frontend-env-config
     data:
-      API_URL: "http://141.37.74.142"
-      WEBSOCKET_URL: "ws://141.37.74.142"
+      API_URL: "http://141.37.123.122"
+      WEBSOCKET_URL: "ws://141.37.123.122"
   target:
     kind: ConfigMap
     name: nowchess-frontend-env-config
+- patch: |-
+    apiVersion: argoproj.io/v1alpha1
+    kind: Rollout
+    metadata:
+      name: nowchess-official-bots
+    spec:
+      template:
+        spec:
+          containers:
+            - name: nowchess-official-bots
+              env:
+                - name: TOURNAMENT_SERVICE_URL
+                  value: http://nowchess-tournament-active:8088
+                - name: TOURNAMENT_SELF_URL
+                  value: http://141.147.25.239
+                - name: TOURNAMENT_BOT_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-tournament-bot-secret
+                      key: TOURNAMENT_BOT_TOKEN
+  target:
+    kind: Rollout
+    name: nowchess-official-bots
+- patch: |-
+    apiVersion: argoproj.io/v1alpha1
+    kind: Rollout
+    metadata:
+      name: nowchess-tournament
+    spec:
+      template:
+        spec:
+          containers:
+            - name: nowchess-tournament
+              env:
+                - name: TOURNAMENT_SELF_URL
+                  value: http://141.147.25.239
+                - name: TOURNAMENT_EXTERNAL_SERVERS
+                  value: http://141.37.123.132:8086
+  target:
+    kind: Rollout
+    name: nowchess-tournament
 images:
 - name: ghcr.io/now-chess/now-chess-systems/account
-  newTag: 0.24.0-x86_v2
+  newTag: 0.26.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/analysis
   newTag: 0.1.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/bot-platform
@@ -40,18 +82,18 @@ images:
 - name: ghcr.io/now-chess/now-chess-systems/coordinator
   newTag: 0.32.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/core
-  newTag: 0.51.0-x86_v2
+  newTag: 0.53.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/frontend
-  newTag: 0.3.0-x86_v2
+  newTag: 0.6.3-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/io
   newTag: 0.23.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/official-bots
-  newTag: 0.18.0-x86_v2
+  newTag: 0.40.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/rule
   newTag: 0.18.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/store
   newTag: 0.24.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/tournament
-  newTag: 0.3.0-x86_v2
+  newTag: 0.9.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/ws
-  newTag: 0.16.0-x86_v2
+  newTag: 0.17.0-x86_v2

secrets/htwg-1/kustomization.yaml

@@ -10,3 +10,4 @@ resources:
   - ncs-internal-secret.yaml
   - remotek6-certs.yaml
   - postgres-credentials.yaml
+  - ncs-tournament-bot-secret.yaml

secrets/htwg-1/ncs-tournament-bot-secret.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: ncs-tournament-bot-secret
+  namespace: nowchess
+spec:
+  encryptedData:
+    TOURNAMENT_BOT_TOKEN: AgBeVXB71v/88ksndl+bn4k8niG7rlKCDxWEinMgxh0vqLjtB/BWwwhhRp7064/jFH7Qt02nb9TF8N4+GIiBNykIGQLQtUBzvMp7I7EGSAHNsFSlXFTURQQJo5+C8L/H9c0UDQ+gunqCinz2mEZw1LlnMgQEqfMN3RCVK9YtW6f76jk3WLpet4g082lfy9cou2FZ+AgQq3zSBf6//OmlqrKRLkve1Nbxy4dS2IAvfLd/OdiC29sFs41nk3TFyrR9jPg8P4cYfd99O9XhabKRrCR8J1TxovCiWWQVBI+E1mmbQW5d6pcw1qO2fYsRGvdVb7DB+IiV3WjMHO6U3T/TV8sHhhAUSMELtD0NIfXyphkevK7Mtd3wPHgk6U2vTpNH3kionywff8jBeM8xWmTCOnHgRlCRoUrigysbAKNwGKjqEdWM3En8qKf4o6aTxFoQODIa09esivqcXs6U7WzqXWxjrZDVwsjauzC+eUyvJQv+yDPHSMjUt35r/CZF9xuOtRPG0ZmKmO6yAlbgLBnSYiOeEa9xPxhh3n7+kV9TijHgnxcsGhfzpxzQLJ49g7TPkSqudC+wZtMKt719HrW3SPiUnIT521ntqvaBGc+O8+AA1lEB5OaIfN1yKjUZmy7zPZhHG7wtTVBaXisXV0qdw62/V9aejgClb2S7OvXx7tuA4hrwKMXNyZ+YlN+10g6mIaxD62naiWRbrXdKy1dpgqtqq4erOUdGX09twWJ1kUWb0azUNV6BJTUinyTjh8bzbdq3cJ+IYm8gSe8QovAe9vpwAt9rRGTE8+lHBzUcEzQZwNQfK8FqqvHa2wDbFTaVt5G6wVNtGb2DLvlcIMKe5zSads0W0rwKXEKl2YUvZtBdVkdQvJD3/GlC4zBa7/TmIqCj
+  template:
+    metadata:
+      name: ncs-tournament-bot-secret
+      namespace: nowchess

spark-analytics/base/kustomization.yaml

@@ -7,3 +7,4 @@ resources:
   - pvc.yaml
   - cronjobs.yaml
   - streaming.yaml
+  - webview.yaml

spark-analytics/base/streaming.yaml

@@ -25,7 +25,7 @@ metadata:
     app.kubernetes.io/name: spark-analytics
     app.kubernetes.io/part-of: nowchess
 spec:
-  replicas: 1
+  replicas: 0
   strategy:
     type: Recreate
   selector:
@@ -53,6 +53,8 @@ spec:
             - --conf
             - spark.executor.instances=1
             - --conf
+            - spark.executor.cores=1
+            - --conf
             - spark.executor.memory=512m
             - --conf
             - spark.driver.memory=512m
@@ -64,7 +66,7 @@ spec:
             - spark.blockManager.port=37777
             - --class
             - de.nowchess.analytics.LiveDashboardJob
-            - local:///app/analytics.jar
+            - file:///app/analytics.jar
             - /spark-checkpoint
           env:
             - name: NOWCHESS_JDBC_URL

spark-analytics/base/webview.yaml

@@ -0,0 +1,741 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: spark-analytics-webview
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+data:
+  serve.py: |
+    #!/usr/bin/env python3
+    """Spark analytics results viewer — queries PostgreSQL analytics tables, serves HTML tables."""
+    import html
+    import json
+    import os
+    import re
+    import urllib.error
+    import urllib.request
+    import psycopg2
+    import psycopg2.extras
+    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+    PORT = int(os.environ.get("PORT", "8080"))
+
+    _url = os.environ.get("NOWCHESS_JDBC_URL", "")
+    _m = re.match(r"jdbc:postgresql://([^:/]+)(?::(\d+))?/([^?]+)", _url)
+    DB_HOST = _m.group(1) if _m else "localhost"
+    DB_PORT = int(_m.group(2)) if (_m and _m.group(2)) else 5432
+    DB_NAME = _m.group(3) if _m else "nowchess"
+    DB_USER = os.environ.get("NOWCHESS_DB_USER", "nowchess")
+    DB_PASS = os.environ.get("NOWCHESS_DB_PASS", "")
+
+    # Each tuple: (url-slug, display label, postgres table name)
+    DATASETS = [
+        ("opening-book",          "Opening Book (Top 1000)",   "analytics_opening_stats"),
+        ("player-stats",          "Player Statistics",          "analytics_player_stats"),
+        ("cluster-archetypes",    "Cluster Archetypes",         "analytics_cluster_archetypes"),
+        ("component-sizes",       "Graph Component Sizes",      "analytics_component_sizes"),
+        ("game-length",           "Game Length Distribution",   "analytics_game_length_distribution"),
+        ("game-length-by-result", "Game Length by Result",      "analytics_game_length_by_result"),
+        ("color-advantage",       "Color Advantage",            "analytics_color_advantage"),
+        ("elo-distribution",      "ELO Distribution",           "analytics_elo_distribution"),
+        ("time-control",          "Time Control Analysis",      "analytics_time_control_stats"),
+        ("hourly-activity",       "Hourly Activity",            "analytics_hourly_activity"),
+        ("weekly-activity",       "Weekly Activity",            "analytics_weekly_activity"),
+        ("rating-mismatch",       "Rating Mismatch (Upsets)",   "analytics_rating_mismatch"),
+        ("termination-stats",     "Termination Types",          "analytics_termination_stats"),
+        ("accuracy-by-rating",    "Accuracy by Rating",         "analytics_accuracy_by_rating"),
+        ("blunder-outcome",       "Blunder Outcome",            "analytics_blunder_outcome"),
+        ("clock-by-rating",       "Clock Pressure by Rating",   "analytics_clock_by_rating"),
+        ("scramble-outcome",      "Scramble Outcome",           "analytics_scramble_outcome"),
+        ("smurf-anomaly",         "Smurf Anomaly Scores",       "analytics_smurf_anomaly"),
+        ("flagged-smurfs",        "Flagged Smurfs",             "analytics_flagged_smurfs"),
+    ]
+
+    CSS = """
+    * { box-sizing: border-box; }
+    body { font-family: 'Segoe UI', sans-serif; margin: 0; background: #0d1117; color: #c9d1d9; }
+    header { background: #161b22; border-bottom: 1px solid #30363d; padding: 1rem 2rem; display: flex; align-items: center; gap: 1rem; }
+    header h1 { margin: 0; color: #58a6ff; font-size: 1.25rem; font-weight: 600; flex: 1; }
+    header span { color: #8b949e; font-size: 0.875rem; }
+    .settings-btn { background: none; border: 1px solid #30363d; border-radius: 6px; color: #8b949e;
+                    cursor: pointer; padding: 4px 8px; font-size: 0.8rem; display: flex; align-items: center; gap: 4px; }
+    .settings-btn:hover { border-color: #58a6ff; color: #58a6ff; }
+    .settings-btn.active { border-color: #3fb950; color: #3fb950; }
+    main { padding: 1.5rem 2rem; }
+    h2 { color: #e6edf3; font-size: 1rem; font-weight: 600; margin: 0 0 1rem; }
+    .cards { display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 1rem; }
+    .card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; }
+    .card h3 { margin: 0 0 0.5rem; font-size: 0.9rem; color: #58a6ff; }
+    .card a { text-decoration: none; color: inherit; display: block; }
+    .card a:hover .card-label { text-decoration: underline; }
+    .badge { display: inline-block; background: #21262d; border: 1px solid #30363d;
+             border-radius: 12px; padding: 1px 8px; font-size: 0.75rem; color: #8b949e; }
+    .badge.ready { border-color: #238636; color: #3fb950; }
+    .back { color: #58a6ff; text-decoration: none; font-size: 0.875rem; }
+    .back:hover { text-decoration: underline; }
+    .meta { color: #8b949e; font-size: 0.8rem; margin: 0.5rem 0 1rem; }
+    table { width: 100%; border-collapse: collapse; font-size: 0.8rem; }
+    thead th { background: #161b22; position: sticky; top: 0; padding: 6px 12px;
+               text-align: left; color: #8b949e; border-bottom: 1px solid #30363d;
+               font-weight: 600; white-space: nowrap; }
+    tbody td { padding: 5px 12px; border-bottom: 1px solid #21262d; }
+    tbody tr:hover td { background: #161b22; }
+    .table-wrap { overflow-x: auto; border: 1px solid #30363d; border-radius: 6px; }
+    .notice { background: #161b22; border: 1px solid #30363d; border-radius: 6px;
+              padding: 1.5rem; color: #8b949e; }
+    /* Settings modal */
+    .modal-overlay { display: none; position: fixed; inset: 0; background: rgba(0,0,0,0.6);
+                     z-index: 100; align-items: center; justify-content: center; }
+    .modal-overlay.open { display: flex; }
+    .modal { background: #161b22; border: 1px solid #30363d; border-radius: 10px;
+             padding: 1.5rem; width: 420px; max-width: 95vw; }
+    .modal h3 { margin: 0 0 0.25rem; color: #e6edf3; font-size: 0.95rem; }
+    .modal p { margin: 0 0 1rem; color: #8b949e; font-size: 0.8rem; }
+    .modal input { width: 100%; background: #0d1117; border: 1px solid #30363d; border-radius: 6px;
+                   color: #c9d1d9; font-size: 0.85rem; padding: 8px 10px; margin-bottom: 0.75rem;
+                   font-family: monospace; }
+    .modal input:focus { outline: none; border-color: #58a6ff; }
+    .modal-actions { display: flex; gap: 0.5rem; justify-content: flex-end; }
+    .btn { border-radius: 6px; padding: 6px 14px; font-size: 0.8rem; cursor: pointer; border: 1px solid transparent; }
+    .btn-primary { background: #238636; color: #fff; border-color: #2ea043; }
+    .btn-primary:hover { background: #2ea043; }
+    .btn-danger { background: none; color: #f85149; border-color: #f85149; }
+    .btn-danger:hover { background: rgba(248,81,73,0.1); }
+    .btn-ghost { background: none; color: #8b949e; border-color: #30363d; }
+    .btn-ghost:hover { color: #c9d1d9; border-color: #8b949e; }
+    /* Explain panel */
+    .explain-bar { display: flex; align-items: center; gap: 0.75rem; margin: 1rem 0; }
+    .explain-btn { background: #1f3a5f; border: 1px solid #388bfd; color: #58a6ff; border-radius: 6px;
+                   padding: 6px 14px; font-size: 0.8rem; cursor: pointer; display: flex; align-items: center; gap: 6px; }
+    .explain-btn:hover { background: #263d6a; }
+    .explain-btn:disabled { opacity: 0.5; cursor: not-allowed; }
+    .no-key-hint { color: #8b949e; font-size: 0.78rem; }
+    .explain-panel { background: #161b22; border: 1px solid #388bfd; border-radius: 8px;
+                     padding: 1.25rem; margin-bottom: 1rem; }
+    .explain-panel h4 { margin: 0 0 0.5rem; color: #58a6ff; font-size: 0.85rem; font-weight: 600; }
+    .explain-panel .explain-text { color: #c9d1d9; font-size: 0.85rem; line-height: 1.6; white-space: pre-wrap; }
+    .explain-panel .explain-error { color: #f85149; font-size: 0.82rem; }
+    .spinner { display: inline-block; width: 12px; height: 12px; border: 2px solid #388bfd;
+               border-top-color: transparent; border-radius: 50%; animation: spin 0.7s linear infinite; }
+    @keyframes spin { to { transform: rotate(360deg); } }
+    /* Per-row AI */
+    .row-ai-btn { background: none; border: 1px solid #30363d; border-radius: 4px; color: #8b949e;
+                  cursor: pointer; padding: 1px 5px; font-size: 0.72rem; line-height: 1.4; }
+    .row-ai-btn:hover { border-color: #388bfd; color: #58a6ff; }
+    .row-ai-btn:disabled { opacity: 0.4; cursor: not-allowed; }
+    .row-ai-btn.done { border-color: #238636; color: #3fb950; }
+    .row-expand-tr td { background: #0a0e14; padding: 0.6rem 1rem 0.6rem 2rem;
+                        font-size: 0.8rem; line-height: 1.6; border-bottom: 1px solid #30363d; }
+    .row-explain-text { color: #c9d1d9; white-space: pre-wrap; }
+    .row-explain-error { color: #f85149; }
+    """
+
+    JS = """
+    const NIM_KEY_STORAGE = 'nim_api_key';
+    const rowCache = {};
+    let nimQueue = Promise.resolve();
+
+    function getKey() { return localStorage.getItem(NIM_KEY_STORAGE) || ''; }
+    function hasKey() { return !!getKey(); }
+
+    function updateSettingsBtn() {
+      const btn = document.getElementById('settings-btn');
+      if (!btn) return;
+      if (hasKey()) {
+        btn.classList.add('active');
+        btn.title = 'NIM API key set — click to change';
+      } else {
+        btn.classList.remove('active');
+        btn.title = 'Configure NVIDIA NIM API key for AI explanations';
+      }
+    }
+
+    function openSettings() {
+      document.getElementById('nim-modal').classList.add('open');
+      const inp = document.getElementById('nim-key-input');
+      inp.value = getKey();
+      inp.focus();
+      updateNoKeyHint();
+    }
+
+    function closeSettings() {
+      document.getElementById('nim-modal').classList.remove('open');
+    }
+
+    function saveSettings() {
+      const val = document.getElementById('nim-key-input').value.trim();
+      if (val) localStorage.setItem(NIM_KEY_STORAGE, val);
+      else localStorage.removeItem(NIM_KEY_STORAGE);
+      closeSettings();
+      updateSettingsBtn();
+      updateNoKeyHint();
+    }
+
+    function clearKey() {
+      localStorage.removeItem(NIM_KEY_STORAGE);
+      document.getElementById('nim-key-input').value = '';
+      updateSettingsBtn();
+      updateNoKeyHint();
+    }
+
+    function updateNoKeyHint() {
+      const hint = document.getElementById('no-key-hint');
+      if (!hint) return;
+      hint.style.display = hasKey() ? 'none' : 'inline';
+      const btn = document.getElementById('explain-btn');
+      if (btn) btn.disabled = !hasKey();
+    }
+
+    async function nimStream(payload, onDelta) {
+      const ctrl = new AbortController();
+      const tid = setTimeout(() => ctrl.abort(), 180000);
+      try {
+        const resp = await fetch('/explain', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(payload),
+          signal: ctrl.signal,
+        });
+        if (resp.status === 429) {
+          const retryAfter = parseInt(resp.headers.get('Retry-After') || '10', 10);
+          const err = new Error('Rate limited — retry in ' + retryAfter + 's');
+          err.retryAfter = retryAfter;
+          throw err;
+        }
+        if (!resp.ok) {
+          const msg = await resp.text();
+          throw new Error('Proxy ' + resp.status + ': ' + msg);
+        }
+        const reader = resp.body.getReader();
+        const dec = new TextDecoder();
+        const SEP = String.fromCharCode(10, 10);  // SSE frame separator (two newlines)
+        let buf = '', full = '';
+        for (;;) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          buf += dec.decode(value, { stream: true });
+          let idx;
+          while ((idx = buf.indexOf(SEP)) >= 0) {
+            const frame = buf.slice(0, idx).trim();
+            buf = buf.slice(idx + 2);
+            if (!frame.startsWith('data:')) continue;
+            const obj = JSON.parse(frame.slice(5).trim());
+            if (obj.error) throw new Error(obj.error);
+            if (obj.delta) { full += obj.delta; onDelta(full); }
+          }
+        }
+        return full || '(no response)';
+      } finally {
+        clearTimeout(tid);
+      }
+    }
+
+    function queueNim(payload, onDelta) {
+      const next = nimQueue.then(() => nimStream(payload, onDelta));
+      nimQueue = next.catch(() => {});
+      return next;
+    }
+
+    async function explainDataset() {
+      const key = getKey();
+      if (!key) { openSettings(); return; }
+
+      const btn = document.getElementById('explain-btn');
+      const panel = document.getElementById('explain-panel');
+      const textEl = document.getElementById('explain-text');
+
+      btn.disabled = true;
+      btn.innerHTML = '<span class="spinner"></span> Thinking…';
+      panel.style.display = 'block';
+      textEl.className = 'explain-text';
+      textEl.textContent = '';
+
+      try {
+        const text = await queueNim(
+          { key: key, mode: 'dataset', dataset: window.DATASET },
+          partial => { textEl.textContent = partial; },
+        );
+        textEl.textContent = text;
+      } catch (e) {
+        textEl.className = 'explain-error';
+        textEl.textContent = 'Error: ' + e.message;
+      } finally {
+        btn.disabled = false;
+        btn.innerHTML = '✦ Analyse top 10 rows';
+        updateNoKeyHint();
+      }
+    }
+
+    async function analyzeRow(idx) {
+      const key = getKey();
+      if (!key) { openSettings(); return; }
+      if (rowCache[idx] !== undefined) return;
+
+      const btn = document.getElementById('row-btn-' + idx);
+      const expandRow = document.getElementById('row-expand-' + idx);
+      const textEl = document.getElementById('row-explain-' + idx);
+      if (!btn || !expandRow || !textEl) return;
+
+      const dataset = window.DATASET;
+      const row = dataset.rows && dataset.rows[idx];
+      if (!row) {
+        expandRow.style.display = 'table-row';
+        textEl.className = 'row-explain-error';
+        textEl.textContent = 'Row data not available.';
+        return;
+      }
+
+      btn.disabled = true;
+      btn.innerHTML = '<span class="spinner"></span>';
+      expandRow.style.display = 'table-row';
+      textEl.className = 'row-explain-text';
+      textEl.textContent = 'Analyzing…';
+
+      try {
+        const text = await queueNim(
+          {
+            key: key, mode: 'row',
+            dataset: { label: dataset.label, headers: dataset.headers, row: row },
+          },
+          partial => { textEl.textContent = partial; },
+        );
+        rowCache[idx] = text;
+        textEl.textContent = text;
+        btn.innerHTML = '✓';
+        btn.classList.add('done');
+      } catch (e) {
+        textEl.className = 'row-explain-error';
+        textEl.textContent = e.message;
+        btn.disabled = false;
+        btn.innerHTML = '⚡';
+      }
+    }
+
+    document.addEventListener('DOMContentLoaded', () => {
+      updateSettingsBtn();
+      updateNoKeyHint();
+      document.getElementById('nim-modal')?.addEventListener('click', e => {
+        if (e.target === e.currentTarget) closeSettings();
+      });
+      document.getElementById('nim-key-input')?.addEventListener('keydown', e => {
+        if (e.key === 'Enter') saveSettings();
+        if (e.key === 'Escape') closeSettings();
+      });
+    });
+    """
+
+    SETTINGS_MODAL = """
+    <div class="modal-overlay" id="nim-modal">
+      <div class="modal">
+        <h3>NVIDIA NIM API Key</h3>
+        <p>Used client-side only — stored in your browser's localStorage, never sent to this server.</p>
+        <input type="password" id="nim-key-input" placeholder="nvapi-…" autocomplete="off" spellcheck="false" />
+        <div class="modal-actions">
+          <button class="btn btn-danger" onclick="clearKey()">Clear</button>
+          <button class="btn btn-ghost" onclick="closeSettings()">Cancel</button>
+          <button class="btn btn-primary" onclick="saveSettings()">Save</button>
+        </div>
+      </div>
+    </div>
+    """
+
+    def get_conn():
+        return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS)
+
+    def all_row_counts() -> dict:
+        counts = {}
+        try:
+            with get_conn() as conn:
+                with conn.cursor() as cur:
+                    for _, _, table in DATASETS:
+                        try:
+                            cur.execute(f"SELECT COUNT(*) FROM {table}")
+                            counts[table] = cur.fetchone()[0]
+                        except Exception:
+                            conn.rollback()
+                            counts[table] = -1
+        except Exception:
+            for _, _, table in DATASETS:
+                counts.setdefault(table, -1)
+        return counts
+
+    def fetch_rows(table: str, limit: int = 10_000):
+        with get_conn() as conn:
+            with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+                cur.execute(f"SELECT * FROM {table} LIMIT %s", (limit,))
+                rows = cur.fetchall()
+                if not rows:
+                    return [], []
+                headers = list(rows[0].keys())
+                data = [
+                    [str(row[h]) if row[h] is not None else "" for h in headers]
+                    for row in rows
+                ]
+                return headers, data
+
+    def page(title: str, body: str, dataset_json: str = "null") -> str:
+        return (
+            f"<!doctype html><html lang=en><head><meta charset=utf-8>"
+            f"<meta name=viewport content='width=device-width,initial-scale=1'>"
+            f"<title>{html.escape(title)} — Spark Analytics</title>"
+            f"<style>{CSS}</style></head><body>"
+            f"<header>"
+            f"<h1>Spark Analytics</h1><span>NowChess · staging</span>"
+            f"<button class='settings-btn' id='settings-btn' onclick='openSettings()' title='Configure NIM API key'>⚙ NIM Key</button>"
+            f"</header>"
+            f"<main>{body}</main>"
+            f"{SETTINGS_MODAL}"
+            f"<script>window.DATASET={dataset_json};\n{JS}</script>"
+            f"</body></html>"
+        )
+
+    def index_html() -> str:
+        counts = all_row_counts()
+        cards = ""
+        for slug, label, table in DATASETS:
+            count = counts.get(table, -1)
+            if count > 0:
+                badge = f'<span class="badge ready">{count} rows</span>'
+            elif count == 0:
+                badge = '<span class="badge">no data yet</span>'
+            else:
+                badge = '<span class="badge">table missing</span>'
+            cards += (
+                f'<div class="card"><a href="/{slug}">'
+                f'<div class="card-label"><h3>{html.escape(label)}</h3></div>'
+                f"{badge}</a></div>"
+            )
+        return page("Results", f"<h2>Datasets</h2><div class='cards'>{cards}</div>")
+
+    def table_html(slug: str, label: str, table: str) -> str:
+        back = '<a class="back" href="/">← All datasets</a>'
+        try:
+            headers, rows = fetch_rows(table)
+        except Exception as e:
+            return page(
+                label,
+                f"{back}<h2>{html.escape(label)}</h2>"
+                f"<div class='notice'>Error querying {html.escape(table)} on {html.escape(DB_HOST)}:{DB_PORT}/{html.escape(DB_NAME)}: {html.escape(str(e))}</div>",
+            )
+        if not headers:
+            return page(
+                label,
+                f"{back}<h2>{html.escape(label)}</h2>"
+                "<div class='notice'>No data yet. Run the CronJob first, then check back.</div>",
+            )
+
+        ai_rows = rows[:200]
+        dataset_json = json.dumps({
+            "label": label,
+            "headers": headers,
+            "sample": rows[:10],
+            "rows": ai_rows,
+            "total_rows": len(rows),
+        })
+
+        explain_bar = (
+            "<div class='explain-bar'>"
+            "<button class='explain-btn' id='explain-btn' onclick='explainDataset()'>✦ Analyse top 10 rows</button>"
+            "<span class='no-key-hint' id='no-key-hint'>— <a href='#' onclick='openSettings();return false;' style='color:#58a6ff'>add NIM key</a> to enable AI explanations</span>"
+            "</div>"
+            "<div class='explain-panel' id='explain-panel' style='display:none'>"
+            "<h4>AI Explanation</h4>"
+            "<div class='explain-text' id='explain-text'></div>"
+            "</div>"
+        )
+
+        num_cols = len(headers)
+        ths = "".join(f"<th>{html.escape(h)}</th>" for h in headers) + "<th></th>"
+        trs = ""
+        for i, row in enumerate(rows[:10_000]):
+            cells = "".join(f"<td>{html.escape(str(c))}</td>" for c in row)
+            if i < 200:
+                ai_cell = (
+                    f"<td><button class='row-ai-btn' id='row-btn-{i}' "
+                    f"onclick='analyzeRow({i})' title='Analyse this row with AI'>⚡</button></td>"
+                )
+                expand = (
+                    f"<tr class='row-expand-tr' id='row-expand-{i}' style='display:none'>"
+                    f"<td colspan='{num_cols + 1}'>"
+                    f"<span id='row-explain-{i}'></span>"
+                    f"</td></tr>"
+                )
+            else:
+                ai_cell = "<td></td>"
+                expand = ""
+            trs += f"<tr>{cells}{ai_cell}</tr>{expand}"
+
+        truncated = f" (showing first 10 000 of {len(rows)})" if len(rows) > 10_000 else ""
+        return page(
+            label,
+            f"{back}<h2>{html.escape(label)}</h2>"
+            f"<p class='meta'>{len(rows)} rows{truncated}</p>"
+            f"{explain_bar}"
+            f"<div class='table-wrap'><table><thead><tr>{ths}</tr></thead>"
+            f"<tbody>{trs}</tbody></table></div>",
+            dataset_json=dataset_json,
+        )
+
+    SLUG_MAP = {s: (label, table) for s, label, table in DATASETS}
+
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, fmt, *args):
+            pass
+
+        def do_GET(self):
+            path = self.path.split("?")[0].lstrip("/")
+            if path == "" or path == "index.html":
+                self._send(index_html())
+            elif path in SLUG_MAP:
+                label, table = SLUG_MAP[path]
+                self._send(table_html(path, label, table))
+            else:
+                self.send_response(404)
+                self.end_headers()
+
+        def do_POST(self):
+            path = self.path.split("?")[0].lstrip("/")
+            if path != "explain":
+                self.send_response(404)
+                self.end_headers()
+                return
+            length = int(self.headers.get("Content-Length", 0))
+            headers_sent = False
+            try:
+                body = json.loads(self.rfile.read(length))
+                api_key = body.get("key", "")
+                mode = body.get("mode", "dataset")
+                dataset = body.get("dataset", {})
+                if not api_key:
+                    raise ValueError("missing key")
+
+                label = dataset.get("label", "")
+                headers = dataset.get("headers", [])
+
+                if mode == "row":
+                    row = dataset.get("row", [])
+                    row_desc = ", ".join(f"{h}: {v}" for h, v in zip(headers, row))
+                    system_prompt = (
+                        "You are a chess analytics expert. Analyze a single data entry from a chess analytics dataset. "
+                        "Be specific and insightful — 3-4 sentences. "
+                        "If the entry involves a chess opening (ECO code or opening name present), explain the opening's "
+                        "strategic ideas, strengths and weaknesses, and why players choose it. "
+                        "For player data, explain what the stats reveal about their playing style. "
+                        "For other types, explain what makes this entry notable."
+                    )
+                    user_prompt = (
+                        f"Dataset: \"{label}\"\n"
+                        f"Columns: {', '.join(headers)}\n\n"
+                        f"Entry to analyze: {row_desc}\n\n"
+                        "Provide a detailed, chess-specific analysis of this entry."
+                    )
+                    max_tokens = 300
+                else:
+                    sample = dataset.get("sample", [])
+                    rows_text = "\n".join(
+                        f"Row {i + 1}: " + ", ".join(f"{h}: {v}" for h, v in zip(headers, row))
+                        for i, row in enumerate(sample)
+                    )
+                    system_prompt = (
+                        "You are a chess analytics expert. Analyze the top 10 entries from a chess analytics dataset. "
+                        "For each entry write 1-2 specific sentences about what is notable. "
+                        "If entries are chess openings (ECO codes or opening names present), name each opening properly "
+                        "and explain its strategic character and why it ranks here. "
+                        "Format your response as 'Row N: [analysis]' for each row."
+                    )
+                    user_prompt = (
+                        f"Dataset: \"{label}\" ({dataset.get('total_rows', 0)} total rows)\n"
+                        f"Columns: {', '.join(headers)}\n\n"
+                        f"Top 10 entries:\n{rows_text}\n\n"
+                        "Analyze each entry."
+                    )
+                    max_tokens = 512
+
+                payload = json.dumps({
+                    "model": "meta/llama-3.3-70b-instruct",
+                    "messages": [
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_prompt},
+                    ],
+                    "max_tokens": max_tokens,
+                    "temperature": 0.4,
+                    "stream": True,
+                }).encode("utf-8")
+                req = urllib.request.Request(
+                    "https://integrate.api.nvidia.com/v1/chat/completions",
+                    data=payload,
+                    headers={
+                        "Authorization": f"Bearer {api_key}",
+                        "Content-Type": "application/json",
+                    },
+                    method="POST",
+                )
+                try:
+                    upstream = urllib.request.urlopen(req, timeout=110)
+                except urllib.error.HTTPError as he:
+                    if he.code == 429:
+                        retry_after = he.headers.get("Retry-After", "10")
+                        resp_body = json.dumps({"error": f"Rate limited — retry in {retry_after}s"}).encode("utf-8")
+                        self.send_response(429)
+                        self.send_header("Content-Type", "application/json")
+                        self.send_header("Content-Length", str(len(resp_body)))
+                        self.send_header("Retry-After", str(retry_after))
+                        self.end_headers()
+                        try:
+                            self.wfile.write(resp_body)
+                        except BrokenPipeError:
+                            pass
+                        return
+                    raise RuntimeError(f"NIM API error {he.code}: {he.reason}")
+
+                # Relay NIM's SSE stream to the browser as our own SSE.
+                self.send_response(200)
+                self.send_header("Content-Type", "text/event-stream")
+                self.send_header("Cache-Control", "no-cache")
+                self.send_header("X-Accel-Buffering", "no")
+                self.end_headers()
+                headers_sent = True
+                for raw in upstream:
+                    line = raw.decode("utf-8", "replace").strip()
+                    if not line.startswith("data:"):
+                        continue
+                    data = line[5:].strip()
+                    if data == "[DONE]":
+                        break
+                    try:
+                        obj = json.loads(data)
+                    except json.JSONDecodeError:
+                        continue
+                    piece = obj.get("choices", [{}])[0].get("delta", {}).get("content", "")
+                    if piece and not self._emit({"delta": piece}):
+                        return
+                self._emit({"done": True})
+            except Exception as e:
+                if not headers_sent:
+                    self.send_response(200)
+                    self.send_header("Content-Type", "text/event-stream")
+                    self.send_header("Cache-Control", "no-cache")
+                    self.end_headers()
+                self._emit({"error": str(e)})
+
+        def _emit(self, obj: dict) -> bool:
+            try:
+                self.wfile.write(("data: " + json.dumps(obj) + "\n\n").encode("utf-8"))
+                self.wfile.flush()
+                return True
+            except (BrokenPipeError, ConnectionResetError):
+                return False
+
+        def _send(self, body: str):
+            data = body.encode("utf-8")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            try:
+                self.wfile.write(data)
+            except BrokenPipeError:
+                pass
+
+        def _send_json(self, obj: dict):
+            data = json.dumps(obj).encode("utf-8")
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            try:
+                self.wfile.write(data)
+            except BrokenPipeError:
+                pass
+
+    if __name__ == "__main__":
+        server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
+        print(f"Listening on :{PORT}  DB={DB_HOST}:{DB_PORT}/{DB_NAME}  JDBC_URL={_url!r}", flush=True)
+        server.serve_forever()
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-analytics-webview
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  replicas: 0
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: spark-analytics-webview
+  template:
+    metadata:
+      labels:
+        app: spark-analytics-webview
+        app.kubernetes.io/name: spark-analytics
+        app.kubernetes.io/part-of: nowchess
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        fsGroup: 65534
+      containers:
+        - name: webview
+          image: python:3.12-slim
+          command: ["sh", "-c", "pip install psycopg2-binary --quiet --no-cache-dir --target=/tmp/pkg && PYTHONPATH=/tmp/pkg python /scripts/serve.py"]
+          ports:
+            - containerPort: 8080
+          env:
+            - name: PORT
+              value: "8080"
+            - name: NOWCHESS_JDBC_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ncs-db-secrets
+                  key: STORE_DB_URL
+            - name: NOWCHESS_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: ncs-db-secrets
+                  key: STORE_DB_USER
+            - name: NOWCHESS_DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: ncs-db-secrets
+                  key: STORE_DB_PASSWORD
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 256Mi
+      volumes:
+        - name: scripts
+          configMap:
+            name: spark-analytics-webview
+            defaultMode: 0755
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-analytics-webview
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  selector:
+    app: spark-analytics-webview
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080

spark-analytics/eu-central-1-prod/kustomization.yaml

@@ -26,14 +26,22 @@ patches:
                       - --master
                       - spark://spark-master:7077
                       - --conf
-                      - spark.executor.instances=2
+                      - spark.executor.instances=1
                       - --conf
-                      - spark.executor.memory=1g
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
                       - --conf
                       - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-opening-book-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
                       - --class
                       - de.nowchess.analytics.OpeningBookJob
-                      - local:///app/analytics.jar
+                      - file:///app/analytics.jar
                       - /spark-output/opening-book
                       - "10"
     target:
@@ -56,14 +64,22 @@ patches:
                       - --master
                       - spark://spark-master:7077
                       - --conf
-                      - spark.executor.instances=2
+                      - spark.executor.instances=1
                       - --conf
-                      - spark.executor.memory=1g
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
                       - --conf
                       - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-player-stats-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
                       - --class
                       - de.nowchess.analytics.PlayerStatsJob
-                      - local:///app/analytics.jar
+                      - file:///app/analytics.jar
                       - /spark-output/player-stats
     target:
       kind: CronJob
@@ -85,14 +101,22 @@ patches:
                       - --master
                       - spark://spark-master:7077
                       - --conf
-                      - spark.executor.instances=2
+                      - spark.executor.instances=1
                       - --conf
-                      - spark.executor.memory=1g
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
                       - --conf
                       - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-player-clustering-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
                       - --class
                       - de.nowchess.analytics.PlayerClusteringJob
-                      - local:///app/analytics.jar
+                      - file:///app/analytics.jar
                       - /spark-output/player-clusters
                       - "4"
     target:
@@ -115,14 +139,22 @@ patches:
                       - --master
                       - spark://spark-master:7077
                       - --conf
-                      - spark.executor.instances=2
+                      - spark.executor.instances=1
                       - --conf
-                      - spark.executor.memory=1g
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
                       - --conf
                       - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-player-graph-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
                       - --class
                       - de.nowchess.analytics.PlayerGraphJob
-                      - local:///app/analytics.jar
+                      - file:///app/analytics.jar
                       - /spark-output/player-graph
     target:
       kind: CronJob
@@ -144,9 +176,17 @@ patches:
                   - --conf
                   - spark.executor.instances=1
                   - --conf
+                  - spark.executor.cores=1
+                  - --conf
                   - spark.executor.memory=512m
                   - --conf
                   - spark.driver.memory=512m
+                  - --conf
+                  - spark.driver.host=spark-live-dashboard-driver
+                  - --conf
+                  - spark.driver.port=37007
+                  - --conf
+                  - spark.blockManager.port=37777
                   - --class
                   - de.nowchess.analytics.LiveDashboardJob
                   - local:///app/analytics.jar

spark-analytics/eu-central-1-staging/kustomization.yaml

@@ -0,0 +1,429 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: nowchess-staging
+resources:
+  - ../base
+  - webview-ingress.yaml
+images:
+  - name: ghcr.io/now-chess/now-chess-systems/analytics
+    newTag: 0.7.0
+patches:
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-opening-book
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    command:
+                      - /opt/spark/bin/spark-submit
+                      - --master
+                      - spark://spark-master:7077
+                      - --conf
+                      - spark.executor.instances=1
+                      - --conf
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
+                      - --conf
+                      - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-opening-book-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
+                      - --class
+                      - de.nowchess.analytics.OpeningBookJob
+                      - file:///app/analytics.jar
+                      - /spark-output/opening-book
+                      - "10"
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-opening-book
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-player-stats
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    command:
+                      - /opt/spark/bin/spark-submit
+                      - --master
+                      - spark://spark-master:7077
+                      - --conf
+                      - spark.executor.instances=1
+                      - --conf
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
+                      - --conf
+                      - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-player-stats-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
+                      - --class
+                      - de.nowchess.analytics.PlayerStatsJob
+                      - file:///app/analytics.jar
+                      - /spark-output/player-stats
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-player-stats
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-player-clustering
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    command:
+                      - /opt/spark/bin/spark-submit
+                      - --master
+                      - spark://spark-master:7077
+                      - --conf
+                      - spark.executor.instances=1
+                      - --conf
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
+                      - --conf
+                      - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-player-clustering-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
+                      - --class
+                      - de.nowchess.analytics.PlayerClusteringJob
+                      - file:///app/analytics.jar
+                      - /spark-output/player-clusters
+                      - "4"
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-player-clustering
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-player-graph
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    command:
+                      - /opt/spark/bin/spark-submit
+                      - --master
+                      - spark://spark-master:7077
+                      - --conf
+                      - spark.executor.instances=1
+                      - --conf
+                      - spark.executor.cores=1
+                      - --conf
+                      - spark.executor.memory=512m
+                      - --conf
+                      - spark.driver.memory=512m
+                      - --conf
+                      - spark.driver.host=spark-player-graph-driver
+                      - --conf
+                      - spark.driver.port=44167
+                      - --conf
+                      - spark.blockManager.port=44777
+                      - --class
+                      - de.nowchess.analytics.PlayerGraphJob
+                      - file:///app/analytics.jar
+                      - /spark-output/player-graph
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-player-graph
+  - patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: spark-live-dashboard
+      spec:
+        template:
+          spec:
+            containers:
+              - name: spark-driver
+                command:
+                  - /opt/spark/bin/spark-submit
+                  - --master
+                  - spark://spark-master:7077
+                  - --conf
+                  - spark.executor.instances=1
+                  - --conf
+                  - spark.executor.cores=1
+                  - --conf
+                  - spark.executor.memory=512m
+                  - --conf
+                  - spark.driver.memory=512m
+                  - --conf
+                  - spark.driver.host=spark-live-dashboard-driver
+                  - --conf
+                  - spark.driver.port=37007
+                  - --conf
+                  - spark.blockManager.port=37777
+                  - --class
+                  - de.nowchess.analytics.LiveDashboardJob
+                  - local:///app/analytics.jar
+                  - /spark-checkpoint
+    target:
+      kind: Deployment
+      name: spark-live-dashboard
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-game-length
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-game-length
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-color-advantage
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-color-advantage
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-elo-distribution
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-elo-distribution
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-time-control
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-time-control
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-daily-activity
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-daily-activity
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-rating-mismatch
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-rating-mismatch
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-termination-stats
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-termination-stats
+  # Eval-annotated dump required for per-move accuracy; the default 2013 dump
+  # carries no [%eval] comments — point this at an evals dump for real output.
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-accuracy-blunder
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-accuracy-blunder
+  # Clock-annotated dump required; the default 2013 dump carries no [%clk]
+  # comments — point this at a dump with clocks for real output.
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-clock-pressure
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-clock-pressure
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-smurf-anomaly
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-smurf-anomaly
+  - patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: spark-analytics-webview
+      spec:
+        replicas: 1
+    target:
+      kind: Deployment
+      name: spark-analytics-webview

spark-analytics/eu-central-1-staging/webview-ingress.yaml

@@ -0,0 +1,32 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: spark-analytics-webview-ingress
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    # AI /explain streams Server-Sent Events — disable response buffering so
+    # chunks flush to the client immediately, and allow gaps between chunks.
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "120"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: spark-results-st.nowchess.janis-eccarius.de
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: spark-analytics-webview
+                port:
+                  number: 8080
+  tls:
+    - hosts:
+        - spark-results-st.nowchess.janis-eccarius.de
+      secretName: spark-analytics-webview-staging-cert

spark-cluster/base/history-server.yaml

@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/name: spark-history-server
     app.kubernetes.io/part-of: nowchess
 spec:
-  replicas: 1
+  replicas: 0
   selector:
     matchLabels:
       app.kubernetes.io/name: spark-history-server

spark-cluster/base/master.yaml

@@ -32,11 +32,11 @@ spec:
               containerPort: 8080
           resources:
             requests:
-              cpu: 200m
+              cpu: 100m
               memory: 256Mi
             limits:
-              cpu: "1"
-              memory: 2Gi
+              cpu: 500m
+              memory: 512Mi
           volumeMounts:
             - name: spark-events
               mountPath: /spark-events

spark-cluster/base/workers.yaml

@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/name: spark-worker
     app.kubernetes.io/part-of: nowchess
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: spark-worker
@@ -25,7 +25,7 @@ spec:
             - spark://spark-master:7077
           env:
             - name: SPARK_WORKER_CORES
-              value: "1"
+              value: "2"
             - name: SPARK_WORKER_MEMORY
               value: "512m"
           ports:
@@ -33,14 +33,19 @@ spec:
               containerPort: 8081
           resources:
             requests:
-              cpu: 200m
+              cpu: 100m
               memory: 256Mi
             limits:
-              cpu: "1"
-              memory: 2Gi
+              cpu: 500m
+              memory: 1Gi
           volumeMounts:
             - name: spark-events
               mountPath: /spark-events
+            - name: spark-output
+              mountPath: /spark-output
       volumes:
         - name: spark-events
           emptyDir: {}
+        - name: spark-output
+          persistentVolumeClaim:
+            claimName: spark-analytics-output

spark-cluster/eu-central-1-prod/kustomization.yaml

@@ -3,3 +3,24 @@ kind: Kustomization
 namespace: nowchess
 resources:
   - ../base
+patches:
+  - patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: spark-master
+      spec:
+        replicas: 0
+    target:
+      kind: Deployment
+      name: spark-master
+  - patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: spark-worker
+      spec:
+        replicas: 0
+    target:
+      kind: Deployment
+      name: spark-worker

spark-cluster/eu-central-1-staging/kustomization.yaml

@@ -0,0 +1,62 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: nowchess-staging
+resources:
+  - ../base
+patches:
+  - patch: |-
+      apiVersion: networking.k8s.io/v1
+      kind: Ingress
+      metadata:
+        name: spark-master-ingress
+        annotations:
+          cert-manager.io/cluster-issuer: "letsencrypt-prod"
+          nginx.ingress.kubernetes.io/ssl-redirect: "true"
+          nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+      spec:
+        rules:
+          - host: spark-st.nowchess.janis-eccarius.de
+            http:
+              paths:
+                - path: /
+                  pathType: Prefix
+                  backend:
+                    service:
+                      name: spark-master
+                      port:
+                        number: 8080
+        tls:
+          - hosts:
+              - spark-st.nowchess.janis-eccarius.de
+            secretName: spark-cluster-staging-cert
+    target:
+      kind: Ingress
+      name: spark-master-ingress
+  - patch: |-
+      apiVersion: networking.k8s.io/v1
+      kind: Ingress
+      metadata:
+        name: spark-history-ingress
+        annotations:
+          cert-manager.io/cluster-issuer: "letsencrypt-prod"
+          nginx.ingress.kubernetes.io/ssl-redirect: "true"
+          nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+      spec:
+        rules:
+          - host: spark-history-st.nowchess.janis-eccarius.de
+            http:
+              paths:
+                - path: /
+                  pathType: Prefix
+                  backend:
+                    service:
+                      name: spark-history-server-svc
+                      port:
+                        number: 18080
+        tls:
+          - hosts:
+              - spark-history-st.nowchess.janis-eccarius.de
+            secretName: spark-history-staging-cert
+    target:
+      kind: Ingress
+      name: spark-history-ingress