feat(spark-analytics): switch webview to PostgreSQL, always write analytics to DB

- Rewrite webview serve.py to query PostgreSQL analytics tables via psycopg2
  instead of reading CSV files from PVC; PVC mount removed from Deployment
- Remove isPgnMode guard from all 4 original jobs so results always write to DB
  regardless of whether input comes from game_records or a Lichess PGN dump
- Add JDBC write-back to all 7 new analytics jobs (game_length_distribution,
  game_length_by_result, color_advantage, elo_distribution, time_control_stats,
  hourly_activity, weekly_activity, rating_mismatch, termination_stats)
- Add analytics_component_sizes JDBC write to PlayerGraphJob
- Add NOWCHESS_PGN_PATH staging patches for the 7 new CronJobs

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

spark-analytics/base/webview.yaml

@@ -8,32 +8,39 @@ metadata:
 data:
   serve.py: |
     #!/usr/bin/env python3
-    """Spark analytics results viewer — reads CSV output from PVC, serves HTML tables."""
+    """Spark analytics results viewer — queries PostgreSQL analytics tables, serves HTML tables."""
-    import csv
-    import glob
     import html
     import os
+    import re
+    import psycopg2
+    import psycopg2.extras
     from http.server import BaseHTTPRequestHandler, HTTPServer
 
-    OUTPUT_DIR = os.environ.get("SPARK_OUTPUT_DIR", "/spark-output")
     PORT = int(os.environ.get("PORT", "8080"))
 
-    # Each tuple: (url-slug, display label, path relative to SPARK_OUTPUT_DIR)
+    _url = os.environ.get("NOWCHESS_JDBC_URL", "jdbc:postgresql://localhost:5432/nowchess")
-    # Paths mirror the CronJob outputDir arg + the subdir each job writes CSV to.
+    _m = re.match(r"jdbc:postgresql://([^:/]+):(\d+)/(.+)", _url)
+    DB_HOST = _m.group(1) if _m else "localhost"
+    DB_PORT = int(_m.group(2)) if _m else 5432
+    DB_NAME = _m.group(3) if _m else "nowchess"
+    DB_USER = os.environ.get("NOWCHESS_DB_USER", "nowchess")
+    DB_PASS = os.environ.get("NOWCHESS_DB_PASS", "")
+
+    # Each tuple: (url-slug, display label, postgres table name)
     DATASETS = [
-        ("opening-book", "Opening Book (Top 1000)", "opening-book/opening_book_top1000"),
+        ("opening-book",          "Opening Book (Top 1000)",   "analytics_opening_stats"),
-        ("player-stats", "Player Statistics", "player-stats/player_stats_csv"),
+        ("player-stats",          "Player Statistics",          "analytics_player_stats"),
-        ("cluster-archetypes", "Cluster Archetypes", "player-clusters/cluster_archetypes"),
+        ("cluster-archetypes",    "Cluster Archetypes",         "analytics_cluster_archetypes"),
-        ("component-sizes", "Graph Component Sizes", "player-graph/component_sizes"),
+        ("component-sizes",       "Graph Component Sizes",      "analytics_component_sizes"),
-        ("game-length", "Game Length Distribution", "game-length/game_length_distribution"),
+        ("game-length",           "Game Length Distribution",   "analytics_game_length_distribution"),
-        ("game-length-by-result", "Game Length by Result", "game-length/game_length_by_result"),
+        ("game-length-by-result", "Game Length by Result",      "analytics_game_length_by_result"),
-        ("color-advantage", "Color Advantage", "color-advantage/color_advantage"),
+        ("color-advantage",       "Color Advantage",            "analytics_color_advantage"),
-        ("elo-distribution", "ELO Distribution", "elo-distribution/elo_distribution"),
+        ("elo-distribution",      "ELO Distribution",           "analytics_elo_distribution"),
-        ("time-control", "Time Control Analysis", "time-control/time_control_stats"),
+        ("time-control",          "Time Control Analysis",      "analytics_time_control_stats"),
-        ("hourly-activity", "Hourly Activity", "daily-activity/hourly_activity"),
+        ("hourly-activity",       "Hourly Activity",            "analytics_hourly_activity"),
-        ("weekly-activity", "Weekly Activity", "daily-activity/weekly_activity"),
+        ("weekly-activity",       "Weekly Activity",            "analytics_weekly_activity"),
-        ("rating-mismatch", "Rating Mismatch (Upsets)", "rating-mismatch/rating_mismatch"),
+        ("rating-mismatch",       "Rating Mismatch (Upsets)",   "analytics_rating_mismatch"),
-        ("termination-stats", "Termination Types", "termination-stats/termination_stats"),
+        ("termination-stats",     "Termination Types",          "analytics_termination_stats"),
     ]
 
     CSS = """
@@ -66,23 +73,31 @@ data:
               padding: 1.5rem; color: #8b949e; }
     """
 
-    def read_csv_parts(subdir: str) -> tuple[list[str], list[list[str]]]:
+    def get_conn():
-        files = sorted(glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv"))
+        return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS)
-        headers: list[str] = []
-        rows: list[list[str]] = []
-        for f in files:
-            with open(f, newline="", encoding="utf-8") as fh:
-                reader = csv.reader(fh)
-                for i, row in enumerate(reader):
-                    if i == 0 and not headers:
-                        headers = row
-                    elif headers:
-                        rows.append(row)
-        return headers, rows
 
-    def dataset_status(subdir: str) -> tuple[int, int]:
+    def table_row_count(table: str) -> int:
-        files = glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv")
+        try:
-        return len(files), sum(os.path.getsize(f) for f in files)
+            with get_conn() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(f"SELECT COUNT(*) FROM {table}")
+                    return cur.fetchone()[0]
+        except Exception:
+            return -1
+
+    def fetch_rows(table: str, limit: int = 10_000):
+        with get_conn() as conn:
+            with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+                cur.execute(f"SELECT * FROM {table} LIMIT %s", (limit,))
+                rows = cur.fetchall()
+                if not rows:
+                    return [], []
+                headers = list(rows[0].keys())
+                data = [
+                    [str(row[h]) if row[h] is not None else "" for h in headers]
+                    for row in rows
+                ]
+                return headers, data
 
     def page(title: str, body: str) -> str:
         return (
@@ -96,26 +111,31 @@ data:
 
     def index_html() -> str:
         cards = ""
-        for slug, label, subdir in DATASETS:
+        for slug, label, table in DATASETS:
-            parts, size_bytes = dataset_status(subdir)
+            count = table_row_count(table)
-            if parts:
+            if count > 0:
-                size_kb = size_bytes // 1024
+                badge = f'<span class="badge ready">{count} rows</span>'
-                badge = f'<span class="badge ready">{parts} part(s) · {size_kb} KB</span>'
+            elif count == 0:
-            else:
                 badge = '<span class="badge">no data yet</span>'
+            else:
+                badge = '<span class="badge">table missing</span>'
             cards += (
                 f'<div class="card"><a href="/{slug}">'
                 f'<div class="card-label"><h3>{html.escape(label)}</h3></div>'
                 f"{badge}</a></div>"
             )
-        return page(
+        return page("Results", f"<h2>Datasets</h2><div class='cards'>{cards}</div>")
-            "Results",
-            f"<h2>Datasets</h2><div class='cards'>{cards}</div>",
-        )
 
-    def table_html(slug: str, label: str, subdir: str) -> str:
+    def table_html(slug: str, label: str, table: str) -> str:
-        headers, rows = read_csv_parts(subdir)
         back = '<a class="back" href="/">← All datasets</a>'
+        try:
+            headers, rows = fetch_rows(table)
+        except Exception as e:
+            return page(
+                label,
+                f"{back}<h2>{html.escape(label)}</h2>"
+                f"<div class='notice'>Error querying {html.escape(table)}: {html.escape(str(e))}</div>",
+            )
         if not headers:
             return page(
                 label,
@@ -136,7 +156,7 @@ data:
             f"<tbody>{trs}</tbody></table></div>",
         )
 
-    SLUG_MAP = {s: (label, subdir) for s, label, subdir in DATASETS}
+    SLUG_MAP = {s: (label, table) for s, label, table in DATASETS}
 
     class Handler(BaseHTTPRequestHandler):
         def log_message(self, fmt, *args):
@@ -147,8 +167,8 @@ data:
             if path == "" or path == "index.html":
                 self._send(index_html())
             elif path in SLUG_MAP:
-                label, subdir = SLUG_MAP[path]
+                label, table = SLUG_MAP[path]
-                self._send(table_html(path, label, subdir))
+                self._send(table_html(path, label, table))
             else:
                 self.send_response(404)
                 self.end_headers()
@@ -163,7 +183,7 @@ data:
 
     if __name__ == "__main__":
         server = HTTPServer(("0.0.0.0", PORT), Handler)
-        print(f"Listening on :{PORT}  OUTPUT_DIR={OUTPUT_DIR}", flush=True)
+        print(f"Listening on :{PORT}  DB={DB_HOST}:{DB_PORT}/{DB_NAME}", flush=True)
         server.serve_forever()
 ---
 apiVersion: apps/v1
@@ -194,25 +214,35 @@ spec:
       containers:
         - name: webview
           image: python:3.12-slim
-          command: ["python", "/scripts/serve.py"]
+          command: ["sh", "-c", "pip install psycopg2-binary --quiet && python /scripts/serve.py"]
           ports:
             - containerPort: 8080
           env:
-            - name: SPARK_OUTPUT_DIR
-              value: /spark-output
             - name: PORT
               value: "8080"
+            - name: NOWCHESS_JDBC_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ncs-db-secrets
+                  key: STORE_DB_URL
+            - name: NOWCHESS_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: ncs-db-secrets
+                  key: STORE_DB_USER
+            - name: NOWCHESS_DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: ncs-db-secrets
+                  key: STORE_DB_PASSWORD
           volumeMounts:
-            - name: spark-output
-              mountPath: /spark-output
-              readOnly: true
             - name: scripts
               mountPath: /scripts
           readinessProbe:
             httpGet:
               path: /
               port: 8080
-            initialDelaySeconds: 3
+            initialDelaySeconds: 15
             periodSeconds: 10
           resources:
             requests:
@@ -222,9 +252,6 @@ spec:
               cpu: 200m
               memory: 256Mi
       volumes:
-        - name: spark-output
-          persistentVolumeClaim:
-            claimName: spark-analytics-output
         - name: scripts
           configMap:
             name: spark-analytics-webview

spark-analytics/eu-central-1-staging/kustomization.yaml

@@ -205,6 +205,132 @@ patches:
     target:
       kind: Deployment
       name: spark-live-dashboard
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-game-length
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-game-length
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-color-advantage
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-color-advantage
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-elo-distribution
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-elo-distribution
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-time-control
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-time-control
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-daily-activity
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-daily-activity
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-rating-mismatch
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-rating-mismatch
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-termination-stats
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+    target:
+      kind: CronJob
+      name: spark-termination-stats
   - patch: |-
       apiVersion: apps/v1
       kind: Deployment