feat(spark-analytics): switch webview to PostgreSQL, always write analytics to DB
- Rewrite webview serve.py to query PostgreSQL analytics tables via psycopg2 instead of reading CSV files from PVC; PVC mount removed from Deployment - Remove isPgnMode guard from all 4 original jobs so results always write to DB regardless of whether input comes from game_records or a Lichess PGN dump - Add JDBC write-back to all 7 new analytics jobs (game_length_distribution, game_length_by_result, color_advantage, elo_distribution, time_control_stats, hourly_activity, weekly_activity, rating_mismatch, termination_stats) - Add analytics_component_sizes JDBC write to PlayerGraphJob - Add NOWCHESS_PGN_PATH staging patches for the 7 new CronJobs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Janis Eccarius
@@ -8,32 +8,39 @@ metadata:
|
||||
data:
|
||||
serve.py: |
|
||||
#!/usr/bin/env python3
|
||||
"""Spark analytics results viewer — reads CSV output from PVC, serves HTML tables."""
|
||||
import csv
|
||||
import glob
|
||||
"""Spark analytics results viewer — queries PostgreSQL analytics tables, serves HTML tables."""
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
OUTPUT_DIR = os.environ.get("SPARK_OUTPUT_DIR", "/spark-output")
|
||||
PORT = int(os.environ.get("PORT", "8080"))
|
||||
|
||||
# Each tuple: (url-slug, display label, path relative to SPARK_OUTPUT_DIR)
|
||||
# Paths mirror the CronJob outputDir arg + the subdir each job writes CSV to.
|
||||
_url = os.environ.get("NOWCHESS_JDBC_URL", "jdbc:postgresql://localhost:5432/nowchess")
|
||||
_m = re.match(r"jdbc:postgresql://([^:/]+):(\d+)/(.+)", _url)
|
||||
DB_HOST = _m.group(1) if _m else "localhost"
|
||||
DB_PORT = int(_m.group(2)) if _m else 5432
|
||||
DB_NAME = _m.group(3) if _m else "nowchess"
|
||||
DB_USER = os.environ.get("NOWCHESS_DB_USER", "nowchess")
|
||||
DB_PASS = os.environ.get("NOWCHESS_DB_PASS", "")
|
||||
|
||||
# Each tuple: (url-slug, display label, postgres table name)
|
||||
DATASETS = [
|
||||
("opening-book", "Opening Book (Top 1000)", "opening-book/opening_book_top1000"),
|
||||
("player-stats", "Player Statistics", "player-stats/player_stats_csv"),
|
||||
("cluster-archetypes", "Cluster Archetypes", "player-clusters/cluster_archetypes"),
|
||||
("component-sizes", "Graph Component Sizes", "player-graph/component_sizes"),
|
||||
("game-length", "Game Length Distribution", "game-length/game_length_distribution"),
|
||||
("game-length-by-result", "Game Length by Result", "game-length/game_length_by_result"),
|
||||
("color-advantage", "Color Advantage", "color-advantage/color_advantage"),
|
||||
("elo-distribution", "ELO Distribution", "elo-distribution/elo_distribution"),
|
||||
("time-control", "Time Control Analysis", "time-control/time_control_stats"),
|
||||
("hourly-activity", "Hourly Activity", "daily-activity/hourly_activity"),
|
||||
("weekly-activity", "Weekly Activity", "daily-activity/weekly_activity"),
|
||||
("rating-mismatch", "Rating Mismatch (Upsets)", "rating-mismatch/rating_mismatch"),
|
||||
("termination-stats", "Termination Types", "termination-stats/termination_stats"),
|
||||
("opening-book", "Opening Book (Top 1000)", "analytics_opening_stats"),
|
||||
("player-stats", "Player Statistics", "analytics_player_stats"),
|
||||
("cluster-archetypes", "Cluster Archetypes", "analytics_cluster_archetypes"),
|
||||
("component-sizes", "Graph Component Sizes", "analytics_component_sizes"),
|
||||
("game-length", "Game Length Distribution", "analytics_game_length_distribution"),
|
||||
("game-length-by-result", "Game Length by Result", "analytics_game_length_by_result"),
|
||||
("color-advantage", "Color Advantage", "analytics_color_advantage"),
|
||||
("elo-distribution", "ELO Distribution", "analytics_elo_distribution"),
|
||||
("time-control", "Time Control Analysis", "analytics_time_control_stats"),
|
||||
("hourly-activity", "Hourly Activity", "analytics_hourly_activity"),
|
||||
("weekly-activity", "Weekly Activity", "analytics_weekly_activity"),
|
||||
("rating-mismatch", "Rating Mismatch (Upsets)", "analytics_rating_mismatch"),
|
||||
("termination-stats", "Termination Types", "analytics_termination_stats"),
|
||||
]
|
||||
|
||||
CSS = """
|
||||
@@ -66,23 +73,31 @@ data:
|
||||
padding: 1.5rem; color: #8b949e; }
|
||||
"""
|
||||
|
||||
def read_csv_parts(subdir: str) -> tuple[list[str], list[list[str]]]:
|
||||
files = sorted(glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv"))
|
||||
headers: list[str] = []
|
||||
rows: list[list[str]] = []
|
||||
for f in files:
|
||||
with open(f, newline="", encoding="utf-8") as fh:
|
||||
reader = csv.reader(fh)
|
||||
for i, row in enumerate(reader):
|
||||
if i == 0 and not headers:
|
||||
headers = row
|
||||
elif headers:
|
||||
rows.append(row)
|
||||
return headers, rows
|
||||
def get_conn():
|
||||
return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS)
|
||||
|
||||
def dataset_status(subdir: str) -> tuple[int, int]:
|
||||
files = glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv")
|
||||
return len(files), sum(os.path.getsize(f) for f in files)
|
||||
def table_row_count(table: str) -> int:
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
return cur.fetchone()[0]
|
||||
except Exception:
|
||||
return -1
|
||||
|
||||
def fetch_rows(table: str, limit: int = 10_000):
|
||||
with get_conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute(f"SELECT * FROM {table} LIMIT %s", (limit,))
|
||||
rows = cur.fetchall()
|
||||
if not rows:
|
||||
return [], []
|
||||
headers = list(rows[0].keys())
|
||||
data = [
|
||||
[str(row[h]) if row[h] is not None else "" for h in headers]
|
||||
for row in rows
|
||||
]
|
||||
return headers, data
|
||||
|
||||
def page(title: str, body: str) -> str:
|
||||
return (
|
||||
@@ -96,26 +111,31 @@ data:
|
||||
|
||||
def index_html() -> str:
|
||||
cards = ""
|
||||
for slug, label, subdir in DATASETS:
|
||||
parts, size_bytes = dataset_status(subdir)
|
||||
if parts:
|
||||
size_kb = size_bytes // 1024
|
||||
badge = f'<span class="badge ready">{parts} part(s) · {size_kb} KB</span>'
|
||||
else:
|
||||
for slug, label, table in DATASETS:
|
||||
count = table_row_count(table)
|
||||
if count > 0:
|
||||
badge = f'<span class="badge ready">{count} rows</span>'
|
||||
elif count == 0:
|
||||
badge = '<span class="badge">no data yet</span>'
|
||||
else:
|
||||
badge = '<span class="badge">table missing</span>'
|
||||
cards += (
|
||||
f'<div class="card"><a href="/{slug}">'
|
||||
f'<div class="card-label"><h3>{html.escape(label)}</h3></div>'
|
||||
f"{badge}</a></div>"
|
||||
)
|
||||
return page(
|
||||
"Results",
|
||||
f"<h2>Datasets</h2><div class='cards'>{cards}</div>",
|
||||
)
|
||||
return page("Results", f"<h2>Datasets</h2><div class='cards'>{cards}</div>")
|
||||
|
||||
def table_html(slug: str, label: str, subdir: str) -> str:
|
||||
headers, rows = read_csv_parts(subdir)
|
||||
def table_html(slug: str, label: str, table: str) -> str:
|
||||
back = '<a class="back" href="/">← All datasets</a>'
|
||||
try:
|
||||
headers, rows = fetch_rows(table)
|
||||
except Exception as e:
|
||||
return page(
|
||||
label,
|
||||
f"{back}<h2>{html.escape(label)}</h2>"
|
||||
f"<div class='notice'>Error querying {html.escape(table)}: {html.escape(str(e))}</div>",
|
||||
)
|
||||
if not headers:
|
||||
return page(
|
||||
label,
|
||||
@@ -136,7 +156,7 @@ data:
|
||||
f"<tbody>{trs}</tbody></table></div>",
|
||||
)
|
||||
|
||||
SLUG_MAP = {s: (label, subdir) for s, label, subdir in DATASETS}
|
||||
SLUG_MAP = {s: (label, table) for s, label, table in DATASETS}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
@@ -147,8 +167,8 @@ data:
|
||||
if path == "" or path == "index.html":
|
||||
self._send(index_html())
|
||||
elif path in SLUG_MAP:
|
||||
label, subdir = SLUG_MAP[path]
|
||||
self._send(table_html(path, label, subdir))
|
||||
label, table = SLUG_MAP[path]
|
||||
self._send(table_html(path, label, table))
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
@@ -163,7 +183,7 @@ data:
|
||||
|
||||
if __name__ == "__main__":
|
||||
server = HTTPServer(("0.0.0.0", PORT), Handler)
|
||||
print(f"Listening on :{PORT} OUTPUT_DIR={OUTPUT_DIR}", flush=True)
|
||||
print(f"Listening on :{PORT} DB={DB_HOST}:{DB_PORT}/{DB_NAME}", flush=True)
|
||||
server.serve_forever()
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
@@ -194,25 +214,35 @@ spec:
|
||||
containers:
|
||||
- name: webview
|
||||
image: python:3.12-slim
|
||||
command: ["python", "/scripts/serve.py"]
|
||||
command: ["sh", "-c", "pip install psycopg2-binary --quiet && python /scripts/serve.py"]
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
- name: SPARK_OUTPUT_DIR
|
||||
value: /spark-output
|
||||
- name: PORT
|
||||
value: "8080"
|
||||
- name: NOWCHESS_JDBC_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_URL
|
||||
- name: NOWCHESS_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_USER
|
||||
- name: NOWCHESS_DB_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ncs-db-secrets
|
||||
key: STORE_DB_PASSWORD
|
||||
volumeMounts:
|
||||
- name: spark-output
|
||||
mountPath: /spark-output
|
||||
readOnly: true
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8080
|
||||
initialDelaySeconds: 3
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
@@ -222,9 +252,6 @@ spec:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
volumes:
|
||||
- name: spark-output
|
||||
persistentVolumeClaim:
|
||||
claimName: spark-analytics-output
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: spark-analytics-webview
|
||||
|
||||
@@ -205,6 +205,132 @@ patches:
|
||||
target:
|
||||
kind: Deployment
|
||||
name: spark-live-dashboard
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-game-length
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-game-length
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-color-advantage
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-color-advantage
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-elo-distribution
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-elo-distribution
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-time-control
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-time-control
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-daily-activity
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-daily-activity
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-rating-mismatch
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-rating-mismatch
|
||||
- patch: |-
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: spark-termination-stats
|
||||
spec:
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: spark-driver
|
||||
env:
|
||||
- name: NOWCHESS_PGN_PATH
|
||||
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
|
||||
target:
|
||||
kind: CronJob
|
||||
name: spark-termination-stats
|
||||
- patch: |-
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
|
||||
Reference in New Issue
Block a user