Janis Eccarius
ca78173a56
- Rewrite webview serve.py to query PostgreSQL analytics tables via psycopg2 instead of reading CSV files from PVC; PVC mount removed from Deployment - Remove isPgnMode guard from all 4 original jobs so results always write to DB regardless of whether input comes from game_records or a Lichess PGN dump - Add JDBC write-back to all 7 new analytics jobs (game_length_distribution, game_length_by_result, color_advantage, elo_distribution, time_control_stats, hourly_activity, weekly_activity, rating_mismatch, termination_stats) - Add analytics_component_sizes JDBC write to PlayerGraphJob - Add NOWCHESS_PGN_PATH staging patches for the 7 new CronJobs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
274 lines
11 KiB
YAML
Executable File
274 lines
11 KiB
YAML
Executable File
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: spark-analytics-webview
|
|
labels:
|
|
app.kubernetes.io/name: spark-analytics
|
|
app.kubernetes.io/part-of: nowchess
|
|
data:
|
|
serve.py: |
|
|
#!/usr/bin/env python3
|
|
"""Spark analytics results viewer — queries PostgreSQL analytics tables, serves HTML tables."""
|
|
import html
|
|
import os
|
|
import re
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
|
|
PORT = int(os.environ.get("PORT", "8080"))
|
|
|
|
_url = os.environ.get("NOWCHESS_JDBC_URL", "jdbc:postgresql://localhost:5432/nowchess")
|
|
_m = re.match(r"jdbc:postgresql://([^:/]+):(\d+)/(.+)", _url)
|
|
DB_HOST = _m.group(1) if _m else "localhost"
|
|
DB_PORT = int(_m.group(2)) if _m else 5432
|
|
DB_NAME = _m.group(3) if _m else "nowchess"
|
|
DB_USER = os.environ.get("NOWCHESS_DB_USER", "nowchess")
|
|
DB_PASS = os.environ.get("NOWCHESS_DB_PASS", "")
|
|
|
|
# Each tuple: (url-slug, display label, postgres table name)
|
|
DATASETS = [
|
|
("opening-book", "Opening Book (Top 1000)", "analytics_opening_stats"),
|
|
("player-stats", "Player Statistics", "analytics_player_stats"),
|
|
("cluster-archetypes", "Cluster Archetypes", "analytics_cluster_archetypes"),
|
|
("component-sizes", "Graph Component Sizes", "analytics_component_sizes"),
|
|
("game-length", "Game Length Distribution", "analytics_game_length_distribution"),
|
|
("game-length-by-result", "Game Length by Result", "analytics_game_length_by_result"),
|
|
("color-advantage", "Color Advantage", "analytics_color_advantage"),
|
|
("elo-distribution", "ELO Distribution", "analytics_elo_distribution"),
|
|
("time-control", "Time Control Analysis", "analytics_time_control_stats"),
|
|
("hourly-activity", "Hourly Activity", "analytics_hourly_activity"),
|
|
("weekly-activity", "Weekly Activity", "analytics_weekly_activity"),
|
|
("rating-mismatch", "Rating Mismatch (Upsets)", "analytics_rating_mismatch"),
|
|
("termination-stats", "Termination Types", "analytics_termination_stats"),
|
|
]
|
|
|
|
CSS = """
|
|
* { box-sizing: border-box; }
|
|
body { font-family: 'Segoe UI', sans-serif; margin: 0; background: #0d1117; color: #c9d1d9; }
|
|
header { background: #161b22; border-bottom: 1px solid #30363d; padding: 1rem 2rem; }
|
|
header h1 { margin: 0; color: #58a6ff; font-size: 1.25rem; font-weight: 600; }
|
|
header span { color: #8b949e; font-size: 0.875rem; margin-left: 0.5rem; }
|
|
main { padding: 1.5rem 2rem; }
|
|
h2 { color: #e6edf3; font-size: 1rem; font-weight: 600; margin: 0 0 1rem; }
|
|
.cards { display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 1rem; }
|
|
.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; }
|
|
.card h3 { margin: 0 0 0.5rem; font-size: 0.9rem; color: #58a6ff; }
|
|
.card a { text-decoration: none; color: inherit; display: block; }
|
|
.card a:hover .card-label { text-decoration: underline; }
|
|
.badge { display: inline-block; background: #21262d; border: 1px solid #30363d;
|
|
border-radius: 12px; padding: 1px 8px; font-size: 0.75rem; color: #8b949e; }
|
|
.badge.ready { border-color: #238636; color: #3fb950; }
|
|
.back { color: #58a6ff; text-decoration: none; font-size: 0.875rem; }
|
|
.back:hover { text-decoration: underline; }
|
|
.meta { color: #8b949e; font-size: 0.8rem; margin: 0.5rem 0 1rem; }
|
|
table { width: 100%; border-collapse: collapse; font-size: 0.8rem; }
|
|
thead th { background: #161b22; position: sticky; top: 0; padding: 6px 12px;
|
|
text-align: left; color: #8b949e; border-bottom: 1px solid #30363d;
|
|
font-weight: 600; white-space: nowrap; }
|
|
tbody td { padding: 5px 12px; border-bottom: 1px solid #21262d; }
|
|
tbody tr:hover td { background: #161b22; }
|
|
.table-wrap { overflow-x: auto; border: 1px solid #30363d; border-radius: 6px; }
|
|
.notice { background: #161b22; border: 1px solid #30363d; border-radius: 6px;
|
|
padding: 1.5rem; color: #8b949e; }
|
|
"""
|
|
|
|
def get_conn():
|
|
return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS)
|
|
|
|
def table_row_count(table: str) -> int:
|
|
try:
|
|
with get_conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(f"SELECT COUNT(*) FROM {table}")
|
|
return cur.fetchone()[0]
|
|
except Exception:
|
|
return -1
|
|
|
|
def fetch_rows(table: str, limit: int = 10_000):
|
|
with get_conn() as conn:
|
|
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
|
cur.execute(f"SELECT * FROM {table} LIMIT %s", (limit,))
|
|
rows = cur.fetchall()
|
|
if not rows:
|
|
return [], []
|
|
headers = list(rows[0].keys())
|
|
data = [
|
|
[str(row[h]) if row[h] is not None else "" for h in headers]
|
|
for row in rows
|
|
]
|
|
return headers, data
|
|
|
|
def page(title: str, body: str) -> str:
|
|
return (
|
|
f"<!doctype html><html lang=en><head><meta charset=utf-8>"
|
|
f"<meta name=viewport content='width=device-width,initial-scale=1'>"
|
|
f"<title>{html.escape(title)} — Spark Analytics</title>"
|
|
f"<style>{CSS}</style></head><body>"
|
|
f"<header><h1>Spark Analytics</h1><span>NowChess · staging</span></header>"
|
|
f"<main>{body}</main></body></html>"
|
|
)
|
|
|
|
def index_html() -> str:
|
|
cards = ""
|
|
for slug, label, table in DATASETS:
|
|
count = table_row_count(table)
|
|
if count > 0:
|
|
badge = f'<span class="badge ready">{count} rows</span>'
|
|
elif count == 0:
|
|
badge = '<span class="badge">no data yet</span>'
|
|
else:
|
|
badge = '<span class="badge">table missing</span>'
|
|
cards += (
|
|
f'<div class="card"><a href="/{slug}">'
|
|
f'<div class="card-label"><h3>{html.escape(label)}</h3></div>'
|
|
f"{badge}</a></div>"
|
|
)
|
|
return page("Results", f"<h2>Datasets</h2><div class='cards'>{cards}</div>")
|
|
|
|
def table_html(slug: str, label: str, table: str) -> str:
|
|
back = '<a class="back" href="/">← All datasets</a>'
|
|
try:
|
|
headers, rows = fetch_rows(table)
|
|
except Exception as e:
|
|
return page(
|
|
label,
|
|
f"{back}<h2>{html.escape(label)}</h2>"
|
|
f"<div class='notice'>Error querying {html.escape(table)}: {html.escape(str(e))}</div>",
|
|
)
|
|
if not headers:
|
|
return page(
|
|
label,
|
|
f"{back}<h2>{html.escape(label)}</h2>"
|
|
"<div class='notice'>No data yet. Run the CronJob first, then check back.</div>",
|
|
)
|
|
ths = "".join(f"<th>{html.escape(h)}</th>" for h in headers)
|
|
trs = "".join(
|
|
"<tr>" + "".join(f"<td>{html.escape(str(c))}</td>" for c in row) + "</tr>"
|
|
for row in rows[:10_000]
|
|
)
|
|
truncated = f" (showing first 10 000 of {len(rows)})" if len(rows) > 10_000 else ""
|
|
return page(
|
|
label,
|
|
f"{back}<h2>{html.escape(label)}</h2>"
|
|
f"<p class='meta'>{len(rows)} rows{truncated}</p>"
|
|
f"<div class='table-wrap'><table><thead><tr>{ths}</tr></thead>"
|
|
f"<tbody>{trs}</tbody></table></div>",
|
|
)
|
|
|
|
SLUG_MAP = {s: (label, table) for s, label, table in DATASETS}
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, fmt, *args):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
path = self.path.split("?")[0].lstrip("/")
|
|
if path == "" or path == "index.html":
|
|
self._send(index_html())
|
|
elif path in SLUG_MAP:
|
|
label, table = SLUG_MAP[path]
|
|
self._send(table_html(path, label, table))
|
|
else:
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
|
|
def _send(self, body: str):
|
|
data = body.encode("utf-8")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
if __name__ == "__main__":
|
|
server = HTTPServer(("0.0.0.0", PORT), Handler)
|
|
print(f"Listening on :{PORT} DB={DB_HOST}:{DB_PORT}/{DB_NAME}", flush=True)
|
|
server.serve_forever()
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: spark-analytics-webview
|
|
labels:
|
|
app.kubernetes.io/name: spark-analytics
|
|
app.kubernetes.io/part-of: nowchess
|
|
spec:
|
|
replicas: 0
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: spark-analytics-webview
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: spark-analytics-webview
|
|
app.kubernetes.io/name: spark-analytics
|
|
app.kubernetes.io/part-of: nowchess
|
|
spec:
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
fsGroup: 65534
|
|
containers:
|
|
- name: webview
|
|
image: python:3.12-slim
|
|
command: ["sh", "-c", "pip install psycopg2-binary --quiet && python /scripts/serve.py"]
|
|
ports:
|
|
- containerPort: 8080
|
|
env:
|
|
- name: PORT
|
|
value: "8080"
|
|
- name: NOWCHESS_JDBC_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: ncs-db-secrets
|
|
key: STORE_DB_URL
|
|
- name: NOWCHESS_DB_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: ncs-db-secrets
|
|
key: STORE_DB_USER
|
|
- name: NOWCHESS_DB_PASS
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: ncs-db-secrets
|
|
key: STORE_DB_PASSWORD
|
|
volumeMounts:
|
|
- name: scripts
|
|
mountPath: /scripts
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /
|
|
port: 8080
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 10
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
volumes:
|
|
- name: scripts
|
|
configMap:
|
|
name: spark-analytics-webview
|
|
defaultMode: 0755
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: spark-analytics-webview
|
|
labels:
|
|
app.kubernetes.io/name: spark-analytics
|
|
app.kubernetes.io/part-of: nowchess
|
|
spec:
|
|
selector:
|
|
app: spark-analytics-webview
|
|
ports:
|
|
- name: http
|
|
port: 8080
|
|
targetPort: 8080
|