diff --git a/spark-analytics/base/webview.yaml b/spark-analytics/base/webview.yaml index 81352f8..38d1730 100755 --- a/spark-analytics/base/webview.yaml +++ b/spark-analytics/base/webview.yaml @@ -8,32 +8,39 @@ metadata: data: serve.py: | #!/usr/bin/env python3 - """Spark analytics results viewer โ€” reads CSV output from PVC, serves HTML tables.""" - import csv - import glob + """Spark analytics results viewer โ€” queries PostgreSQL analytics tables, serves HTML tables.""" import html import os + import re + import psycopg2 + import psycopg2.extras from http.server import BaseHTTPRequestHandler, HTTPServer - OUTPUT_DIR = os.environ.get("SPARK_OUTPUT_DIR", "/spark-output") PORT = int(os.environ.get("PORT", "8080")) - # Each tuple: (url-slug, display label, path relative to SPARK_OUTPUT_DIR) - # Paths mirror the CronJob outputDir arg + the subdir each job writes CSV to. + _url = os.environ.get("NOWCHESS_JDBC_URL", "jdbc:postgresql://localhost:5432/nowchess") + _m = re.match(r"jdbc:postgresql://([^:/]+):(\d+)/(.+)", _url) + DB_HOST = _m.group(1) if _m else "localhost" + DB_PORT = int(_m.group(2)) if _m else 5432 + DB_NAME = _m.group(3) if _m else "nowchess" + DB_USER = os.environ.get("NOWCHESS_DB_USER", "nowchess") + DB_PASS = os.environ.get("NOWCHESS_DB_PASS", "") + + # Each tuple: (url-slug, display label, postgres table name) DATASETS = [ - ("opening-book", "Opening Book (Top 1000)", "opening-book/opening_book_top1000"), - ("player-stats", "Player Statistics", "player-stats/player_stats_csv"), - ("cluster-archetypes", "Cluster Archetypes", "player-clusters/cluster_archetypes"), - ("component-sizes", "Graph Component Sizes", "player-graph/component_sizes"), - ("game-length", "Game Length Distribution", "game-length/game_length_distribution"), - ("game-length-by-result", "Game Length by Result", "game-length/game_length_by_result"), - ("color-advantage", "Color Advantage", "color-advantage/color_advantage"), - ("elo-distribution", "ELO Distribution", "elo-distribution/elo_distribution"), - ("time-control", "Time Control Analysis", "time-control/time_control_stats"), - ("hourly-activity", "Hourly Activity", "daily-activity/hourly_activity"), - ("weekly-activity", "Weekly Activity", "daily-activity/weekly_activity"), - ("rating-mismatch", "Rating Mismatch (Upsets)", "rating-mismatch/rating_mismatch"), - ("termination-stats", "Termination Types", "termination-stats/termination_stats"), + ("opening-book", "Opening Book (Top 1000)", "analytics_opening_stats"), + ("player-stats", "Player Statistics", "analytics_player_stats"), + ("cluster-archetypes", "Cluster Archetypes", "analytics_cluster_archetypes"), + ("component-sizes", "Graph Component Sizes", "analytics_component_sizes"), + ("game-length", "Game Length Distribution", "analytics_game_length_distribution"), + ("game-length-by-result", "Game Length by Result", "analytics_game_length_by_result"), + ("color-advantage", "Color Advantage", "analytics_color_advantage"), + ("elo-distribution", "ELO Distribution", "analytics_elo_distribution"), + ("time-control", "Time Control Analysis", "analytics_time_control_stats"), + ("hourly-activity", "Hourly Activity", "analytics_hourly_activity"), + ("weekly-activity", "Weekly Activity", "analytics_weekly_activity"), + ("rating-mismatch", "Rating Mismatch (Upsets)", "analytics_rating_mismatch"), + ("termination-stats", "Termination Types", "analytics_termination_stats"), ] CSS = """ @@ -66,23 +73,31 @@ data: padding: 1.5rem; color: #8b949e; } """ - def read_csv_parts(subdir: str) -> tuple[list[str], list[list[str]]]: - files = sorted(glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv")) - headers: list[str] = [] - rows: list[list[str]] = [] - for f in files: - with open(f, newline="", encoding="utf-8") as fh: - reader = csv.reader(fh) - for i, row in enumerate(reader): - if i == 0 and not headers: - headers = row - elif headers: - rows.append(row) - return headers, rows + def get_conn(): + return psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS) - def dataset_status(subdir: str) -> tuple[int, int]: - files = glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv") - return len(files), sum(os.path.getsize(f) for f in files) + def table_row_count(table: str) -> int: + try: + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute(f"SELECT COUNT(*) FROM {table}") + return cur.fetchone()[0] + except Exception: + return -1 + + def fetch_rows(table: str, limit: int = 10_000): + with get_conn() as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(f"SELECT * FROM {table} LIMIT %s", (limit,)) + rows = cur.fetchall() + if not rows: + return [], [] + headers = list(rows[0].keys()) + data = [ + [str(row[h]) if row[h] is not None else "" for h in headers] + for row in rows + ] + return headers, data def page(title: str, body: str) -> str: return ( @@ -96,26 +111,31 @@ data: def index_html() -> str: cards = "" - for slug, label, subdir in DATASETS: - parts, size_bytes = dataset_status(subdir) - if parts: - size_kb = size_bytes // 1024 - badge = f'{parts} part(s) ยท {size_kb} KB' - else: + for slug, label, table in DATASETS: + count = table_row_count(table) + if count > 0: + badge = f'{count} rows' + elif count == 0: badge = 'no data yet' + else: + badge = 'table missing' cards += ( f'
' f'

{html.escape(label)}

' f"{badge}
" ) - return page( - "Results", - f"

Datasets

{cards}
", - ) + return page("Results", f"

Datasets

{cards}
") - def table_html(slug: str, label: str, subdir: str) -> str: - headers, rows = read_csv_parts(subdir) + def table_html(slug: str, label: str, table: str) -> str: back = 'โ† All datasets' + try: + headers, rows = fetch_rows(table) + except Exception as e: + return page( + label, + f"{back}

{html.escape(label)}

" + f"
Error querying {html.escape(table)}: {html.escape(str(e))}
", + ) if not headers: return page( label, @@ -136,7 +156,7 @@ data: f"{trs}", ) - SLUG_MAP = {s: (label, subdir) for s, label, subdir in DATASETS} + SLUG_MAP = {s: (label, table) for s, label, table in DATASETS} class Handler(BaseHTTPRequestHandler): def log_message(self, fmt, *args): @@ -147,8 +167,8 @@ data: if path == "" or path == "index.html": self._send(index_html()) elif path in SLUG_MAP: - label, subdir = SLUG_MAP[path] - self._send(table_html(path, label, subdir)) + label, table = SLUG_MAP[path] + self._send(table_html(path, label, table)) else: self.send_response(404) self.end_headers() @@ -163,7 +183,7 @@ data: if __name__ == "__main__": server = HTTPServer(("0.0.0.0", PORT), Handler) - print(f"Listening on :{PORT} OUTPUT_DIR={OUTPUT_DIR}", flush=True) + print(f"Listening on :{PORT} DB={DB_HOST}:{DB_PORT}/{DB_NAME}", flush=True) server.serve_forever() --- apiVersion: apps/v1 @@ -194,25 +214,35 @@ spec: containers: - name: webview image: python:3.12-slim - command: ["python", "/scripts/serve.py"] + command: ["sh", "-c", "pip install psycopg2-binary --quiet && python /scripts/serve.py"] ports: - containerPort: 8080 env: - - name: SPARK_OUTPUT_DIR - value: /spark-output - name: PORT value: "8080" + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD volumeMounts: - - name: spark-output - mountPath: /spark-output - readOnly: true - name: scripts mountPath: /scripts readinessProbe: httpGet: path: / port: 8080 - initialDelaySeconds: 3 + initialDelaySeconds: 15 periodSeconds: 10 resources: requests: @@ -222,9 +252,6 @@ spec: cpu: 200m memory: 256Mi volumes: - - name: spark-output - persistentVolumeClaim: - claimName: spark-analytics-output - name: scripts configMap: name: spark-analytics-webview diff --git a/spark-analytics/eu-central-1-staging/kustomization.yaml b/spark-analytics/eu-central-1-staging/kustomization.yaml index 0e52fd2..501a5d3 100755 --- a/spark-analytics/eu-central-1-staging/kustomization.yaml +++ b/spark-analytics/eu-central-1-staging/kustomization.yaml @@ -205,6 +205,132 @@ patches: target: kind: Deployment name: spark-live-dashboard + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-game-length + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-game-length + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-color-advantage + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-color-advantage + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-elo-distribution + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-elo-distribution + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-time-control + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-time-control + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-daily-activity + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-daily-activity + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-rating-mismatch + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-rating-mismatch + - patch: |- + apiVersion: batch/v1 + kind: CronJob + metadata: + name: spark-termination-stats + spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: spark-driver + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" + target: + kind: CronJob + name: spark-termination-stats - patch: |- apiVersion: apps/v1 kind: Deployment