From a190d50a098939cf210c76ce4f408913eca5b848 Mon Sep 17 00:00:00 2001 From: Janis Eccarius Date: Sun, 21 Jun 2026 15:01:25 +0200 Subject: [PATCH] feat(spark-analytics): add 7 CronJobs, results webview, and LiChess staging demo Adds CronJobs and headless driver Services for GameLengthJob, ColorAdvantageJob, EloDistributionJob, TimeControlJob, DailyActivityJob, RatingMismatchJob, and TerminationStatsJob. All suspended by default; scheduled Saturday mornings. Adds spark-analytics-webview: a stdlib Python HTTP server (ConfigMap + Deployment + Service) that reads CSV output from the spark-analytics-output PVC and renders 13 datasets as browsable HTML tables. Disabled (replicas=0) in base. Staging overlay enables the webview (replicas=1), exposes it via Ingress at spark-results-st.nowchess.janis-eccarius.de, and sets NOWCHESS_PGN_PATH on all CronJobs to the Lichess Jan-2013 rated standard PGN dump for demo runs without a NowChess database. Co-Authored-By: Claude Sonnet 4.6 --- spark-analytics/base/cronjobs.yaml | 713 ++++++++++++++++++ spark-analytics/base/kustomization.yaml | 1 + spark-analytics/base/webview.yaml | 246 ++++++ .../eu-central-1-staging/kustomization.yaml | 23 + .../eu-central-1-staging/webview-ingress.yaml | 28 + 5 files changed, 1011 insertions(+) create mode 100755 spark-analytics/base/webview.yaml create mode 100755 spark-analytics/eu-central-1-staging/webview-ingress.yaml diff --git a/spark-analytics/base/cronjobs.yaml b/spark-analytics/base/cronjobs.yaml index ecd776f..e406463 100755 --- a/spark-analytics/base/cronjobs.yaml +++ b/spark-analytics/base/cronjobs.yaml @@ -74,6 +74,139 @@ spec: port: 44777 targetPort: 44777 --- +apiVersion: v1 +kind: Service +metadata: + name: spark-game-length-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-game-length + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-color-advantage-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-color-advantage + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-elo-distribution-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-elo-distribution + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-time-control-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-time-control + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-daily-activity-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-daily-activity + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-rating-mismatch-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-rating-mismatch + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-termination-stats-driver + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + clusterIP: None + selector: + app: spark-termination-stats + ports: + - name: driver + port: 44167 + targetPort: 44167 + - name: blockmanager + port: 44777 + targetPort: 44777 +--- apiVersion: batch/v1 kind: CronJob metadata: @@ -407,3 +540,583 @@ spec: - name: spark-output persistentVolumeClaim: claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-game-length + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 2 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-game-length + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-game-length-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.GameLengthJob + - file:///app/analytics.jar + - /spark-output/game-length + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-color-advantage + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 3 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-color-advantage + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-color-advantage-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.ColorAdvantageJob + - file:///app/analytics.jar + - /spark-output/color-advantage + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-elo-distribution + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 4 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-elo-distribution + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-elo-distribution-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.EloDistributionJob + - file:///app/analytics.jar + - /spark-output/elo-distribution + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-time-control + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 5 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-time-control + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-time-control-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.TimeControlJob + - file:///app/analytics.jar + - /spark-output/time-control + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-daily-activity + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 6 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-daily-activity + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-daily-activity-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.DailyActivityJob + - file:///app/analytics.jar + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-rating-mismatch + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 7 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-rating-mismatch + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-rating-mismatch-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.RatingMismatchJob + - file:///app/analytics.jar + - /spark-output/rating-mismatch + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: spark-termination-stats + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + schedule: "0 8 * * 6" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: + app: spark-termination-stats + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + serviceAccountName: spark-analytics + restartPolicy: OnFailure + securityContext: + fsGroup: 185 + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: spark-driver + image: ghcr.io/now-chess/now-chess-systems/analytics:latest + command: + - /opt/spark/bin/spark-submit + - --master + - spark://spark-master:7077 + - --conf + - spark.executor.instances=1 + - --conf + - spark.executor.cores=1 + - --conf + - spark.executor.memory=512m + - --conf + - spark.driver.memory=512m + - --conf + - spark.driver.host=spark-termination-stats-driver + - --conf + - spark.driver.port=44167 + - --conf + - spark.blockManager.port=44777 + - --class + - de.nowchess.analytics.TerminationStatsJob + - file:///app/analytics.jar + - /spark-output/termination-stats + env: + - name: NOWCHESS_JDBC_URL + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_URL + - name: NOWCHESS_DB_USER + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_USER + - name: NOWCHESS_DB_PASS + valueFrom: + secretKeyRef: + name: ncs-db-secrets + key: STORE_DB_PASSWORD + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: spark-output + mountPath: /spark-output + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output diff --git a/spark-analytics/base/kustomization.yaml b/spark-analytics/base/kustomization.yaml index 1a2aa6b..7f6e777 100755 --- a/spark-analytics/base/kustomization.yaml +++ b/spark-analytics/base/kustomization.yaml @@ -7,3 +7,4 @@ resources: - pvc.yaml - cronjobs.yaml - streaming.yaml + - webview.yaml diff --git a/spark-analytics/base/webview.yaml b/spark-analytics/base/webview.yaml new file mode 100755 index 0000000..81352f8 --- /dev/null +++ b/spark-analytics/base/webview.yaml @@ -0,0 +1,246 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: spark-analytics-webview + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +data: + serve.py: | + #!/usr/bin/env python3 + """Spark analytics results viewer — reads CSV output from PVC, serves HTML tables.""" + import csv + import glob + import html + import os + from http.server import BaseHTTPRequestHandler, HTTPServer + + OUTPUT_DIR = os.environ.get("SPARK_OUTPUT_DIR", "/spark-output") + PORT = int(os.environ.get("PORT", "8080")) + + # Each tuple: (url-slug, display label, path relative to SPARK_OUTPUT_DIR) + # Paths mirror the CronJob outputDir arg + the subdir each job writes CSV to. + DATASETS = [ + ("opening-book", "Opening Book (Top 1000)", "opening-book/opening_book_top1000"), + ("player-stats", "Player Statistics", "player-stats/player_stats_csv"), + ("cluster-archetypes", "Cluster Archetypes", "player-clusters/cluster_archetypes"), + ("component-sizes", "Graph Component Sizes", "player-graph/component_sizes"), + ("game-length", "Game Length Distribution", "game-length/game_length_distribution"), + ("game-length-by-result", "Game Length by Result", "game-length/game_length_by_result"), + ("color-advantage", "Color Advantage", "color-advantage/color_advantage"), + ("elo-distribution", "ELO Distribution", "elo-distribution/elo_distribution"), + ("time-control", "Time Control Analysis", "time-control/time_control_stats"), + ("hourly-activity", "Hourly Activity", "daily-activity/hourly_activity"), + ("weekly-activity", "Weekly Activity", "daily-activity/weekly_activity"), + ("rating-mismatch", "Rating Mismatch (Upsets)", "rating-mismatch/rating_mismatch"), + ("termination-stats", "Termination Types", "termination-stats/termination_stats"), + ] + + CSS = """ + * { box-sizing: border-box; } + body { font-family: 'Segoe UI', sans-serif; margin: 0; background: #0d1117; color: #c9d1d9; } + header { background: #161b22; border-bottom: 1px solid #30363d; padding: 1rem 2rem; } + header h1 { margin: 0; color: #58a6ff; font-size: 1.25rem; font-weight: 600; } + header span { color: #8b949e; font-size: 0.875rem; margin-left: 0.5rem; } + main { padding: 1.5rem 2rem; } + h2 { color: #e6edf3; font-size: 1rem; font-weight: 600; margin: 0 0 1rem; } + .cards { display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 1rem; } + .card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; } + .card h3 { margin: 0 0 0.5rem; font-size: 0.9rem; color: #58a6ff; } + .card a { text-decoration: none; color: inherit; display: block; } + .card a:hover .card-label { text-decoration: underline; } + .badge { display: inline-block; background: #21262d; border: 1px solid #30363d; + border-radius: 12px; padding: 1px 8px; font-size: 0.75rem; color: #8b949e; } + .badge.ready { border-color: #238636; color: #3fb950; } + .back { color: #58a6ff; text-decoration: none; font-size: 0.875rem; } + .back:hover { text-decoration: underline; } + .meta { color: #8b949e; font-size: 0.8rem; margin: 0.5rem 0 1rem; } + table { width: 100%; border-collapse: collapse; font-size: 0.8rem; } + thead th { background: #161b22; position: sticky; top: 0; padding: 6px 12px; + text-align: left; color: #8b949e; border-bottom: 1px solid #30363d; + font-weight: 600; white-space: nowrap; } + tbody td { padding: 5px 12px; border-bottom: 1px solid #21262d; } + tbody tr:hover td { background: #161b22; } + .table-wrap { overflow-x: auto; border: 1px solid #30363d; border-radius: 6px; } + .notice { background: #161b22; border: 1px solid #30363d; border-radius: 6px; + padding: 1.5rem; color: #8b949e; } + """ + + def read_csv_parts(subdir: str) -> tuple[list[str], list[list[str]]]: + files = sorted(glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv")) + headers: list[str] = [] + rows: list[list[str]] = [] + for f in files: + with open(f, newline="", encoding="utf-8") as fh: + reader = csv.reader(fh) + for i, row in enumerate(reader): + if i == 0 and not headers: + headers = row + elif headers: + rows.append(row) + return headers, rows + + def dataset_status(subdir: str) -> tuple[int, int]: + files = glob.glob(f"{OUTPUT_DIR}/{subdir}/part-*.csv") + return len(files), sum(os.path.getsize(f) for f in files) + + def page(title: str, body: str) -> str: + return ( + f"" + f"" + f"{html.escape(title)} — Spark Analytics" + f"" + f"

Spark Analytics

NowChess · staging
" + f"
{body}
" + ) + + def index_html() -> str: + cards = "" + for slug, label, subdir in DATASETS: + parts, size_bytes = dataset_status(subdir) + if parts: + size_kb = size_bytes // 1024 + badge = f'{parts} part(s) · {size_kb} KB' + else: + badge = 'no data yet' + cards += ( + f'
' + f'

{html.escape(label)}

' + f"{badge}
" + ) + return page( + "Results", + f"

Datasets

{cards}
", + ) + + def table_html(slug: str, label: str, subdir: str) -> str: + headers, rows = read_csv_parts(subdir) + back = '← All datasets' + if not headers: + return page( + label, + f"{back}

{html.escape(label)}

" + "
No data yet. Run the CronJob first, then check back.
", + ) + ths = "".join(f"{html.escape(h)}" for h in headers) + trs = "".join( + "" + "".join(f"{html.escape(str(c))}" for c in row) + "" + for row in rows[:10_000] + ) + truncated = f" (showing first 10 000 of {len(rows)})" if len(rows) > 10_000 else "" + return page( + label, + f"{back}

{html.escape(label)}

" + f"

{len(rows)} rows{truncated}

" + f"
{ths}" + f"{trs}
", + ) + + SLUG_MAP = {s: (label, subdir) for s, label, subdir in DATASETS} + + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + pass + + def do_GET(self): + path = self.path.split("?")[0].lstrip("/") + if path == "" or path == "index.html": + self._send(index_html()) + elif path in SLUG_MAP: + label, subdir = SLUG_MAP[path] + self._send(table_html(path, label, subdir)) + else: + self.send_response(404) + self.end_headers() + + def _send(self, body: str): + data = body.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", PORT), Handler) + print(f"Listening on :{PORT} OUTPUT_DIR={OUTPUT_DIR}", flush=True) + server.serve_forever() +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spark-analytics-webview + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + replicas: 0 + strategy: + type: Recreate + selector: + matchLabels: + app: spark-analytics-webview + template: + metadata: + labels: + app: spark-analytics-webview + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + containers: + - name: webview + image: python:3.12-slim + command: ["python", "/scripts/serve.py"] + ports: + - containerPort: 8080 + env: + - name: SPARK_OUTPUT_DIR + value: /spark-output + - name: PORT + value: "8080" + volumeMounts: + - name: spark-output + mountPath: /spark-output + readOnly: true + - name: scripts + mountPath: /scripts + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 10 + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + volumes: + - name: spark-output + persistentVolumeClaim: + claimName: spark-analytics-output + - name: scripts + configMap: + name: spark-analytics-webview + defaultMode: 0755 +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-analytics-webview + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess +spec: + selector: + app: spark-analytics-webview + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/spark-analytics/eu-central-1-staging/kustomization.yaml b/spark-analytics/eu-central-1-staging/kustomization.yaml index fa8ee6e..0e52fd2 100755 --- a/spark-analytics/eu-central-1-staging/kustomization.yaml +++ b/spark-analytics/eu-central-1-staging/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization namespace: nowchess-staging resources: - ../base + - webview-ingress.yaml images: - name: ghcr.io/now-chess/now-chess-systems/analytics newTag: 0.2.0 @@ -42,6 +43,9 @@ patches: - file:///app/analytics.jar - /spark-output/opening-book - "10" + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" target: kind: CronJob name: spark-opening-book @@ -79,6 +83,9 @@ patches: - de.nowchess.analytics.PlayerStatsJob - file:///app/analytics.jar - /spark-output/player-stats + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" target: kind: CronJob name: spark-player-stats @@ -117,6 +124,9 @@ patches: - file:///app/analytics.jar - /spark-output/player-clusters - "4" + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" target: kind: CronJob name: spark-player-clustering @@ -154,6 +164,9 @@ patches: - de.nowchess.analytics.PlayerGraphJob - file:///app/analytics.jar - /spark-output/player-graph + env: + - name: NOWCHESS_PGN_PATH + value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst" target: kind: CronJob name: spark-player-graph @@ -192,3 +205,13 @@ patches: target: kind: Deployment name: spark-live-dashboard + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: spark-analytics-webview + spec: + replicas: 1 + target: + kind: Deployment + name: spark-analytics-webview diff --git a/spark-analytics/eu-central-1-staging/webview-ingress.yaml b/spark-analytics/eu-central-1-staging/webview-ingress.yaml new file mode 100755 index 0000000..6fb51b5 --- /dev/null +++ b/spark-analytics/eu-central-1-staging/webview-ingress.yaml @@ -0,0 +1,28 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: spark-analytics-webview-ingress + labels: + app.kubernetes.io/name: spark-analytics + app.kubernetes.io/part-of: nowchess + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" +spec: + ingressClassName: nginx + rules: + - host: spark-results-st.nowchess.janis-eccarius.de + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: spark-analytics-webview + port: + number: 8080 + tls: + - hosts: + - spark-results-st.nowchess.janis-eccarius.de + secretName: spark-analytics-webview-staging-cert