feat(spark-analytics): add AccuracyBlunder, ClockPressure, SmurfAnomaly jobs and webview datasets

Adds three new CronJobs (spark-accuracy-blunder, spark-clock-pressure, spark-smurf-anomaly) and registers their six output tables in the analytics webview (accuracy_by_rating, blunder_outcome, clock_by_rating, scramble_outcome, smurf_anomaly, flagged_smurfs). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-24 00:17:02 +02:00
parent b3fffec838
commit f64b838030
3 changed files with 398 additions and 0 deletions
@@ -1197,3 +1197,331 @@ spec:
            - name: spark-output
              persistentVolumeClaim:
                claimName: spark-analytics-output
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-accuracy-blunder-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-accuracy-blunder
+  ports:
+    - name: driver
+      port: 44167
+      targetPort: 44167
+    - name: blockmanager
+      port: 44777
+      targetPort: 44777
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-clock-pressure-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-clock-pressure
+  ports:
+    - name: driver
+      port: 44167
+      targetPort: 44167
+    - name: blockmanager
+      port: 44777
+      targetPort: 44777
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-smurf-anomaly-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-smurf-anomaly
+  ports:
+    - name: driver
+      port: 44167
+      targetPort: 44167
+    - name: blockmanager
+      port: 44777
+      targetPort: 44777
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: spark-accuracy-blunder
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  schedule: "0 9 * * 6"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: spark-accuracy-blunder
+            app.kubernetes.io/name: spark-analytics
+            app.kubernetes.io/part-of: nowchess
+        spec:
+          serviceAccountName: spark-analytics
+          restartPolicy: OnFailure
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: spark-worker
+                  topologyKey: kubernetes.io/hostname
+          securityContext:
+            fsGroup: 185
+          imagePullSecrets:
+            - name: ghcr-pull-secret
+          containers:
+            - name: spark-driver
+              image: ghcr.io/now-chess/now-chess-systems/analytics:latest
+              command:
+                - /opt/spark/bin/spark-submit
+                - --master
+                - spark://spark-master:7077
+                - --conf
+                - spark.executor.instances=1
+                - --conf
+                - spark.executor.cores=1
+                - --conf
+                - spark.executor.memory=512m
+                - --conf
+                - spark.driver.memory=512m
+                - --conf
+                - spark.driver.host=spark-accuracy-blunder-driver
+                - --conf
+                - spark.driver.port=44167
+                - --conf
+                - spark.blockManager.port=44777
+                - --class
+                - de.nowchess.analytics.AccuracyBlunderJob
+                - file:///app/analytics.jar
+                - /spark-output/accuracy
+              env:
+                - name: NOWCHESS_JDBC_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_URL
+                - name: NOWCHESS_DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_USER
+                - name: NOWCHESS_DB_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_PASSWORD
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+              volumeMounts:
+                - name: spark-output
+                  mountPath: /spark-output
+          volumes:
+            - name: spark-output
+              persistentVolumeClaim:
+                claimName: spark-analytics-output
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: spark-clock-pressure
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  schedule: "0 10 * * 6"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: spark-clock-pressure
+            app.kubernetes.io/name: spark-analytics
+            app.kubernetes.io/part-of: nowchess
+        spec:
+          serviceAccountName: spark-analytics
+          restartPolicy: OnFailure
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: spark-worker
+                  topologyKey: kubernetes.io/hostname
+          securityContext:
+            fsGroup: 185
+          imagePullSecrets:
+            - name: ghcr-pull-secret
+          containers:
+            - name: spark-driver
+              image: ghcr.io/now-chess/now-chess-systems/analytics:latest
+              command:
+                - /opt/spark/bin/spark-submit
+                - --master
+                - spark://spark-master:7077
+                - --conf
+                - spark.executor.instances=1
+                - --conf
+                - spark.executor.cores=1
+                - --conf
+                - spark.executor.memory=512m
+                - --conf
+                - spark.driver.memory=512m
+                - --conf
+                - spark.driver.host=spark-clock-pressure-driver
+                - --conf
+                - spark.driver.port=44167
+                - --conf
+                - spark.blockManager.port=44777
+                - --class
+                - de.nowchess.analytics.ClockPressureJob
+                - file:///app/analytics.jar
+                - /spark-output/clock-pressure
+              env:
+                - name: NOWCHESS_JDBC_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_URL
+                - name: NOWCHESS_DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_USER
+                - name: NOWCHESS_DB_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_PASSWORD
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+              volumeMounts:
+                - name: spark-output
+                  mountPath: /spark-output
+          volumes:
+            - name: spark-output
+              persistentVolumeClaim:
+                claimName: spark-analytics-output
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: spark-smurf-anomaly
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  schedule: "0 11 * * 6"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: spark-smurf-anomaly
+            app.kubernetes.io/name: spark-analytics
+            app.kubernetes.io/part-of: nowchess
+        spec:
+          serviceAccountName: spark-analytics
+          restartPolicy: OnFailure
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: spark-worker
+                  topologyKey: kubernetes.io/hostname
+          securityContext:
+            fsGroup: 185
+          imagePullSecrets:
+            - name: ghcr-pull-secret
+          containers:
+            - name: spark-driver
+              image: ghcr.io/now-chess/now-chess-systems/analytics:latest
+              command:
+                - /opt/spark/bin/spark-submit
+                - --master
+                - spark://spark-master:7077
+                - --conf
+                - spark.executor.instances=1
+                - --conf
+                - spark.executor.cores=1
+                - --conf
+                - spark.executor.memory=512m
+                - --conf
+                - spark.driver.memory=512m
+                - --conf
+                - spark.driver.host=spark-smurf-anomaly-driver
+                - --conf
+                - spark.driver.port=44167
+                - --conf
+                - spark.blockManager.port=44777
+                - --class
+                - de.nowchess.analytics.SmurfAnomalyJob
+                - file:///app/analytics.jar
+                - /spark-output/smurf-anomaly
+                - "15"
+              env:
+                - name: NOWCHESS_JDBC_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_URL
+                - name: NOWCHESS_DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_USER
+                - name: NOWCHESS_DB_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_PASSWORD
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+              volumeMounts:
+                - name: spark-output
+                  mountPath: /spark-output
+          volumes:
+            - name: spark-output
+              persistentVolumeClaim:
+                claimName: spark-analytics-output
@@ -44,6 +44,12 @@ data:
        ("weekly-activity",       "Weekly Activity",            "analytics_weekly_activity"),
        ("rating-mismatch",       "Rating Mismatch (Upsets)",   "analytics_rating_mismatch"),
        ("termination-stats",     "Termination Types",          "analytics_termination_stats"),
+        ("accuracy-by-rating",    "Accuracy by Rating",         "analytics_accuracy_by_rating"),
+        ("blunder-outcome",       "Blunder Outcome",            "analytics_blunder_outcome"),
+        ("clock-by-rating",       "Clock Pressure by Rating",   "analytics_clock_by_rating"),
+        ("scramble-outcome",      "Scramble Outcome",           "analytics_scramble_outcome"),
+        ("smurf-anomaly",         "Smurf Anomaly Scores",       "analytics_smurf_anomaly"),
+        ("flagged-smurfs",        "Flagged Smurfs",             "analytics_flagged_smurfs"),
    ]

    CSS = """
@@ -353,6 +353,70 @@ patches:
    target:
      kind: CronJob
      name: spark-termination-stats
+  # Eval-annotated dump required for per-move accuracy; the default 2013 dump
+  # carries no [%eval] comments — point this at an evals dump for real output.
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-accuracy-blunder
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-accuracy-blunder
+  # Clock-annotated dump required; the default 2013 dump carries no [%clk]
+  # comments — point this at a dump with clocks for real output.
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-clock-pressure
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-clock-pressure
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-smurf-anomaly
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-smurf-anomaly
  - patch: |-
      apiVersion: apps/v1
      kind: Deployment