NowChess/GitOps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(spark-analytics): add AccuracyBlunder, ClockPressure, SmurfAnomaly jobs and webview datasets

Browse Source 
Adds three new CronJobs (spark-accuracy-blunder, spark-clock-pressure,
spark-smurf-anomaly) and registers their six output tables in the
analytics webview (accuracy_by_rating, blunder_outcome, clock_by_rating,
scramble_outcome, smurf_anomaly, flagged_smurfs).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Janis Eccarius
2026-06-24 00:17:02 +02:00
parent b3fffec838
commit f64b838030
3 changed files with 398 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spark-analytics/base/cronjobs.yaml
+328
View File
@@ -1197,3 +1197,331 @@ spec:
- name: spark-output - name: spark-output
persistentVolumeClaim: persistentVolumeClaim:
claimName: spark-analytics-output claimName: spark-analytics-output
---
apiVersion: v1
kind: Service
metadata:
name: spark-accuracy-blunder-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-accuracy-blunder
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: v1
kind: Service
metadata:
name: spark-clock-pressure-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-clock-pressure
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: v1
kind: Service
metadata:
name: spark-smurf-anomaly-driver
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
clusterIP: None
selector:
app: spark-smurf-anomaly
ports:
- name: driver
port: 44167
targetPort: 44167
- name: blockmanager
port: 44777
targetPort: 44777
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-accuracy-blunder
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 9 * * 6"
suspend: true
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app: spark-accuracy-blunder
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=1
- --conf
- spark.executor.cores=1
- --conf
- spark.executor.memory=512m
- --conf
- spark.driver.memory=512m
- --conf
- spark.driver.host=spark-accuracy-blunder-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class
- de.nowchess.analytics.AccuracyBlunderJob
- file:///app/analytics.jar
- /spark-output/accuracy
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-clock-pressure
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 10 * * 6"
suspend: true
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app: spark-clock-pressure
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=1
- --conf
- spark.executor.cores=1
- --conf
- spark.executor.memory=512m
- --conf
- spark.driver.memory=512m
- --conf
- spark.driver.host=spark-clock-pressure-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class
- de.nowchess.analytics.ClockPressureJob
- file:///app/analytics.jar
- /spark-output/clock-pressure
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-smurf-anomaly
labels:
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
schedule: "0 11 * * 6"
suspend: true
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
app: spark-smurf-anomaly
app.kubernetes.io/name: spark-analytics
app.kubernetes.io/part-of: nowchess
spec:
serviceAccountName: spark-analytics
restartPolicy: OnFailure
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: spark-worker
topologyKey: kubernetes.io/hostname
securityContext:
fsGroup: 185
imagePullSecrets:
- name: ghcr-pull-secret
containers:
- name: spark-driver
image: ghcr.io/now-chess/now-chess-systems/analytics:latest
command:
- /opt/spark/bin/spark-submit
- --master
- spark://spark-master:7077
- --conf
- spark.executor.instances=1
- --conf
- spark.executor.cores=1
- --conf
- spark.executor.memory=512m
- --conf
- spark.driver.memory=512m
- --conf
- spark.driver.host=spark-smurf-anomaly-driver
- --conf
- spark.driver.port=44167
- --conf
- spark.blockManager.port=44777
- --class
- de.nowchess.analytics.SmurfAnomalyJob
- file:///app/analytics.jar
- /spark-output/smurf-anomaly
- "15"
env:
- name: NOWCHESS_JDBC_URL
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_URL
- name: NOWCHESS_DB_USER
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_USER
- name: NOWCHESS_DB_PASS
valueFrom:
secretKeyRef:
name: ncs-db-secrets
key: STORE_DB_PASSWORD
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: spark-output
mountPath: /spark-output
volumes:
- name: spark-output
persistentVolumeClaim:
claimName: spark-analytics-output
spark-analytics/base/webview.yaml
+6
View File
@@ -44,6 +44,12 @@ data:
("weekly-activity", "Weekly Activity", "analytics_weekly_activity"), ("weekly-activity", "Weekly Activity", "analytics_weekly_activity"),
("rating-mismatch", "Rating Mismatch (Upsets)", "analytics_rating_mismatch"), ("rating-mismatch", "Rating Mismatch (Upsets)", "analytics_rating_mismatch"),
("termination-stats", "Termination Types", "analytics_termination_stats"), ("termination-stats", "Termination Types", "analytics_termination_stats"),
("accuracy-by-rating", "Accuracy by Rating", "analytics_accuracy_by_rating"),
("blunder-outcome", "Blunder Outcome", "analytics_blunder_outcome"),
("clock-by-rating", "Clock Pressure by Rating", "analytics_clock_by_rating"),
("scramble-outcome", "Scramble Outcome", "analytics_scramble_outcome"),
("smurf-anomaly", "Smurf Anomaly Scores", "analytics_smurf_anomaly"),
("flagged-smurfs", "Flagged Smurfs", "analytics_flagged_smurfs"),
] ]
CSS = """ CSS = """
spark-analytics/eu-central-1-staging/kustomization.yaml
+64
View File
@@ -353,6 +353,70 @@ patches:
target: target:
kind: CronJob kind: CronJob
name: spark-termination-stats name: spark-termination-stats
# Eval-annotated dump required for per-move accuracy; the default 2013 dump
# carries no [%eval] comments — point this at an evals dump for real output.
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-accuracy-blunder
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
env:
- name: NOWCHESS_PGN_PATH
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
- name: NOWCHESS_PGN_CACHE_DIR
value: "/spark-output/.pgn-cache"
target:
kind: CronJob
name: spark-accuracy-blunder
# Clock-annotated dump required; the default 2013 dump carries no [%clk]
# comments — point this at a dump with clocks for real output.
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-clock-pressure
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
env:
- name: NOWCHESS_PGN_PATH
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
- name: NOWCHESS_PGN_CACHE_DIR
value: "/spark-output/.pgn-cache"
target:
kind: CronJob
name: spark-clock-pressure
- patch: |-
apiVersion: batch/v1
kind: CronJob
metadata:
name: spark-smurf-anomaly
spec:
jobTemplate:
spec:
template:
spec:
containers:
- name: spark-driver
env:
- name: NOWCHESS_PGN_PATH
value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
- name: NOWCHESS_PGN_CACHE_DIR
value: "/spark-output/.pgn-cache"
target:
kind: CronJob
name: spark-smurf-anomaly
- patch: |- - patch: |-
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment