chore(kargo): update image tag official-bots to 0.44.0-x86_v2 on htwg-1-prod (#545)

chore(kargo): update image tag official-bots to 0.43.0-x86_v2 on htwg-1-prod

[View in Kargo UI](https://kargo.nowchess.janis-eccarius.de/project/nowchess-kargo/stage/htwg-1-prod)

---------

Co-authored-by: Kargo <no-reply@kargo.io>
Reviewed-on: #542
Co-authored-by: svc.kargo <kargo@blackhole.local>
Co-committed-by: svc.kargo <kargo@blackhole.local>

eu-central-1/argo-apps/kargo/kargo.yaml

@@ -11,7 +11,7 @@ spec:
   sources:
     - repoURL: ghcr.io/akuity/kargo-charts
       chart: kargo
-      targetRevision: 1.10.7
+      targetRevision: 1.10.8
       helm:
         valueFiles:
           - $values/kargo/eu-central-1/values.yaml

nowchess/eu-central-1-prod/kustomization.yaml

@@ -43,11 +43,11 @@ images:
 - name: ghcr.io/now-chess/now-chess-systems/core
   newTag: 0.51.0
 - name: ghcr.io/now-chess/now-chess-systems/frontend
-  newTag: 0.6.3
+  newTag: 0.7.0
 - name: ghcr.io/now-chess/now-chess-systems/io
   newTag: 0.23.0
 - name: ghcr.io/now-chess/now-chess-systems/official-bots
-  newTag: 0.36.0
+  newTag: 0.44.0
 - name: ghcr.io/now-chess/now-chess-systems/rule
   newTag: 0.18.0
 - name: ghcr.io/now-chess/now-chess-systems/store

nowchess/eu-central-1-staging/kustomization.yaml

@@ -120,11 +120,11 @@ images:
 - name: ghcr.io/now-chess/now-chess-systems/core
   newTag: 0.53.0
 - name: ghcr.io/now-chess/now-chess-systems/frontend
-  newTag: 0.6.3
+  newTag: 0.7.0
 - name: ghcr.io/now-chess/now-chess-systems/io
   newTag: 0.23.0
 - name: ghcr.io/now-chess/now-chess-systems/official-bots
-  newTag: 0.36.0
+  newTag: 0.44.0
 - name: ghcr.io/now-chess/now-chess-systems/rule
   newTag: 0.18.0
 - name: ghcr.io/now-chess/now-chess-systems/store

nowchess/htwg-1-prod/kustomization.yaml

@@ -84,11 +84,11 @@ images:
 - name: ghcr.io/now-chess/now-chess-systems/core
   newTag: 0.53.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/frontend
-  newTag: 0.6.3-x86_v2
+  newTag: 0.7.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/io
   newTag: 0.23.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/official-bots
-  newTag: 0.35.0-x86_v2
+  newTag: 0.44.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/rule
   newTag: 0.18.0-x86_v2
 - name: ghcr.io/now-chess/now-chess-systems/store

spark-analytics/base/cronjobs.yaml

@@ -1197,3 +1197,331 @@ spec:
             - name: spark-output
               persistentVolumeClaim:
                 claimName: spark-analytics-output
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-accuracy-blunder-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-accuracy-blunder
+  ports:
+    - name: driver
+      port: 44167
+      targetPort: 44167
+    - name: blockmanager
+      port: 44777
+      targetPort: 44777
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-clock-pressure-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-clock-pressure
+  ports:
+    - name: driver
+      port: 44167
+      targetPort: 44167
+    - name: blockmanager
+      port: 44777
+      targetPort: 44777
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-smurf-anomaly-driver
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  clusterIP: None
+  selector:
+    app: spark-smurf-anomaly
+  ports:
+    - name: driver
+      port: 44167
+      targetPort: 44167
+    - name: blockmanager
+      port: 44777
+      targetPort: 44777
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: spark-accuracy-blunder
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  schedule: "0 9 * * 6"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: spark-accuracy-blunder
+            app.kubernetes.io/name: spark-analytics
+            app.kubernetes.io/part-of: nowchess
+        spec:
+          serviceAccountName: spark-analytics
+          restartPolicy: OnFailure
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: spark-worker
+                  topologyKey: kubernetes.io/hostname
+          securityContext:
+            fsGroup: 185
+          imagePullSecrets:
+            - name: ghcr-pull-secret
+          containers:
+            - name: spark-driver
+              image: ghcr.io/now-chess/now-chess-systems/analytics:latest
+              command:
+                - /opt/spark/bin/spark-submit
+                - --master
+                - spark://spark-master:7077
+                - --conf
+                - spark.executor.instances=1
+                - --conf
+                - spark.executor.cores=1
+                - --conf
+                - spark.executor.memory=512m
+                - --conf
+                - spark.driver.memory=512m
+                - --conf
+                - spark.driver.host=spark-accuracy-blunder-driver
+                - --conf
+                - spark.driver.port=44167
+                - --conf
+                - spark.blockManager.port=44777
+                - --class
+                - de.nowchess.analytics.AccuracyBlunderJob
+                - file:///app/analytics.jar
+                - /spark-output/accuracy
+              env:
+                - name: NOWCHESS_JDBC_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_URL
+                - name: NOWCHESS_DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_USER
+                - name: NOWCHESS_DB_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_PASSWORD
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+              volumeMounts:
+                - name: spark-output
+                  mountPath: /spark-output
+          volumes:
+            - name: spark-output
+              persistentVolumeClaim:
+                claimName: spark-analytics-output
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: spark-clock-pressure
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  schedule: "0 10 * * 6"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: spark-clock-pressure
+            app.kubernetes.io/name: spark-analytics
+            app.kubernetes.io/part-of: nowchess
+        spec:
+          serviceAccountName: spark-analytics
+          restartPolicy: OnFailure
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: spark-worker
+                  topologyKey: kubernetes.io/hostname
+          securityContext:
+            fsGroup: 185
+          imagePullSecrets:
+            - name: ghcr-pull-secret
+          containers:
+            - name: spark-driver
+              image: ghcr.io/now-chess/now-chess-systems/analytics:latest
+              command:
+                - /opt/spark/bin/spark-submit
+                - --master
+                - spark://spark-master:7077
+                - --conf
+                - spark.executor.instances=1
+                - --conf
+                - spark.executor.cores=1
+                - --conf
+                - spark.executor.memory=512m
+                - --conf
+                - spark.driver.memory=512m
+                - --conf
+                - spark.driver.host=spark-clock-pressure-driver
+                - --conf
+                - spark.driver.port=44167
+                - --conf
+                - spark.blockManager.port=44777
+                - --class
+                - de.nowchess.analytics.ClockPressureJob
+                - file:///app/analytics.jar
+                - /spark-output/clock-pressure
+              env:
+                - name: NOWCHESS_JDBC_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_URL
+                - name: NOWCHESS_DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_USER
+                - name: NOWCHESS_DB_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_PASSWORD
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+              volumeMounts:
+                - name: spark-output
+                  mountPath: /spark-output
+          volumes:
+            - name: spark-output
+              persistentVolumeClaim:
+                claimName: spark-analytics-output
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: spark-smurf-anomaly
+  labels:
+    app.kubernetes.io/name: spark-analytics
+    app.kubernetes.io/part-of: nowchess
+spec:
+  schedule: "0 11 * * 6"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: spark-smurf-anomaly
+            app.kubernetes.io/name: spark-analytics
+            app.kubernetes.io/part-of: nowchess
+        spec:
+          serviceAccountName: spark-analytics
+          restartPolicy: OnFailure
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: spark-worker
+                  topologyKey: kubernetes.io/hostname
+          securityContext:
+            fsGroup: 185
+          imagePullSecrets:
+            - name: ghcr-pull-secret
+          containers:
+            - name: spark-driver
+              image: ghcr.io/now-chess/now-chess-systems/analytics:latest
+              command:
+                - /opt/spark/bin/spark-submit
+                - --master
+                - spark://spark-master:7077
+                - --conf
+                - spark.executor.instances=1
+                - --conf
+                - spark.executor.cores=1
+                - --conf
+                - spark.executor.memory=512m
+                - --conf
+                - spark.driver.memory=512m
+                - --conf
+                - spark.driver.host=spark-smurf-anomaly-driver
+                - --conf
+                - spark.driver.port=44167
+                - --conf
+                - spark.blockManager.port=44777
+                - --class
+                - de.nowchess.analytics.SmurfAnomalyJob
+                - file:///app/analytics.jar
+                - /spark-output/smurf-anomaly
+                - "15"
+              env:
+                - name: NOWCHESS_JDBC_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_URL
+                - name: NOWCHESS_DB_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_USER
+                - name: NOWCHESS_DB_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: ncs-db-secrets
+                      key: STORE_DB_PASSWORD
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+              volumeMounts:
+                - name: spark-output
+                  mountPath: /spark-output
+          volumes:
+            - name: spark-output
+              persistentVolumeClaim:
+                claimName: spark-analytics-output

spark-analytics/base/webview.yaml

@@ -44,6 +44,12 @@ data:
         ("weekly-activity",       "Weekly Activity",            "analytics_weekly_activity"),
         ("rating-mismatch",       "Rating Mismatch (Upsets)",   "analytics_rating_mismatch"),
         ("termination-stats",     "Termination Types",          "analytics_termination_stats"),
+        ("accuracy-by-rating",    "Accuracy by Rating",         "analytics_accuracy_by_rating"),
+        ("blunder-outcome",       "Blunder Outcome",            "analytics_blunder_outcome"),
+        ("clock-by-rating",       "Clock Pressure by Rating",   "analytics_clock_by_rating"),
+        ("scramble-outcome",      "Scramble Outcome",           "analytics_scramble_outcome"),
+        ("smurf-anomaly",         "Smurf Anomaly Scores",       "analytics_smurf_anomaly"),
+        ("flagged-smurfs",        "Flagged Smurfs",             "analytics_flagged_smurfs"),
     ]
 
     CSS = """
@@ -181,37 +187,52 @@ data:
       if (btn) btn.disabled = !hasKey();
     }
 
-    async function nimFetch(payload) {
+    async function nimStream(payload, onDelta) {
       const ctrl = new AbortController();
-      const tid = setTimeout(() => ctrl.abort(), 65000);
+      const tid = setTimeout(() => ctrl.abort(), 180000);
-      let resp;
       try {
-        resp = await fetch('/explain', {
+        const resp = await fetch('/explain', {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify(payload),
           signal: ctrl.signal,
         });
+        if (resp.status === 429) {
+          const retryAfter = parseInt(resp.headers.get('Retry-After') || '10', 10);
+          const err = new Error('Rate limited — retry in ' + retryAfter + 's');
+          err.retryAfter = retryAfter;
+          throw err;
+        }
+        if (!resp.ok) {
+          const msg = await resp.text();
+          throw new Error('Proxy ' + resp.status + ': ' + msg);
+        }
+        const reader = resp.body.getReader();
+        const dec = new TextDecoder();
+        const SEP = String.fromCharCode(10, 10);  // SSE frame separator (two newlines)
+        let buf = '', full = '';
+        for (;;) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          buf += dec.decode(value, { stream: true });
+          let idx;
+          while ((idx = buf.indexOf(SEP)) >= 0) {
+            const frame = buf.slice(0, idx).trim();
+            buf = buf.slice(idx + 2);
+            if (!frame.startsWith('data:')) continue;
+            const obj = JSON.parse(frame.slice(5).trim());
+            if (obj.error) throw new Error(obj.error);
+            if (obj.delta) { full += obj.delta; onDelta(full); }
+          }
+        }
+        return full || '(no response)';
       } finally {
         clearTimeout(tid);
       }
-      if (resp.status === 429) {
-        const retryAfter = parseInt(resp.headers.get('Retry-After') || '10', 10);
-        const err = new Error('Rate limited — retry in ' + retryAfter + 's');
-        err.retryAfter = retryAfter;
-        throw err;
-      }
-      if (!resp.ok) {
-        const msg = await resp.text();
-        throw new Error('Proxy ' + resp.status + ': ' + msg);
-      }
-      const data = await resp.json();
-      if (data.error) throw new Error(data.error);
-      return data.text || '(no response)';
     }
 
-    function queueNim(payload) {
+    function queueNim(payload, onDelta) {
-      const next = nimQueue.then(() => nimFetch(payload));
+      const next = nimQueue.then(() => nimStream(payload, onDelta));
       nimQueue = next.catch(() => {});
       return next;
     }
@@ -231,7 +252,10 @@ data:
       textEl.textContent = '';
 
       try {
-        const text = await queueNim({ key: key, mode: 'dataset', dataset: window.DATASET });
+        const text = await queueNim(
+          { key: key, mode: 'dataset', dataset: window.DATASET },
+          partial => { textEl.textContent = partial; },
+        );
         textEl.textContent = text;
       } catch (e) {
         textEl.className = 'explain-error';
@@ -269,10 +293,13 @@ data:
       textEl.textContent = 'Analyzing…';
 
       try {
-        const text = await queueNim({
+        const text = await queueNim(
-          key: key, mode: 'row',
+          {
-          dataset: { label: dataset.label, headers: dataset.headers, row: row },
+            key: key, mode: 'row',
-        });
+            dataset: { label: dataset.label, headers: dataset.headers, row: row },
+          },
+          partial => { textEl.textContent = partial; },
+        );
         rowCache[idx] = text;
         textEl.textContent = text;
         btn.innerHTML = '✓';
@@ -474,6 +501,7 @@ data:
                 self.end_headers()
                 return
             length = int(self.headers.get("Content-Length", 0))
+            headers_sent = False
             try:
                 body = json.loads(self.rfile.read(length))
                 api_key = body.get("key", "")
@@ -532,7 +560,7 @@ data:
                     ],
                     "max_tokens": max_tokens,
                     "temperature": 0.4,
-                    "stream": False,
+                    "stream": True,
                 }).encode("utf-8")
                 req = urllib.request.Request(
                     "https://integrate.api.nvidia.com/v1/chat/completions",
@@ -544,10 +572,7 @@ data:
                     method="POST",
                 )
                 try:
-                    with urllib.request.urlopen(req, timeout=60) as r:
+                    upstream = urllib.request.urlopen(req, timeout=110)
-                        nim_data = json.loads(r.read())
-                    text = nim_data.get("choices", [{}])[0].get("message", {}).get("content", "(no response)")
-                    self._send_json({"text": text})
                 except urllib.error.HTTPError as he:
                     if he.code == 429:
                         retry_after = he.headers.get("Retry-After", "10")
@@ -561,10 +586,46 @@ data:
                             self.wfile.write(resp_body)
                         except BrokenPipeError:
                             pass
-                    else:
+                        return
-                        self._send_json({"error": f"NIM API error {he.code}: {he.reason}"})
+                    raise RuntimeError(f"NIM API error {he.code}: {he.reason}")
+
+                # Relay NIM's SSE stream to the browser as our own SSE.
+                self.send_response(200)
+                self.send_header("Content-Type", "text/event-stream")
+                self.send_header("Cache-Control", "no-cache")
+                self.send_header("X-Accel-Buffering", "no")
+                self.end_headers()
+                headers_sent = True
+                for raw in upstream:
+                    line = raw.decode("utf-8", "replace").strip()
+                    if not line.startswith("data:"):
+                        continue
+                    data = line[5:].strip()
+                    if data == "[DONE]":
+                        break
+                    try:
+                        obj = json.loads(data)
+                    except json.JSONDecodeError:
+                        continue
+                    piece = obj.get("choices", [{}])[0].get("delta", {}).get("content", "")
+                    if piece and not self._emit({"delta": piece}):
+                        return
+                self._emit({"done": True})
             except Exception as e:
-                self._send_json({"error": str(e)})
+                if not headers_sent:
+                    self.send_response(200)
+                    self.send_header("Content-Type", "text/event-stream")
+                    self.send_header("Cache-Control", "no-cache")
+                    self.end_headers()
+                self._emit({"error": str(e)})
+
+        def _emit(self, obj: dict) -> bool:
+            try:
+                self.wfile.write(("data: " + json.dumps(obj) + "\n\n").encode("utf-8"))
+                self.wfile.flush()
+                return True
+            except (BrokenPipeError, ConnectionResetError):
+                return False
 
         def _send(self, body: str):
             data = body.encode("utf-8")

spark-analytics/eu-central-1-staging/kustomization.yaml

@@ -353,6 +353,70 @@ patches:
     target:
       kind: CronJob
       name: spark-termination-stats
+  # Eval-annotated dump required for per-move accuracy; the default 2013 dump
+  # carries no [%eval] comments — point this at an evals dump for real output.
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-accuracy-blunder
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-accuracy-blunder
+  # Clock-annotated dump required; the default 2013 dump carries no [%clk]
+  # comments — point this at a dump with clocks for real output.
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-clock-pressure
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-clock-pressure
+  - patch: |-
+      apiVersion: batch/v1
+      kind: CronJob
+      metadata:
+        name: spark-smurf-anomaly
+      spec:
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: spark-driver
+                    env:
+                      - name: NOWCHESS_PGN_PATH
+                        value: "https://database.lichess.org/standard/lichess_db_standard_rated_2013-01.pgn.zst"
+                      - name: NOWCHESS_PGN_CACHE_DIR
+                        value: "/spark-output/.pgn-cache"
+    target:
+      kind: CronJob
+      name: spark-smurf-anomaly
   - patch: |-
       apiVersion: apps/v1
       kind: Deployment

spark-analytics/eu-central-1-staging/webview-ingress.yaml

@@ -9,6 +9,10 @@ metadata:
     cert-manager.io/cluster-issuer: "letsencrypt-prod"
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    # AI /explain streams Server-Sent Events — disable response buffering so
+    # chunks flush to the client immediately, and allow gaps between chunks.
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "120"
 spec:
   ingressClassName: nginx
   rules: