fix(spark-analytics): increase NIM timeout and trim max_tokens to avoid read timeouts
70B model at 30-50 tok/s was exceeding the 30s urllib timeout for larger outputs. Raise server timeout to 60s, add browser AbortController at 65s so the UI fails gracefully. Trim max_tokens (row: 300, dataset: 512) to keep generation well inside the window. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -182,11 +182,19 @@ data:
|
||||
}
|
||||
|
||||
async function nimFetch(payload) {
|
||||
const resp = await fetch('/explain', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const ctrl = new AbortController();
|
||||
const tid = setTimeout(() => ctrl.abort(), 65000);
|
||||
let resp;
|
||||
try {
|
||||
resp = await fetch('/explain', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(tid);
|
||||
}
|
||||
if (resp.status === 429) {
|
||||
const retryAfter = parseInt(resp.headers.get('Retry-After') || '10', 10);
|
||||
const err = new Error('Rate limited — retry in ' + retryAfter + 's');
|
||||
@@ -494,7 +502,7 @@ data:
|
||||
f"Entry to analyze: {row_desc}\n\n"
|
||||
"Provide a detailed, chess-specific analysis of this entry."
|
||||
)
|
||||
max_tokens = 600
|
||||
max_tokens = 300
|
||||
else:
|
||||
sample = dataset.get("sample", [])
|
||||
rows_text = "\n".join(
|
||||
@@ -514,7 +522,7 @@ data:
|
||||
f"Top 10 entries:\n{rows_text}\n\n"
|
||||
"Analyze each entry."
|
||||
)
|
||||
max_tokens = 1024
|
||||
max_tokens = 512
|
||||
|
||||
payload = json.dumps({
|
||||
"model": "meta/llama-3.3-70b-instruct",
|
||||
@@ -536,7 +544,7 @@ data:
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
nim_data = json.loads(r.read())
|
||||
text = nim_data.get("choices", [{}])[0].get("message", {}).get("content", "(no response)")
|
||||
self._send_json({"text": text})
|
||||
|
||||
Reference in New Issue
Block a user