feat: NCS-21 Write Scripts to automate certain tasks (#15)
Build & Test (NowChessSystems) TeamCity build finished
Build & Test (NowChessSystems) TeamCity build finished
Co-authored-by: shahdlala66 <shahd.lala66@gmail.com> Reviewed-on: #15 Co-authored-by: Janis <janis.e.20@gmx.de> Co-committed-by: Janis <janis.e.20@gmx.de>
This commit was merged in pull request #15.
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Gap Reporter
|
||||
Scans JUnit XML test results under modules/*/build/test-results/*.xml and
|
||||
outputs a minimal summary optimised for agent consumption.
|
||||
|
||||
Usage:
|
||||
python test_gaps.py # scan all modules (default)
|
||||
python test_gaps.py --module chess # single module
|
||||
python test_gaps.py --module all # explicit all
|
||||
python test_gaps.py --modules-dir ./modules
|
||||
python test_gaps.py --results-subdir build/test-results
|
||||
"""
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class TestCase:
|
||||
classname: str
|
||||
name: str
|
||||
time: float
|
||||
failure: Optional[str] = None # message if failed
|
||||
error: Optional[str] = None # message if errored
|
||||
skipped: bool = False
|
||||
|
||||
@property
|
||||
def short_class(self) -> str:
|
||||
return self.classname.split(".")[-1]
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
if self.failure is not None:
|
||||
return "FAIL"
|
||||
if self.error is not None:
|
||||
return "ERROR"
|
||||
if self.skipped:
|
||||
return "SKIP"
|
||||
return "OK"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SuiteResult:
|
||||
name: str
|
||||
total: int
|
||||
failures: int
|
||||
errors: int
|
||||
skipped: int
|
||||
time: float
|
||||
cases: list[TestCase] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def passed(self) -> int:
|
||||
return self.total - self.failures - self.errors - self.skipped
|
||||
|
||||
@property
|
||||
def is_clean(self) -> bool:
|
||||
return self.failures == 0 and self.errors == 0
|
||||
|
||||
@property
|
||||
def bad_cases(self) -> list[TestCase]:
|
||||
return [c for c in self.cases if c.status in ("FAIL", "ERROR")]
|
||||
|
||||
@property
|
||||
def skipped_cases(self) -> list[TestCase]:
|
||||
return [c for c in self.cases if c.skipped]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModuleResult:
|
||||
name: str
|
||||
suites: list[SuiteResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total(self) -> int: return sum(s.total for s in self.suites)
|
||||
@property
|
||||
def failures(self) -> int: return sum(s.failures for s in self.suites)
|
||||
@property
|
||||
def errors(self) -> int: return sum(s.errors for s in self.suites)
|
||||
@property
|
||||
def skipped(self) -> int: return sum(s.skipped for s in self.suites)
|
||||
@property
|
||||
def passed(self) -> int: return sum(s.passed for s in self.suites)
|
||||
@property
|
||||
def is_clean(self) -> bool: return self.failures == 0 and self.errors == 0
|
||||
|
||||
@property
|
||||
def bad_cases(self) -> list[TestCase]:
|
||||
return [c for s in self.suites for c in s.bad_cases]
|
||||
|
||||
@property
|
||||
def skipped_cases(self) -> list[TestCase]:
|
||||
return [c for s in self.suites for c in s.skipped_cases]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_suite_xml(xml_path: Path) -> SuiteResult:
|
||||
tree = ET.parse(xml_path)
|
||||
root = tree.getroot()
|
||||
|
||||
# Handle both <testsuite> root and <testsuites> wrapper
|
||||
suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
|
||||
|
||||
# Merge multiple suites from one file into a single SuiteResult
|
||||
total = failures = errors = skipped = 0
|
||||
elapsed = 0.0
|
||||
name = xml_path.stem
|
||||
cases: list[TestCase] = []
|
||||
|
||||
for suite in suites:
|
||||
total += int(suite.get("tests", 0))
|
||||
failures += int(suite.get("failures", 0))
|
||||
errors += int(suite.get("errors", 0))
|
||||
skipped += int(suite.get("skipped", 0))
|
||||
elapsed += float(suite.get("time", 0.0))
|
||||
if suite.get("name"):
|
||||
name = suite.get("name")
|
||||
|
||||
for tc in suite.findall("testcase"):
|
||||
fail_el = tc.find("failure")
|
||||
err_el = tc.find("error")
|
||||
skip_el = tc.find("skipped")
|
||||
cases.append(TestCase(
|
||||
classname=tc.get("classname", ""),
|
||||
name=tc.get("name", ""),
|
||||
time=float(tc.get("time", 0.0)),
|
||||
failure=fail_el.get("message", fail_el.text or "") if fail_el is not None else None,
|
||||
error=err_el.get("message", err_el.text or "") if err_el is not None else None,
|
||||
skipped=skip_el is not None,
|
||||
))
|
||||
|
||||
return SuiteResult(
|
||||
name=name, total=total, failures=failures,
|
||||
errors=errors, skipped=skipped, time=elapsed, cases=cases,
|
||||
)
|
||||
|
||||
|
||||
def load_module(module_dir: Path, results_subdir: str) -> Optional[ModuleResult]:
|
||||
results_dir = module_dir / results_subdir
|
||||
if not results_dir.is_dir():
|
||||
return None
|
||||
|
||||
xml_files = sorted(results_dir.glob("*.xml"))
|
||||
if not xml_files:
|
||||
return None
|
||||
|
||||
mod = ModuleResult(name=module_dir.name)
|
||||
for xml_path in xml_files:
|
||||
try:
|
||||
mod.suites.append(parse_suite_xml(xml_path))
|
||||
except ET.ParseError:
|
||||
pass # skip malformed files silently
|
||||
return mod if mod.suites else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Formatter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _truncate(text: str, max_len: int = 120) -> str:
|
||||
text = " ".join(text.split()) # collapse whitespace
|
||||
return text[:max_len] + "…" if len(text) > max_len else text
|
||||
|
||||
|
||||
def format_module(mod: ModuleResult) -> str:
|
||||
parts = [f"[{mod.name}]"]
|
||||
|
||||
if mod.is_clean and mod.skipped == 0:
|
||||
parts.append(f"tests: {mod.total} ✅")
|
||||
return " ".join(parts)
|
||||
|
||||
parts.append(f"tests: {mod.total}")
|
||||
if mod.failures: parts.append(f"failed: {mod.failures}")
|
||||
if mod.errors: parts.append(f"errors: {mod.errors}")
|
||||
if mod.skipped: parts.append(f"skipped: {mod.skipped}")
|
||||
|
||||
# Agent hint only when there are actual failures/errors
|
||||
if not mod.is_clean:
|
||||
parts.append(f" # hint: run ./test {mod.name} for details")
|
||||
|
||||
lines = [" ".join(parts)]
|
||||
|
||||
# List each failed/errored test — this IS the actionable info
|
||||
for tc in mod.bad_cases:
|
||||
msg = tc.failure if tc.failure is not None else tc.error
|
||||
label = f" {tc.status}: {tc.short_class} > {tc.name}"
|
||||
if msg:
|
||||
label += f" [{_truncate(msg, 80)}]"
|
||||
lines.append(label)
|
||||
|
||||
# Skipped: compact, one line total
|
||||
if mod.skipped_cases:
|
||||
skipped_names = ", ".join(
|
||||
f"{c.short_class}.{c.name}" for c in mod.skipped_cases[:5]
|
||||
)
|
||||
if len(mod.skipped_cases) > 5:
|
||||
skipped_names += f" (+{len(mod.skipped_cases) - 5} more)"
|
||||
lines.append(f" SKIP: {skipped_names}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run(modules_dir: str, results_subdir: str, module_filter: Optional[str]) -> None:
|
||||
base = Path(modules_dir)
|
||||
if not base.is_dir():
|
||||
print(f"ERROR: modules directory not found: {base}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Resolve which module dirs to scan
|
||||
if module_filter and module_filter != "all":
|
||||
mod_dir = base / module_filter
|
||||
if not mod_dir.is_dir():
|
||||
print(f"ERROR: module not found: {mod_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
candidates = [mod_dir]
|
||||
else:
|
||||
candidates = sorted(p for p in base.iterdir() if p.is_dir())
|
||||
|
||||
results: list[str] = []
|
||||
missing: list[str] = []
|
||||
|
||||
for mod_dir in candidates:
|
||||
if mod_dir.name.startswith("build"):
|
||||
continue
|
||||
mod = load_module(mod_dir, results_subdir)
|
||||
if mod is None:
|
||||
missing.append(mod_dir.name)
|
||||
continue
|
||||
results.append(format_module(mod))
|
||||
|
||||
print("\n".join(results))
|
||||
|
||||
if missing:
|
||||
print(
|
||||
f"\n# Modules without test results: {', '.join(missing)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Minimal test-gap reporter for JUnit XML results across modules."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--module", "-m",
|
||||
nargs="?",
|
||||
const="all",
|
||||
default="all",
|
||||
help="Module name to scan, or 'all' (default: all)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modules-dir",
|
||||
default="./modules",
|
||||
help="Root directory containing one sub-directory per module (default: ./modules)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--results-subdir",
|
||||
default="build/test-results/test",
|
||||
help="Sub-path inside each module dir where *.xml files live (default: build/test-results/test)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
filter_ = None if args.module == "all" else args.module
|
||||
run(args.modules_dir, args.results_subdir, filter_)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user