ci: show github errors nicer formatted

tests/run.py

@@ -3,6 +3,7 @@
 import argparse
 import json
 import multiprocessing
+import os
 import shlex
 import subprocess
 import sys
@@ -16,6 +17,37 @@ GREEN = "\033[92m"
 RED = "\033[91m"
 RESET = "\033[0m"
 
+def is_github_actions() -> bool:
+    """Check if running in GitHub Actions environment."""
+    return os.getenv("GITHUB_ACTIONS") == "true"
+
+def github_error(message: str, title: str = "") -> None:
+    """Output GitHub Actions error annotation."""
+    if title:
+        print(f"::error title={title}::{message}")
+    else:
+        print(f"::error::{message}")
+
+def github_warning(message: str, title: str = "") -> None:
+    """Output GitHub Actions warning annotation."""
+    if title:
+        print(f"::warning title={title}::{message}")
+    else:
+        print(f"::warning::{message}")
+
+def format_nix_error(error_text: str) -> str:
+    """Format nix evaluation error for better readability."""
+    lines = error_text.strip().split('\n')
+    # Try to extract the most relevant error line
+    for line in lines:
+        if 'error:' in line.lower():
+            return line.strip()
+    # If no specific error line found, return first non-empty line
+    for line in lines:
+        if line.strip():
+            return line.strip()
+    return error_text.strip()
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Run hardware tests")
     parser.add_argument(
@@ -63,8 +95,19 @@ def run_eval_test(gcroot_dir: Path, jobs: int) -> list[str]:
             attr = data.get("attr")
             if "error" in data:
                 failed_profiles.append(attr)
+                error_msg = data['error']
+                formatted_error = format_nix_error(error_msg)
+                
+                # Output for terminal
                 print(f"{RED}FAIL {attr}:{RESET}", file=sys.stderr)
-                print(f"{RED}{data['error']}{RESET}", file=sys.stderr)
+                print(f"{RED}{error_msg}{RESET}", file=sys.stderr)
+                
+                # Output for GitHub Actions
+                if is_github_actions():
+                    github_error(
+                        formatted_error,
+                        title=f"Hardware profile evaluation failed: {attr}"
+                    )
             else:
                 print(f"{GREEN}OK {attr}{RESET}")
     return failed_profiles
@@ -80,9 +123,18 @@ def main() -> None:
         failed_profiles = run_eval_test(gcroot_dir, args.jobs)
 
     if len(failed_profiles) > 0:
-        print(f"\n{RED}The following {len(failed_profiles)} test(s) failed:{RESET}")
+        failure_msg = f"The following {len(failed_profiles)} test(s) failed:"
+        print(f"\n{RED}{failure_msg}{RESET}")
         for profile in failed_profiles:
             print(f" '{profile}'")
+        
+        # GitHub Actions summary
+        if is_github_actions():
+            github_error(
+                f"{failure_msg} {', '.join(failed_profiles)}",
+                title="Hardware Profile Tests Failed"
+            )
+        
         sys.exit(1)