[FEAT] Diagnostic combined-stdout console diff

mod_api/routes/system.py

@@ -4,6 +4,7 @@
 GET    /system/health                        Health check (unauthenticated)
 GET    /system/queue                         Queue status — active + queued runs
 GET    /runs/{id}/artifacts                  Run artifacts from GCS + local
+GET    /runs/{id}/console-diff               Diagnostic stdout vs previous run
 GET    /system/about                         Versions this platform runs against
 GET    /system/maintenance                   Maintenance state per platform
 PATCH  /system/maintenance/{platform}        Pause or resume a platform
@@ -20,11 +21,13 @@
 system:write, so a token can be narrowed further than the role allows.
 """
 
+import difflib
 import os
+import tempfile
 from datetime import datetime, timezone
 
 from flask import g, jsonify, request
-from sqlalchemy import text
+from sqlalchemy import func, text
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import joinedload
 
@@ -39,8 +42,10 @@
 from mod_api.schemas.system import (BlockedUserCreateSchema,
                                     ForbiddenExtensionCreateSchema,
                                     MaintenanceUpdateSchema)
+from mod_api.services.diff_service import compute_log_diff, read_log_lines
 from mod_api.services.status import batch_get_run_data, is_dummy_row
-from mod_api.services.storage import (get_log_file_path,
+from mod_api.services.storage import (get_combined_stdout_path,
+                                      get_log_file_path,
                                       get_test_results_base_path,
                                       resolve_artifact)
 from mod_api.utils import paginated_response, safe_resolve, single_response
@@ -370,6 +375,148 @@
     return paginated_response(paged, total, limit, offset)
 
 
+MAX_CONSOLE_DIFF_BYTES = 10 * 1024 * 1024
+UNIFIED_DIFF_LINE_CAP = 10000
+
+
+def _unlink_if_temp(path):
+    """Delete GCS downloads from get_combined_stdout_path, not local repo copies."""
+    if not path:
+        return
+    name = os.path.basename(path)
+    if not (name.startswith('console-') and name.endswith('.log')):
+        return
+    try:
+        temp_root = os.path.realpath(tempfile.gettempdir())
+        real = os.path.realpath(path)
+        if real == temp_root or real.startswith(temp_root + os.sep):
+            os.unlink(real)
+    except OSError:
+        pass
+
+
+def _previous_completed_run(test):
+    """Most recent completed run on the same platform with a lower id."""
+    latest = g.db.query(
+        TestProgress.test_id,
+        func.max(TestProgress.id).label('max_id'),
+    ).group_by(TestProgress.test_id).subquery()
+    return (
+        g.db.query(Test)
+        .join(latest, latest.c.test_id == Test.id)
+        .join(TestProgress, TestProgress.id == latest.c.max_id)
+        .filter(
+            Test.platform == test.platform,
+            Test.id < test.id,
+            TestProgress.status == TestStatus.completed,
+        )
+        .order_by(Test.id.desc())
+        .first()
+    )
+
+
+@mod_api.route('/runs/<run_id>/console-diff', methods=['GET'])
+@require_scope(Scope.RESULTS_READ)
+@validate_path_id('run_id')
+def get_console_diff(run_id):
+    """
+    Diagnostic unified/structured diff of combined VM stdout vs another run.
+
+    Default comparison is the previous completed run on the same platform.
+    This does not change CI pass/fail or goldens — stdout is a whole-run log.
+    """
+    test = Test.query.filter(Test.id == run_id).first()
+    if test is None:
+        return make_error_response(
+            'not_found', f'Run {run_id} not found.', http_status=404)
+
+    format_type = request.args.get('format', 'unified')
+    if format_type not in ('unified', 'structured'):
+        return make_error_response(
+            'validation_error',
+            'format must be unified or structured.',
+            http_status=400)
+
+    against_raw = request.args.get('against')
+    if against_raw is None or against_raw == '':
+        against_run = _previous_completed_run(test)
+        if against_run is None:
+            return make_error_response(
+                'not_found',
+                'No previous completed run on this platform to compare against.',
+                http_status=404)
+    else:
+        try:
+            against_id = int(against_raw)
+        except (TypeError, ValueError):
+            return make_error_response(
+                'validation_error', 'against must be an integer run id.',
+                http_status=400)
+        if against_id == run_id:
+            return make_error_response(
+                'validation_error', 'against must be a different run.',
+                http_status=400)
+        against_run = Test.query.filter(Test.id == against_id).first()
+        if against_run is None:
+            return make_error_response(
+                'not_found', f'Run {against_id} not found.', http_status=404)
+
+    current_path = get_combined_stdout_path(run_id)
+    previous_path = get_combined_stdout_path(against_run.id)
+    try:
+        if current_path is None:
+            return make_error_response(
+                'not_found',
+                f'combined_stdout.log for run {run_id} is not in storage.',
+                http_status=404)
+        if previous_path is None:
+            return make_error_response(
+                'not_found',
+                f'combined_stdout.log for run {against_run.id} is not in storage.',
+                http_status=404)
+
+        ids = {
+            'run_id': run_id,
+            'against_run_id': against_run.id,
+            'kind': 'console',
+        }
+
+        try:
+            if format_type == 'unified':
+                previous_lines = read_log_lines(
+                    previous_path, MAX_CONSOLE_DIFF_BYTES)
+                current_lines = read_log_lines(
+                    current_path, MAX_CONSOLE_DIFF_BYTES)
+                differ = list(difflib.unified_diff(
+                    previous_lines,
+                    current_lines,
+                    fromfile=f'run {against_run.id}',
+                    tofile=f'run {run_id}',
+                    lineterm='',
+                ))
+                if len(differ) > UNIFIED_DIFF_LINE_CAP:
+                    differ = differ[:UNIFIED_DIFF_LINE_CAP]
+                    differ.append('... Diff truncated due to length ...')
+                return single_response({
+                    **ids,
+                    'format': 'unified',
+                    'content': '\n'.join(differ),
+                })
+
+            context_lines = request.args.get('context_lines', 3, type=int)
+            diff_result = compute_log_diff(
+                previous_path, current_path, context_lines=context_lines)
+            diff_result.update(ids)
+            diff_result['format'] = 'structured'
+            return single_response(diff_result)
+        except ValueError as exc:
+            return make_error_response(
+                'unprocessable', str(exc), http_status=422)
+    finally:
+        _unlink_if_temp(current_path)
+        _unlink_if_temp(previous_path)
+
+
 @mod_api.route('/system/about', methods=['GET'])
 @require_scope(Scope.SYSTEM_READ)
 def system_about():

mod_api/services/diff_service.py

@@ -218,3 +218,67 @@ def file_sha256(file_path: str) -> Optional[str]:
         return sha.hexdigest()
     except (OSError, IOError):
         return None
+
+
+def read_log_lines(file_path: str, max_size_bytes: int = 10 * 1024 * 1024) -> List[str]:
+    """Read a console log for diagnostic diffs (not TestResults-bound)."""
+    file_size = os.path.getsize(file_path)
+    if file_size > max_size_bytes:
+        raise ValueError(
+            f"File too large ({file_size} bytes > {max_size_bytes} limit)")
+    try:
+        with open(file_path, encoding='utf8') as f:
+            return [line.rstrip('\n\r') for line in f.readlines()]
+    except UnicodeDecodeError:
+        with open(file_path, encoding='cp1252', errors='replace') as f:
+            return [line.rstrip('\n\r') for line in f.readlines()]
+
+
+def compute_log_diff(
+    previous_path: str,
+    current_path: str,
+    context_lines: int = 3,
+    max_hunks: int = 500,
+) -> Dict[str, Any]:
+    """Structured diff of two combined_stdout logs. Does not affect CI verdicts."""
+    context_lines = max(1, min(context_lines, 50))
+
+    if not os.path.isfile(previous_path):
+        return {
+            'status': 'missing_expected',
+            'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0},
+            'hunks': [],
+        }
+
+    if not os.path.isfile(current_path):
+        return {
+            'status': 'missing_actual',
+            'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0},
+            'hunks': [],
+        }
+
+    previous_lines = read_log_lines(previous_path)
+    current_lines = read_log_lines(current_path)
+
+    if previous_lines == current_lines:
+        return {
+            'status': 'identical',
+            'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0},
+            'hunks': [],
+        }
+
+    hunks = _compute_hunks(previous_lines, current_lines, context_lines, max_hunks)
+    added = sum(
+        1 for h in hunks for line in h['lines'] if line['kind'] == 'added')
+    removed = sum(
+        1 for h in hunks for line in h['lines'] if line['kind'] == 'removed')
+
+    return {
+        'status': 'different',
+        'summary': {
+            'added_lines': added,
+            'removed_lines': removed,
+            'changed_hunks': len(hunks),
+        },
+        'hunks': hunks,
+    }

mod_api/services/storage.py

@@ -8,6 +8,7 @@
 
 import logging
 import os
+import tempfile
 from datetime import timedelta
 from typing import Optional, Tuple
 
@@ -75,3 +76,49 @@ def get_test_results_base_path() -> str:
     """Return the base directory where TestResults files are stored."""
     from run import config
     return os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TestResults')
+
+
+def combined_stdout_relative_path(run_id: int) -> str:
+    """GCS / SAMPLE_REPOSITORY path of a run's combined VM stdout log."""
+    return f'test_artifacts/{int(run_id)}/combined_stdout.log'
+
+
+def get_combined_stdout_path(run_id: int) -> Optional[str]:
+    """
+    Absolute path to combined_stdout.log for this run, or None.
+
+    Prefers the local SAMPLE_REPOSITORY copy. If that is missing and GCS
+    has the blob, download it to a temp file (caller must delete it).
+    """
+    from run import config, storage_client_bucket
+
+    relative = combined_stdout_relative_path(run_id)
+    sample_repo = config.get('SAMPLE_REPOSITORY', '')
+    if not sample_repo:
+        local_path = ''
+        local_exists = False
+    else:
+        local_path = os.path.join(sample_repo, relative)
+        real_base = os.path.realpath(sample_repo)
+        real_path = os.path.realpath(local_path)
+        if not (real_path.startswith(real_base + os.sep) or real_path == real_base):
+            return None
+        local_exists = os.path.isfile(local_path)
+
+    if local_exists:
+        return local_path
+
+    if not storage_client_bucket:
+        return None
+
+    try:
+        blob = storage_client_bucket.blob(relative)
+        if not blob.exists():
+            return None
+        fd, tmp_path = tempfile.mkstemp(prefix=f'console-{run_id}-', suffix='.log')
+        os.close(fd)
+        blob.download_to_filename(tmp_path)
+        return tmp_path
+    except Exception as e:
+        logger.warning(f"Failed to fetch combined_stdout for run {run_id}: {e}")
+        return None

openapi-ci-api.yaml

@@ -3592,6 +3592,83 @@ paths:
         default:
           $ref: "#/components/responses/Error"
 
+  /runs/{run_id}/console-diff:
+    get:
+      tags: [System]
+      summary: Diagnostic diff of combined stdout vs another run
+      operationId: getConsoleDiff
+      description: >
+        Compare this run's combined_stdout.log to another run's copy.
+        Default `against` is the previous completed run on the same platform.
+        This is a diagnostic view only and does not change CI pass/fail.
+      security:
+        - bearerAuth: []
+      x-required-scope: results:read
+      parameters:
+        - $ref: "#/components/parameters/RunId"
+        - name: against
+          in: query
+          description: Run id to compare against. Defaults to previous completed same-platform run.
+          schema:
+            type: integer
+            minimum: 1
+        - name: format
+          in: query
+          schema:
+            type: string
+            enum: [unified, structured]
+            default: unified
+        - name: context_lines
+          in: query
+          schema:
+            type: integer
+            minimum: 1
+            maximum: 50
+            default: 3
+      responses:
+        "200":
+          description: Diagnostic console log diff
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  run_id:
+                    type: integer
+                  against_run_id:
+                    type: integer
+                  kind:
+                    type: string
+                    enum: [console]
+                  format:
+                    type: string
+                    enum: [unified, structured]
+                  content:
+                    type: string
+                  status:
+                    type: string
+                  summary:
+                    type: object
+                  hunks:
+                    type: array
+                    items:
+                      type: object
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          $ref: "#/components/responses/NotFound"
+        "422":
+          $ref: "#/components/responses/UnprocessableEntity"
+        "429":
+          $ref: "#/components/responses/RateLimited"
+        default:
+          $ref: "#/components/responses/Error"
+
+
 #
 # COMPONENTS
 #