gps-denied-onboard/e2e/runner/helpers/ttff_evaluator.py

"""Cold-start TTFF evaluator for NFT-PERF-03 (AZ-430 / AC-NEW-1).

The SUT promises a Time-To-First-Fix budget of 30 s p95 (and a relaxed
max ceiling of 45 s for tail-latency outlier detection) when started
from cold on Tier-2 (Jetson Orin Nano Super) hardware. AZ-430 collects
N≥``MIN_ITERATION_COUNT`` cold-start TTFF samples; this module owns the
pure-logic side: distribution stats + budget gates + evidence CSV.

Per AZ-430:

* AC-3: ``p95(TTFF) ≤ TTFF_P95_BUDGET_S`` (=30 s).
* AC-4: ``max(TTFF) ≤ TTFF_MAX_BUDGET_S`` (=45 s).

Public-boundary discipline: does NOT import any
``src/gps_denied_onboard`` symbol. Re-uses
``streaming_evaluator._percentile`` for the linear-interpolation p95.
"""

from __future__ import annotations

import csv
from dataclasses import dataclass
from pathlib import Path
from typing import Sequence

from .streaming_evaluator import _percentile

TTFF_P95_BUDGET_S = 30.0
TTFF_MAX_BUDGET_S = 45.0
MIN_ITERATION_COUNT = 10


@dataclass(frozen=True)
class ColdStartIteration:
    """One cold-start iteration outcome.

    ``ttff_s`` is the measured ``t_first_emission − t_first_frame_arrival``
    in seconds. ``None`` means the iteration timed out before producing
    its first emission — categorical miss (treated as budget breach for
    the aggregate verdict).
    """

    iteration_id: str
    first_frame_arrival_ms: int
    first_emission_ms: int | None
    ttff_s: float | None

    @property
    def emitted(self) -> bool:
        return self.first_emission_ms is not None


@dataclass(frozen=True)
class TtffReport:
    """Aggregate NFT-PERF-03 result over N iterations."""

    iterations: tuple[ColdStartIteration, ...]
    p50_s: float | None
    p95_s: float | None
    p99_s: float | None
    max_s: float | None
    missed_starts: int  # iterations where ``ttff_s is None``
    min_iteration_count: int
    p95_budget_s: float
    max_budget_s: float

    @property
    def iteration_count(self) -> int:
        return len(self.iterations)

    @property
    def passes_iteration_count(self) -> bool:
        return self.iteration_count >= self.min_iteration_count

    @property
    def passes_p95(self) -> bool:
        return (
            self.missed_starts == 0
            and self.p95_s is not None
            and self.p95_s <= self.p95_budget_s
        )

    @property
    def passes_max(self) -> bool:
        return (
            self.missed_starts == 0
            and self.max_s is not None
            and self.max_s <= self.max_budget_s
        )

    @property
    def passes(self) -> bool:
        return self.passes_iteration_count and self.passes_p95 and self.passes_max


def measure_iteration(
    iteration_id: str,
    *,
    first_frame_arrival_ms: int,
    first_emission_ms: int | None,
) -> ColdStartIteration:
    """Project a captured iteration into a typed sample.

    Negative TTFF (emission before first frame) is a fixture-shape error
    and raises ``ValueError`` so the breach surfaces immediately instead
    of producing a non-sensible report.
    """
    if first_emission_ms is None:
        return ColdStartIteration(
            iteration_id=iteration_id,
            first_frame_arrival_ms=int(first_frame_arrival_ms),
            first_emission_ms=None,
            ttff_s=None,
        )
    delta_ms = int(first_emission_ms) - int(first_frame_arrival_ms)
    if delta_ms < 0:
        raise ValueError(
            f"ttff iteration {iteration_id}: first_emission_ms "
            f"({first_emission_ms}) precedes first_frame_arrival_ms "
            f"({first_frame_arrival_ms}); fixture shape invalid"
        )
    return ColdStartIteration(
        iteration_id=iteration_id,
        first_frame_arrival_ms=int(first_frame_arrival_ms),
        first_emission_ms=int(first_emission_ms),
        ttff_s=delta_ms / 1000.0,
    )


def evaluate(
    iterations: Sequence[ColdStartIteration],
    *,
    p95_budget_s: float = TTFF_P95_BUDGET_S,
    max_budget_s: float = TTFF_MAX_BUDGET_S,
    min_iteration_count: int = MIN_ITERATION_COUNT,
) -> TtffReport:
    """Aggregate iterations into AC-3 + AC-4 verdicts."""
    valid = [it.ttff_s for it in iterations if it.ttff_s is not None]
    missed = sum(1 for it in iterations if not it.emitted)
    return TtffReport(
        iterations=tuple(iterations),
        p50_s=_percentile(valid, 50.0),
        p95_s=_percentile(valid, 95.0),
        p99_s=_percentile(valid, 99.0),
        max_s=max(valid) if valid else None,
        missed_starts=missed,
        min_iteration_count=min_iteration_count,
        p95_budget_s=p95_budget_s,
        max_budget_s=max_budget_s,
    )


def write_csv_evidence(out_path: Path, report: TtffReport) -> Path:
    """Aggregate-summary CSV (one row per run)."""
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w", newline="") as fh:
        writer = csv.writer(fh)
        writer.writerow(
            [
                "iteration_count",
                "min_iteration_count",
                "missed_starts",
                "p50_s",
                "p95_s",
                "p99_s",
                "max_s",
                "p95_budget_s",
                "max_budget_s",
                "ac1_iteration_count_passes",
                "ac3_p95_passes",
                "ac4_max_passes",
                "passes",
            ]
        )
        writer.writerow(
            [
                report.iteration_count,
                report.min_iteration_count,
                report.missed_starts,
                "" if report.p50_s is None else f"{report.p50_s:.3f}",
                "" if report.p95_s is None else f"{report.p95_s:.3f}",
                "" if report.p99_s is None else f"{report.p99_s:.3f}",
                "" if report.max_s is None else f"{report.max_s:.3f}",
                f"{report.p95_budget_s:.3f}",
                f"{report.max_budget_s:.3f}",
                "true" if report.passes_iteration_count else "false",
                "true" if report.passes_p95 else "false",
                "true" if report.passes_max else "false",
                "true" if report.passes else "false",
            ]
        )
    return out_path


def write_per_iteration_csv(out_path: Path, report: TtffReport) -> Path:
    """One row per iteration — detail used during AC-4 outlier investigation."""
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w", newline="") as fh:
        writer = csv.writer(fh)
        writer.writerow(
            [
                "iteration_id",
                "first_frame_arrival_ms",
                "first_emission_ms",
                "ttff_s",
            ]
        )
        for it in report.iterations:
            writer.writerow(
                [
                    it.iteration_id,
                    it.first_frame_arrival_ms,
                    "" if it.first_emission_ms is None else it.first_emission_ms,
                    "" if it.ttff_s is None else f"{it.ttff_s:.3f}",
                ]
            )
    return out_path