[AZ-585] [AZ-586] ResLim+Perf NFT tests; close test cycle 1

Batch 4 of test implementation cycle 1 (existing-code Step 6, final batch).

- AZ-585 SteadyStateLoadTests + ColdStartRssTests: NFT-RES-LIM-01..04.
  SteadyStateLoadFixture runs one 5-min sustained-load window and samples
  RSS (docker stats), Npgsql conns (pg_stat_activity), and FDs
  (/proc/1/fd) every 5s; three test methods assert independently. All
  SkippableFact-gated on docker primitives.
- AZ-586 PerformanceTests: NFT-PERF-01..04. Sequential single-client,
  5 warm-ups + N measured calls, P50+P95 via LatencyPercentiles, recorded
  to PERF_RESULTS_FILE. Tagged Category=Perf so default gate excludes them.

Infrastructure:
- entrypoint.sh now applies --filter "${TEST_FILTER:-Category!=Perf}"
  per AZ-586 (default CI gate excludes performance).
- MetricCsvRecorder: idempotent CSV appender keyed on env var, used by
  both Perf and ResLim categories.

Step 6 (Implement Tests) is complete. Final report at
_docs/03_implementation/implementation_report_tests.md handoffs the
full-suite gate to test-run/SKILL.md (Step 7).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-15 09:11:53 +03:00
parent 26126e6216
commit 001e80fe96
14 changed files with 1181 additions and 52 deletions
@@ -0,0 +1,206 @@
using System.Diagnostics;
using System.Globalization;
using System.Net.Http.Headers;
using Azaion.Missions.E2E.Helpers;
using Npgsql;
using Xunit;
namespace Azaion.Missions.E2E.Fixtures;
/// <summary>
/// Shared 5-minute steady-state load fixture for NFT-RES-LIM-01 / -02 / -03.
/// Runs the load generator once, samples RSS / connection count / FD count
/// every 5s, and exposes the time series + sentinel "did the SUT exit" flag.
/// </summary>
/// <remarks>
/// <para>The fixture is class-scoped (xUnit <see cref="IClassFixture{TFixture}"/>)
/// and shared across all three NFT-RES-LIM-01/02/03 tests so the 5-minute
/// window runs once per CI invocation.</para>
/// <para>Disabled when <c>COMPOSE_RESTART_ENABLED != 1</c> or docker CLI
/// is missing. Disabled state is observable via <see cref="Enabled"/>; tests
/// must call <see cref="Xunit.Skip.IfNot(bool, string)"/> at the top of the
/// method body — initialising the fixture without docker would throw inside
/// <see cref="InitializeAsync"/> and surface as a hard failure instead of
/// the explicit skip the spec requires.</para>
/// </remarks>
public sealed class SteadyStateLoadFixture : IAsyncLifetime
{
public const int SampleIntervalSeconds = 5;
public const int LoadDurationSeconds = 300;
public const int TargetRps = 50;
public const string ContainerName = "missions-sut";
public bool Enabled =>
Environment.GetEnvironmentVariable("COMPOSE_RESTART_ENABLED") == "1";
public bool LoadGeneratorMetTargetRps { get; private set; }
public bool SutExitedDuringWindow { get; private set; }
public string? SkipReason { get; private set; }
public List<long> RssBytesSamples { get; } = new();
public List<int> NpgsqlConnectionSamples { get; } = new();
public List<int> FileDescriptorSamples { get; } = new();
public List<DateTime> SampleTimestamps { get; } = new();
public async Task InitializeAsync()
{
if (!Enabled)
{
SkipReason = "COMPOSE_RESTART_ENABLED!=1 — docker CLI primitives unavailable";
return;
}
if (!CommandAvailable("docker"))
{
SkipReason = "docker CLI not on PATH in this consumer image";
return;
}
using var http = new HttpClient { BaseAddress = new Uri(TestEnvironment.MissionsBaseUrl) };
var token = await new TokenMinter(TestEnvironment.JwksMockBaseUrl + "/sign").MintDefaultAsync();
http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.Jwt);
var cancel = new CancellationTokenSource();
var endpoints = new[] { "/vehicles", "/missions", "/missions?page=1&pageSize=20" };
long requestsSent = 0;
var loadTask = Task.Run(async () =>
{
var ix = 0;
while (!cancel.IsCancellationRequested)
{
try
{
var url = endpoints[ix++ % endpoints.Length];
using var resp = await http.GetAsync(url, cancel.Token);
Interlocked.Increment(ref requestsSent);
}
catch (HttpRequestException) { /* surfaces via SutExitedDuringWindow below */ }
catch (OperationCanceledException) { return; }
}
}, cancel.Token);
var samplingDeadline = DateTime.UtcNow.AddSeconds(LoadDurationSeconds);
while (DateTime.UtcNow < samplingDeadline)
{
await Task.Delay(TimeSpan.FromSeconds(SampleIntervalSeconds), cancel.Token);
if (!ContainerIsRunning(ContainerName))
{
SutExitedDuringWindow = true;
break;
}
SampleTimestamps.Add(DateTime.UtcNow);
RssBytesSamples.Add(ReadRssBytes(ContainerName));
NpgsqlConnectionSamples.Add(ReadNpgsqlConnectionCount());
FileDescriptorSamples.Add(ReadFileDescriptorCount(ContainerName));
}
cancel.Cancel();
try { await loadTask; } catch (OperationCanceledException) { }
// Sustained 50 RPS over 300s = 15000 requests; allow 10% slack for
// CI variance / connection-refused retries.
LoadGeneratorMetTargetRps =
requestsSent >= (long)(TargetRps * LoadDurationSeconds * 0.9);
}
public Task DisposeAsync() => Task.CompletedTask;
private static long ReadRssBytes(string containerName)
{
// `docker stats --no-stream --format '{{.MemUsage}}'` prints e.g.
// "187.4MiB / 7.7GiB". We need the LHS in bytes.
var raw = Run("docker",
$"stats --no-stream --format '{{{{.MemUsage}}}}' {containerName}");
var lhs = raw.Split('/')[0].Trim().Trim('\'');
return ParseHumanBytes(lhs);
}
private static int ReadNpgsqlConnectionCount()
{
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
conn.Open();
using var cmd = conn.CreateCommand();
cmd.CommandText = """
SELECT count(*)::INTEGER FROM pg_stat_activity
WHERE application_name LIKE 'Npgsql%'
OR (usename = 'postgres' AND backend_type = 'client backend');
""";
return Convert.ToInt32(cmd.ExecuteScalar());
}
private static int ReadFileDescriptorCount(string containerName)
{
// `pgrep` is not guaranteed in the runtime image; we walk /proc
// directly. `/proc/1/comm` is the entrypoint process name; for the
// ASP.NET Core SDK image this is `dotnet`.
var stdout = Run("docker",
$"exec {containerName} sh -c 'ls /proc/1/fd | wc -l'");
return int.Parse(stdout.Trim(), CultureInfo.InvariantCulture);
}
private static long ParseHumanBytes(string text)
{
// "187.4MiB" / "1.2GiB" / "234KiB" / "987B"
var unitIx = text.IndexOfAny(new[] { 'K', 'M', 'G', 'T', 'B' });
if (unitIx < 0) return long.Parse(text, CultureInfo.InvariantCulture);
var num = double.Parse(text.Substring(0, unitIx), CultureInfo.InvariantCulture);
var unit = text.Substring(unitIx);
return unit switch
{
"B" => (long)num,
"KiB" or "KB" or "K" => (long)(num * 1024),
"MiB" or "MB" or "M" => (long)(num * 1024 * 1024),
"GiB" or "GB" or "G" => (long)(num * 1024 * 1024 * 1024),
"TiB" or "TB" or "T" => (long)(num * 1024L * 1024 * 1024 * 1024),
_ => throw new FormatException($"unknown human-bytes unit in '{text}'")
};
}
private static bool ContainerIsRunning(string containerName)
{
try
{
var stdout = Run("docker",
$"inspect --format '{{{{.State.Running}}}}' {containerName}");
return stdout.Trim().Trim('\'').Equals("true", StringComparison.Ordinal);
}
catch (InvalidOperationException) { return false; }
}
private static bool CommandAvailable(string command)
{
try
{
var psi = new ProcessStartInfo(command, "--version")
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
using var p = Process.Start(psi);
if (p is null) return false;
p.WaitForExit();
return p.ExitCode == 0;
}
catch (System.ComponentModel.Win32Exception) { return false; }
}
private static string Run(string file, string args)
{
var psi = new ProcessStartInfo(file, args)
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
using var p = Process.Start(psi)
?? throw new InvalidOperationException($"failed to launch `{file} {args}`");
var stdout = p.StandardOutput.ReadToEnd();
var stderr = p.StandardError.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
throw new InvalidOperationException(
$"`{file} {args}` exited {p.ExitCode}: {stderr}");
return stdout;
}
}
@@ -0,0 +1,38 @@
namespace Azaion.Missions.E2E.Helpers;
/// <summary>
/// Median + percentile helper for the NFT-PERF-* and NFT-RES-LIM-01
/// scenarios. Inputs are wall-clock latency samples (or RSS samples)
/// in any orderable numeric type; the helper sorts a defensive copy
/// and uses the "nearest-rank" definition of percentile (matching the
/// percentile defaults used in `docker stats` and most CI dashboards).
/// </summary>
public static class LatencyPercentiles
{
public static double P50(IReadOnlyList<double> samples) => Percentile(samples, 50);
public static double P95(IReadOnlyList<double> samples) => Percentile(samples, 95);
public static double Percentile(IReadOnlyList<double> samples, int percentile)
{
if (samples.Count == 0)
throw new ArgumentException("samples must contain at least one value", nameof(samples));
if (percentile < 0 || percentile > 100)
throw new ArgumentOutOfRangeException(nameof(percentile), "percentile must be in [0, 100]");
var sorted = samples.ToArray();
Array.Sort(sorted);
// Nearest-rank: rank = ceil(p/100 * N); index = rank - 1.
var rank = (int)Math.Ceiling(percentile / 100.0 * sorted.Length);
if (rank < 1) rank = 1;
if (rank > sorted.Length) rank = sorted.Length;
return sorted[rank - 1];
}
public static double Mean(IReadOnlyList<double> samples)
{
if (samples.Count == 0)
throw new ArgumentException("samples must contain at least one value", nameof(samples));
return samples.Average();
}
}
@@ -0,0 +1,58 @@
using System.Globalization;
namespace Azaion.Missions.E2E.Helpers;
/// <summary>
/// Appends one row per NFT-PERF / NFT-RES-LIM scenario to a side-channel
/// CSV referenced by an environment variable. The Reporting.Cli converter
/// only knows about compile-time <c>[Trait]</c> data — runtime measurements
/// (P50/P95, MAX_FD, P95_RSS_MiB, etc.) need this separate file so
/// deployment planning + trend dashboards can read them.
/// </summary>
/// <remarks>
/// File schema (idempotent header written on first append):
/// <code>Timestamp,Category,Scenario,Result,Traces,ErrorMessage</code>
/// The Traces column carries the dynamic key=value pairs the spec requires
/// (e.g., <c>"AC-3.6; P50_MS=23.4; P95_MS=41.8"</c>); the recorder just
/// joins them with semicolons — callers compose the right shape.
/// </remarks>
public sealed class MetricCsvRecorder
{
private readonly string? _path;
private static readonly object Lock = new();
/// <param name="envVar">name of the env var that carries the target CSV path
/// (e.g., <c>PERF_RESULTS_FILE</c> for NFT-PERF, <c>RESLIM_RESULTS_FILE</c>
/// for NFT-RES-LIM). When the env var is missing or whitespace, every
/// <see cref="Record"/> call is a no-op — the recorder is intentionally
/// silent inside the standard CI run.</param>
public MetricCsvRecorder(string envVar)
{
var v = Environment.GetEnvironmentVariable(envVar);
_path = string.IsNullOrWhiteSpace(v) ? null : v;
}
public bool IsEnabled => _path is not null;
public void Record(string category, string scenario, string result, string traces, string? errorMessage = null)
{
if (_path is null) return;
lock (Lock)
{
var dir = Path.GetDirectoryName(_path);
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
var newFile = !File.Exists(_path);
using var sw = new StreamWriter(_path, append: true);
if (newFile)
sw.WriteLine("Timestamp,Category,Scenario,Result,Traces,ErrorMessage");
sw.WriteLine(
$"{DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)},"
+ $"{Csv(category)},{Csv(scenario)},{Csv(result)},{Csv(traces)},{Csv(errorMessage ?? "")}");
}
}
private static string Csv(string value) =>
value.Contains(',') || value.Contains('"') || value.Contains('\n')
? "\"" + value.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""
: value;
}
@@ -0,0 +1,380 @@
using System.Diagnostics;
using System.Globalization;
using System.Net.Http.Headers;
using Azaion.Missions.E2E.Fixtures;
using Azaion.Missions.E2E.Helpers;
using Npgsql;
using Xunit;
namespace Azaion.Missions.E2E.Tests.Performance;
/// <summary>
/// NFT-PERF-01..04 — wall-clock latency observations against the dockerised
/// <c>missions</c> service. Excluded from the default CI gate via
/// <c>--filter "Category!=Perf"</c> in <c>entrypoint.sh</c>; run via
/// <c>scripts/run-performance-tests.sh</c>.
/// </summary>
/// <remarks>
/// Each scenario follows the same shape: seed deterministic data, warm-up
/// 5 calls (excluded from the percentile), run N measured sequential calls
/// recording <see cref="Stopwatch"/> wall-clock, compute P50 + P95, record
/// them to the runtime CSV referenced by <c>PERF_RESULTS_FILE</c>, then
/// assert against the documented gate. Sequential single-client execution
/// keeps HTTP/1.1 connection-reuse and JIT warm-up deterministic.
/// </remarks>
[Collection("Perf")]
[Trait("Category", "Perf")]
public sealed class PerformanceTests : TestBase, IClassFixture<DbResetFixture>
{
private static readonly MetricCsvRecorder Csv = new("PERF_RESULTS_FILE");
private const int WarmupCalls = 5;
[Fact(Timeout = 60_000)]
[Trait("Traces", "AC-3.6")]
[Trait("max_ms", "30000")]
public async Task NFT_PERF_01_minimal_cascade_delete_p50_within_50ms()
{
// Arrange — 105 missions (100 measured + 5 warmup), each with one waypoint.
DbResetFixture.ResetDatabase(TestEnvironment.DbSideChannel);
Seeds.Apply(Seeds.OneDefaultVehicle.Sql);
var (measured, warmup) = SeedSequentialMissions(105, waypointsPerMission: 1);
await AttachAuthAsync();
await WarmupDeletesAsync(warmup);
// Act
var latenciesMs = await MeasureSequentialDeletesAsync(measured);
var p50 = LatencyPercentiles.P50(latenciesMs);
var p95 = LatencyPercentiles.P95(latenciesMs);
Csv.Record(
category: "Perf",
scenario: "NFT-PERF-01",
result: p50 <= 50.0 ? "pass" : "fail",
traces: $"AC-3.6; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
// Assert
Assert.True(p50 <= 50.0,
$"NFT-PERF-01 P50 budget exceeded: P50={p50:F2}ms (gate=50ms), P95={p95:F2}ms");
}
[Fact(Timeout = 120_000)]
[Trait("Traces", "AC-3.1,AC-3.6")]
[Trait("max_ms", "60000")]
[Trait("provisional", "yes")]
public async Task NFT_PERF_02_full_chain_cascade_delete_p50_within_200ms_provisional()
{
// PROVISIONAL — lock at measured + 50% on first green run.
// Arrange — 55 F3-shaped missions (50 measured + 5 warmup).
DbResetFixture.ResetDatabase(TestEnvironment.DbSideChannel);
Seeds.Apply(Seeds.OneDefaultVehicle.Sql);
StubSchema.EnsureCreated();
var (measured, warmup) = SeedF3MissionsCascadeChains(55);
await AttachAuthAsync();
await WarmupDeletesAsync(warmup);
// Act
var latenciesMs = await MeasureSequentialDeletesAsync(measured);
var p50 = LatencyPercentiles.P50(latenciesMs);
var p95 = LatencyPercentiles.P95(latenciesMs);
Csv.Record(
category: "Perf",
scenario: "NFT-PERF-02",
result: p50 <= 200.0 ? "pass" : "fail",
traces: $"AC-3.1; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
// Assert
Assert.True(p50 <= 200.0,
$"NFT-PERF-02 P50 (provisional 200ms) exceeded: P50={p50:F2}ms, P95={p95:F2}ms");
}
[Fact(Timeout = 30_000)]
[Trait("Traces", "AC-7.3")]
[Trait("max_ms", "5000")]
public async Task NFT_PERF_03_health_p50_within_10ms()
{
// Arrange — no seed needed; /health is anonymous.
for (int i = 0; i < WarmupCalls; i++)
{
using var resp = await Missions.GetAsync("/health");
resp.EnsureSuccessStatusCode();
}
// Act
var latenciesMs = new List<double>(100);
for (int i = 0; i < 100; i++)
{
var sw = Stopwatch.StartNew();
using var resp = await Missions.GetAsync("/health");
sw.Stop();
resp.EnsureSuccessStatusCode();
latenciesMs.Add(sw.Elapsed.TotalMilliseconds);
}
var p50 = LatencyPercentiles.P50(latenciesMs);
var p95 = LatencyPercentiles.P95(latenciesMs);
Csv.Record(
category: "Perf",
scenario: "NFT-PERF-03",
result: p50 <= 10.0 ? "pass" : "fail",
traces: $"AC-7.3; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
// Assert
Assert.True(p50 <= 10.0,
$"NFT-PERF-03 P50 budget exceeded: P50={p50:F2}ms (gate=10ms), P95={p95:F2}ms");
}
[Fact(Timeout = 90_000)]
[Trait("Traces", "AC-2.3")]
[Trait("max_ms", "30000")]
[Trait("provisional", "yes")]
public async Task NFT_PERF_04_missions_list_pagination_p95_within_100ms_provisional()
{
// PROVISIONAL — lock at measured + 50% on first green run.
// Arrange — 1000 missions referencing seed_one_default_vehicle.
DbResetFixture.ResetDatabase(TestEnvironment.DbSideChannel);
Seeds.Apply(Seeds.OneDefaultVehicle.Sql);
SeedSequentialMissionsNoWaypoints(1000);
await AttachAuthAsync();
for (int i = 0; i < WarmupCalls; i++)
{
using var resp = await Missions.GetAsync("/missions?page=1&pageSize=20");
resp.EnsureSuccessStatusCode();
}
// Act
var latenciesMs = new List<double>(100);
for (int i = 0; i < 100; i++)
{
var sw = Stopwatch.StartNew();
using var resp = await Missions.GetAsync("/missions?page=1&pageSize=20");
sw.Stop();
resp.EnsureSuccessStatusCode();
latenciesMs.Add(sw.Elapsed.TotalMilliseconds);
}
var p50 = LatencyPercentiles.P50(latenciesMs);
var p95 = LatencyPercentiles.P95(latenciesMs);
Csv.Record(
category: "Perf",
scenario: "NFT-PERF-04",
result: p95 <= 100.0 ? "pass" : "fail",
traces: $"AC-2.3; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
// Assert
Assert.True(p95 <= 100.0,
$"NFT-PERF-04 P95 (provisional 100ms) exceeded: P50={p50:F2}ms, P95={p95:F2}ms");
}
private async Task AttachAuthAsync()
{
var t = await Tokens.MintDefaultAsync();
Missions.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue("Bearer", t.Jwt);
}
private async Task WarmupDeletesAsync(IReadOnlyList<Guid> warmupMissionIds)
{
foreach (var id in warmupMissionIds)
{
using var resp = await Missions.DeleteAsync($"/missions/{id}");
// 200 or 204 are both acceptable; the cascade walks regardless.
// 4xx would indicate a seed problem — fail loudly.
if (!resp.IsSuccessStatusCode && (int)resp.StatusCode != 404)
throw new InvalidOperationException(
$"warmup DELETE /missions/{id} returned {(int)resp.StatusCode}");
}
}
private async Task<List<double>> MeasureSequentialDeletesAsync(IReadOnlyList<Guid> missionIds)
{
var latencies = new List<double>(missionIds.Count);
foreach (var id in missionIds)
{
var sw = Stopwatch.StartNew();
using var resp = await Missions.DeleteAsync($"/missions/{id}");
sw.Stop();
if (!resp.IsSuccessStatusCode && (int)resp.StatusCode != 404)
throw new InvalidOperationException(
$"measured DELETE /missions/{id} returned {(int)resp.StatusCode}");
latencies.Add(sw.Elapsed.TotalMilliseconds);
}
return latencies;
}
/// <summary>
/// Returns (measured, warmup) where the FIRST 5 IDs are the warmup set
/// and the remaining (count-5) IDs are the measured set. Each mission
/// gets the requested number of waypoints with deterministic IDs.
/// </summary>
private static (List<Guid> Measured, List<Guid> Warmup) SeedSequentialMissions(
int count, int waypointsPerMission)
{
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
conn.Open();
using var tx = conn.BeginTransaction();
var ids = new List<Guid>(count);
var seed = new Random(98765);
using (var insertMission = conn.CreateCommand())
{
insertMission.Transaction = tx;
insertMission.CommandText = """
INSERT INTO missions (id, name, vehicle_id)
VALUES (@id, @name, @vehicle_id);
""";
insertMission.Parameters.Add(new NpgsqlParameter("id", NpgsqlTypes.NpgsqlDbType.Uuid));
insertMission.Parameters.Add(new NpgsqlParameter("name", NpgsqlTypes.NpgsqlDbType.Text));
insertMission.Parameters.Add(new NpgsqlParameter("vehicle_id", NpgsqlTypes.NpgsqlDbType.Uuid));
using var insertWaypoint = conn.CreateCommand();
insertWaypoint.Transaction = tx;
insertWaypoint.CommandText = """
INSERT INTO waypoints (id, mission_id, lat, lon, mgrs, order_num)
VALUES (@id, @mission_id, 50.45, 30.52, '36UYA1234567', @order_num);
""";
insertWaypoint.Parameters.Add(new NpgsqlParameter("id", NpgsqlTypes.NpgsqlDbType.Uuid));
insertWaypoint.Parameters.Add(new NpgsqlParameter("mission_id", NpgsqlTypes.NpgsqlDbType.Uuid));
insertWaypoint.Parameters.Add(new NpgsqlParameter("order_num", NpgsqlTypes.NpgsqlDbType.Integer));
for (int i = 0; i < count; i++)
{
var id = NewDeterministicGuid(seed);
ids.Add(id);
insertMission.Parameters["id"].Value = id;
insertMission.Parameters["name"].Value = $"perf-mission-{i:D4}";
insertMission.Parameters["vehicle_id"].Value = Seeds.OneDefaultVehicle.Id;
insertMission.ExecuteNonQuery();
for (int w = 0; w < waypointsPerMission; w++)
{
insertWaypoint.Parameters["id"].Value = NewDeterministicGuid(seed);
insertWaypoint.Parameters["mission_id"].Value = id;
insertWaypoint.Parameters["order_num"].Value = w;
insertWaypoint.ExecuteNonQuery();
}
}
}
tx.Commit();
var warmup = ids.Take(WarmupCalls).ToList();
var measured = ids.Skip(WarmupCalls).ToList();
return (measured, warmup);
}
private static void SeedSequentialMissionsNoWaypoints(int count)
{
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
conn.Open();
using var tx = conn.BeginTransaction();
using var cmd = conn.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = """
INSERT INTO missions (id, name, vehicle_id)
VALUES (@id, @name, @vehicle_id);
""";
cmd.Parameters.Add(new NpgsqlParameter("id", NpgsqlTypes.NpgsqlDbType.Uuid));
cmd.Parameters.Add(new NpgsqlParameter("name", NpgsqlTypes.NpgsqlDbType.Text));
cmd.Parameters.Add(new NpgsqlParameter("vehicle_id", NpgsqlTypes.NpgsqlDbType.Uuid));
var seed = new Random(13579);
for (int i = 0; i < count; i++)
{
cmd.Parameters["id"].Value = NewDeterministicGuid(seed);
cmd.Parameters["name"].Value = $"list-perf-{i:D4}";
cmd.Parameters["vehicle_id"].Value = Seeds.OneDefaultVehicle.Id;
cmd.ExecuteNonQuery();
}
tx.Commit();
}
/// <summary>
/// Seeds <paramref name="count"/> missions, each with the F3 cascade shape:
/// 3 map_objects + 2 waypoints + (per waypoint: 2 media → 2 annotations → 2 detection).
/// </summary>
private static (List<Guid> Measured, List<Guid> Warmup) SeedF3MissionsCascadeChains(int count)
{
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
conn.Open();
var ids = new List<Guid>(count);
var seed = new Random(24680);
for (int i = 0; i < count; i++)
{
using var tx = conn.BeginTransaction();
var missionId = NewDeterministicGuid(seed);
ids.Add(missionId);
ExecScalar(conn, tx, """
INSERT INTO missions (id, name, vehicle_id) VALUES (@id, @name, @vid);
""", ("id", missionId), ("name", $"f3-perf-{i:D4}"),
("vid", Seeds.OneDefaultVehicle.Id));
for (int m = 0; m < 3; m++)
ExecScalar(conn, tx, """
INSERT INTO map_objects (id, mission_id, h3_index, mgrs)
VALUES (@id, @mid, '8a2a1072b59ffff', '36UYA1234567');
""", ("id", NewDeterministicGuid(seed)), ("mid", missionId));
for (int w = 0; w < 2; w++)
{
var wpId = NewDeterministicGuid(seed);
ExecScalar(conn, tx, """
INSERT INTO waypoints (id, mission_id, lat, lon, mgrs, order_num)
VALUES (@id, @mid, 50.45, 30.52, '36UYA1234567', @ord);
""", ("id", wpId), ("mid", missionId), ("ord", w));
for (int md = 0; md < 2; md++)
{
var mediaId = $"media-{Guid.NewGuid():N}";
ExecScalar(conn, tx, """
INSERT INTO media (id, waypoint_id) VALUES (@id, @wid);
""", ("id", mediaId), ("wid", wpId));
var annId = $"ann-{Guid.NewGuid():N}";
ExecScalar(conn, tx, """
INSERT INTO annotations (id, media_id) VALUES (@id, @mid);
""", ("id", annId), ("mid", mediaId));
ExecScalar(conn, tx, """
INSERT INTO detection (id, annotation_id) VALUES (@id, @aid);
""", ("id", NewDeterministicGuid(seed)), ("aid", annId));
}
}
tx.Commit();
}
var warmup = ids.Take(WarmupCalls).ToList();
var measured = ids.Skip(WarmupCalls).ToList();
return (measured, warmup);
}
private static void ExecScalar(NpgsqlConnection conn, NpgsqlTransaction tx, string sql,
params (string Name, object Value)[] args)
{
using var cmd = conn.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = sql;
foreach (var (name, value) in args)
cmd.Parameters.AddWithValue(name, value);
cmd.ExecuteNonQuery();
}
private static Guid NewDeterministicGuid(Random rng)
{
var bytes = new byte[16];
rng.NextBytes(bytes);
// Force version 4 + variant 1 so the value is a valid UUID Postgres accepts.
bytes[7] = (byte)((bytes[7] & 0x0F) | 0x40);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes);
}
}
@@ -1,23 +0,0 @@
using Xunit;
namespace Azaion.Missions.E2E.Tests.Performance;
/// <summary>
/// Discovery-only smoke test for the Performance category. Real Performance
/// scenarios (NFT-PERF-01..04) land in AZ-586.
/// </summary>
public sealed class Sanity
{
[Fact]
[Trait("Category", "Perf")]
[Trait("Traces", "AC-3")]
public void Discovery_smoke_test_runs()
{
// Arrange
const int sentinel = 1;
// Act
var result = sentinel + 0;
// Assert
Assert.Equal(1, result);
}
}
@@ -0,0 +1,131 @@
using System.Diagnostics;
using System.Globalization;
using System.Net;
using Azaion.Missions.E2E.Helpers;
using Xunit;
namespace Azaion.Missions.E2E.Tests.ResourceLimits;
/// <summary>
/// NFT-RES-LIM-04 — cold-start RSS. Driven independently from the
/// steady-state window because it requires a fresh container start; lives
/// in the <c>MigratorRestart</c> collection so it serialises with the
/// other docker-compose-restarting tests rather than racing them.
/// </summary>
/// <remarks>
/// The 30-second wait between health-OK and the measurement is the spec's
/// way of letting the JIT and the JWKS prefetch settle without doing any
/// real work — measuring at health-OK alone would conflate the genuine cold
/// baseline with bootstrap noise.
/// </remarks>
[Collection("MigratorRestart")]
[Trait("Category", "ResLim")]
public sealed class ColdStartRssTests
{
private static readonly MetricCsvRecorder Csv = new("RESLIM_RESULTS_FILE");
private const long ProvisionalColdRssCapMiB = 200;
private const string ComposeFile = "/workspace/docker-compose.test.yml";
[SkippableFact]
[Trait("Traces", "H1|H3")]
[Trait("max_ms", "120000")]
public async Task NFT_RES_LIM_04_cold_start_rss_within_provisional_200_MiB()
{
Skip.IfNot(Environment.GetEnvironmentVariable("COMPOSE_RESTART_ENABLED") == "1",
"COMPOSE_RESTART_ENABLED!=1 — docker compose restart unavailable in this consumer image");
Skip.IfNot(MissionsContainerHelper.Enabled,
"MissionsContainerHelper disabled — docker CLI unavailable");
// Arrange — bring missions down hard and start it fresh. The
// surrounding "MigratorRestart" collection serialises us against
// any other test that touches the SUT.
DockerCompose("stop missions");
DockerCompose("rm -f missions");
DockerCompose("up -d missions");
await WaitForHealthOkAsync(TimeSpan.FromSeconds(60));
// Act — wait 30s after health-OK so JIT/JWKS settle, then measure.
await Task.Delay(TimeSpan.FromSeconds(30));
var rssBytes = ReadRssBytes("missions-sut");
var rssMiB = rssBytes / (double)(1024 * 1024);
var pass = rssMiB <= ProvisionalColdRssCapMiB;
Csv.Record(
category: "ResLim",
scenario: "NFT-RES-LIM-04",
result: pass ? "pass" : "fail",
traces: $"H1|H3; COLD_RSS_MiB={rssMiB.ToString("F1", CultureInfo.InvariantCulture)}");
// Assert — provisional gate.
Assert.True(pass,
$"cold-start RSS {rssMiB:F1} MiB exceeds provisional {ProvisionalColdRssCapMiB} MiB gate");
}
private static async Task WaitForHealthOkAsync(TimeSpan timeout)
{
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(2) };
var deadline = DateTime.UtcNow + timeout;
var healthUrl = new Uri(TestEnvironment.MissionsBaseUrl + "/health");
while (DateTime.UtcNow < deadline)
{
try
{
using var resp = await http.GetAsync(healthUrl);
if (resp.StatusCode == HttpStatusCode.OK) return;
}
catch (HttpRequestException) { /* not yet listening */ }
catch (TaskCanceledException) { /* slow first response */ }
await Task.Delay(500);
}
throw new TimeoutException(
$"missions did not become healthy within {timeout.TotalSeconds:F0}s of cold start");
}
private static long ReadRssBytes(string containerName)
{
var raw = Run("docker",
$"stats --no-stream --format '{{{{.MemUsage}}}}' {containerName}");
var lhs = raw.Split('/')[0].Trim().Trim('\'');
return ParseHumanBytes(lhs);
}
private static long ParseHumanBytes(string text)
{
var unitIx = text.IndexOfAny(new[] { 'K', 'M', 'G', 'T', 'B' });
if (unitIx < 0) return long.Parse(text, CultureInfo.InvariantCulture);
var num = double.Parse(text.Substring(0, unitIx), CultureInfo.InvariantCulture);
var unit = text.Substring(unitIx);
return unit switch
{
"B" => (long)num,
"KiB" or "KB" or "K" => (long)(num * 1024),
"MiB" or "MB" or "M" => (long)(num * 1024 * 1024),
"GiB" or "GB" or "G" => (long)(num * 1024 * 1024 * 1024),
"TiB" or "TB" or "T" => (long)(num * 1024L * 1024 * 1024 * 1024),
_ => throw new FormatException($"unknown human-bytes unit in '{text}'")
};
}
private static void DockerCompose(string subcommand) =>
Run("docker", $"compose -f {ComposeFile} {subcommand}");
private static string Run(string file, string args)
{
var psi = new ProcessStartInfo(file, args)
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false
};
using var p = Process.Start(psi)
?? throw new InvalidOperationException($"failed to launch `{file} {args}`");
var stdout = p.StandardOutput.ReadToEnd();
var stderr = p.StandardError.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
throw new InvalidOperationException(
$"`{file} {args}` exited {p.ExitCode}: {stderr}");
return stdout;
}
}
@@ -1,23 +0,0 @@
using Xunit;
namespace Azaion.Missions.E2E.Tests.ResourceLimits;
/// <summary>
/// Discovery-only smoke test for the ResourceLimits category. Real
/// ResourceLimits scenarios (NFT-RES-LIM-01..04) land in AZ-585.
/// </summary>
public sealed class Sanity
{
[Fact]
[Trait("Category", "ResLim")]
[Trait("Traces", "AC-3")]
public void Discovery_smoke_test_runs()
{
// Arrange
const int sentinel = 1;
// Act
var result = sentinel + 0;
// Assert
Assert.Equal(1, result);
}
}
@@ -0,0 +1,156 @@
using System.Globalization;
using Azaion.Missions.E2E.Fixtures;
using Azaion.Missions.E2E.Helpers;
using Xunit;
namespace Azaion.Missions.E2E.Tests.ResourceLimits;
/// <summary>
/// NFT-RES-LIM-01..03 — three observations on a SINGLE 5-minute sustained
/// load window. The window itself lives in
/// <see cref="SteadyStateLoadFixture"/> (class-scoped, runs once); each
/// test asserts one metric against its provisional gate.
/// </summary>
/// <remarks>
/// The fixture skips itself when docker primitives are unavailable; the
/// tests detect that via <see cref="SteadyStateLoadFixture.SkipReason"/>
/// and surface the same reason through <c>Skip.IfNot</c>. The fixture
/// also flips <see cref="SteadyStateLoadFixture.SutExitedDuringWindow"/>
/// if the SUT crashes mid-window — every test fails fast with a clear
/// message rather than reporting a misleading metric.
/// </remarks>
[Collection("ResLimSteadyState")]
[Trait("Category", "ResLim")]
public sealed class SteadyStateLoadTests : TestBase, IClassFixture<SteadyStateLoadFixture>
{
private static readonly MetricCsvRecorder Csv = new("RESLIM_RESULTS_FILE");
private const long ProvisionalRssCapMiB = 250;
private const int ProvisionalConnectionCap = 100;
private const int ProvisionalFdCap = 1024;
private readonly SteadyStateLoadFixture _load;
public SteadyStateLoadTests(SteadyStateLoadFixture load) => _load = load;
[SkippableFact]
[Trait("Traces", "H1|H6|O10")]
[Trait("max_ms", "360000")]
public void NFT_RES_LIM_01_steady_state_rss_within_provisional_gate_and_no_leak()
{
Skip.If(_load.SkipReason is not null, _load.SkipReason);
Skip.IfNot(_load.LoadGeneratorMetTargetRps,
"runner cannot sustain target load (NFR Reliability — not a SUT defect)");
Assert.False(_load.SutExitedDuringWindow, "SUT exited during measurement window");
// Arrange
var samplesMiB = _load.RssBytesSamples.Select(b => b / (double)(1024 * 1024)).ToList();
Assert.True(samplesMiB.Count >= 30,
$"expected ≥ 30 RSS samples over 5-min window, got {samplesMiB.Count}");
// Act
var p95 = LatencyPercentiles.P95(samplesMiB);
var finalMiB = samplesMiB[^1];
var leakRatio = Math.Abs(finalMiB - p95) / Math.Max(p95, 1.0);
var withinCap = p95 <= ProvisionalRssCapMiB;
var noLeak = leakRatio <= 0.20;
var pass = withinCap && noLeak;
Csv.Record(
category: "ResLim",
scenario: "NFT-RES-LIM-01",
result: pass ? "pass" : "fail",
traces: $"H1|H6|O10; "
+ $"P95_RSS_MiB={p95.ToString("F1", CultureInfo.InvariantCulture)}; "
+ $"FINAL_RSS_MiB={finalMiB.ToString("F1", CultureInfo.InvariantCulture)}; "
+ $"LEAK_RATIO={leakRatio.ToString("F2", CultureInfo.InvariantCulture)}");
// Assert — provisional gate; lock at measured + 50% after first green run.
Assert.True(withinCap,
$"P95 RSS {p95:F1} MiB exceeds provisional {ProvisionalRssCapMiB} MiB gate");
Assert.True(noLeak,
$"final RSS {finalMiB:F1} MiB diverges {leakRatio:P0} from P95 {p95:F1} MiB (gate 20%)");
}
[SkippableFact]
[Trait("Traces", "O10")]
[Trait("max_ms", "360000")]
public void NFT_RES_LIM_02_npgsql_connection_pool_within_100_no_unbounded_growth()
{
Skip.If(_load.SkipReason is not null, _load.SkipReason);
Skip.IfNot(_load.LoadGeneratorMetTargetRps,
"runner cannot sustain target load (NFR Reliability — not a SUT defect)");
Assert.False(_load.SutExitedDuringWindow, "SUT exited during measurement window");
var samples = _load.NpgsqlConnectionSamples;
Assert.True(samples.Count >= 30,
$"expected ≥ 30 connection samples over 5-min window, got {samples.Count}");
// Act
var max = samples.Max();
var firstMinuteSampleCount = 60 / SteadyStateLoadFixture.SampleIntervalSeconds;
var firstMinute = samples.Take(firstMinuteSampleCount).ToList();
var firstMinuteMean = firstMinute.Average();
var finalCount = samples[^1];
var withinCap = max <= ProvisionalConnectionCap;
var noUnboundedGrowth = finalCount <= 1.3 * Math.Max(firstMinuteMean, 1.0);
var pass = withinCap && noUnboundedGrowth;
Csv.Record(
category: "ResLim",
scenario: "NFT-RES-LIM-02",
result: pass ? "pass" : "fail",
traces: $"O10; MAX_NPGSQL_CONNS={max}; "
+ $"FINAL_CONNS={finalCount}; "
+ $"MINUTE1_MEAN={firstMinuteMean.ToString("F1", CultureInfo.InvariantCulture)}");
// Assert
Assert.True(withinCap,
$"max Npgsql connections {max} exceeds provisional cap {ProvisionalConnectionCap}");
Assert.True(noUnboundedGrowth,
$"final connection count {finalCount} > 1.3 × first-minute mean {firstMinuteMean:F1}");
}
[SkippableFact]
[Trait("Traces", "H6|O10")]
[Trait("max_ms", "360000")]
public void NFT_RES_LIM_03_file_descriptors_within_1024_no_leak()
{
Skip.If(_load.SkipReason is not null, _load.SkipReason);
Skip.IfNot(_load.LoadGeneratorMetTargetRps,
"runner cannot sustain target load (NFR Reliability — not a SUT defect)");
Assert.False(_load.SutExitedDuringWindow, "SUT exited during measurement window");
var samples = _load.FileDescriptorSamples;
Assert.True(samples.Count >= 30,
$"expected ≥ 30 FD samples over 5-min window, got {samples.Count}");
// Act
var max = samples.Max();
var minuteOneSampleCount = 60 / SteadyStateLoadFixture.SampleIntervalSeconds;
// The spec calls out "count at t=1min" — anchor on the sample whose
// timestamp is closest to (start + 60s).
var minuteOneIx = Math.Min(minuteOneSampleCount - 1, samples.Count - 1);
var minuteOneCount = samples[minuteOneIx];
var finalCount = samples[^1];
var withinCap = max <= ProvisionalFdCap;
var noLeak = finalCount <= 1.3 * Math.Max(minuteOneCount, 1);
var pass = withinCap && noLeak;
Csv.Record(
category: "ResLim",
scenario: "NFT-RES-LIM-03",
result: pass ? "pass" : "fail",
traces: $"H6|O10; MAX_FD={max}; "
+ $"FINAL_FD={finalCount}; MINUTE1_FD={minuteOneCount}");
// Assert
Assert.True(withinCap,
$"max FD count {max} exceeds provisional cap {ProvisionalFdCap}");
Assert.True(noLeak,
$"final FD count {finalCount} > 1.3 × minute-1 count {minuteOneCount}");
}
}
@@ -14,9 +14,16 @@ set -eu
mkdir -p "$RESULTS_DIR"
set +e
## Performance scenarios (Category=Perf) are excluded from the default gate
## per AZ-586. They are invoked from scripts/run-performance-tests.sh which
## passes its own --filter Category=Perf. ResLim tests (Category=ResLim) stay
## in the default gate because their docker-CLI gate causes them to skip
## with an explicit reason when COMPOSE_RESTART_ENABLED is not set.
TEST_FILTER="${TEST_FILTER:-Category!=Perf}"
dotnet test /src/Azaion.Missions.E2E.Tests.csproj \
--no-build \
--configuration Release \
--filter "$TEST_FILTER" \
--logger "trx;LogFileName=results.trx" \
--logger "console;verbosity=normal" \
--results-directory "$RESULTS_DIR"