mirror of
https://github.com/azaion/missions.git
synced 2026-06-21 09:51:07 +00:00
[AZ-585] [AZ-586] ResLim+Perf NFT tests; close test cycle 1
Batch 4 of test implementation cycle 1 (existing-code Step 6, final batch).
- AZ-585 SteadyStateLoadTests + ColdStartRssTests: NFT-RES-LIM-01..04.
SteadyStateLoadFixture runs one 5-min sustained-load window and samples
RSS (docker stats), Npgsql conns (pg_stat_activity), and FDs
(/proc/1/fd) every 5s; three test methods assert independently. All
SkippableFact-gated on docker primitives.
- AZ-586 PerformanceTests: NFT-PERF-01..04. Sequential single-client,
5 warm-ups + N measured calls, P50+P95 via LatencyPercentiles, recorded
to PERF_RESULTS_FILE. Tagged Category=Perf so default gate excludes them.
Infrastructure:
- entrypoint.sh now applies --filter "${TEST_FILTER:-Category!=Perf}"
per AZ-586 (default CI gate excludes performance).
- MetricCsvRecorder: idempotent CSV appender keyed on env var, used by
both Perf and ResLim categories.
Step 6 (Implement Tests) is complete. Final report at
_docs/03_implementation/implementation_report_tests.md handoffs the
full-suite gate to test-run/SKILL.md (Step 7).
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Net.Http.Headers;
|
||||
using Azaion.Missions.E2E.Helpers;
|
||||
using Npgsql;
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Shared 5-minute steady-state load fixture for NFT-RES-LIM-01 / -02 / -03.
|
||||
/// Runs the load generator once, samples RSS / connection count / FD count
|
||||
/// every 5s, and exposes the time series + sentinel "did the SUT exit" flag.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>The fixture is class-scoped (xUnit <see cref="IClassFixture{TFixture}"/>)
|
||||
/// and shared across all three NFT-RES-LIM-01/02/03 tests so the 5-minute
|
||||
/// window runs once per CI invocation.</para>
|
||||
/// <para>Disabled when <c>COMPOSE_RESTART_ENABLED != 1</c> or docker CLI
|
||||
/// is missing. Disabled state is observable via <see cref="Enabled"/>; tests
|
||||
/// must call <see cref="Xunit.Skip.IfNot(bool, string)"/> at the top of the
|
||||
/// method body — initialising the fixture without docker would throw inside
|
||||
/// <see cref="InitializeAsync"/> and surface as a hard failure instead of
|
||||
/// the explicit skip the spec requires.</para>
|
||||
/// </remarks>
|
||||
public sealed class SteadyStateLoadFixture : IAsyncLifetime
|
||||
{
|
||||
public const int SampleIntervalSeconds = 5;
|
||||
public const int LoadDurationSeconds = 300;
|
||||
public const int TargetRps = 50;
|
||||
public const string ContainerName = "missions-sut";
|
||||
|
||||
public bool Enabled =>
|
||||
Environment.GetEnvironmentVariable("COMPOSE_RESTART_ENABLED") == "1";
|
||||
|
||||
public bool LoadGeneratorMetTargetRps { get; private set; }
|
||||
public bool SutExitedDuringWindow { get; private set; }
|
||||
public string? SkipReason { get; private set; }
|
||||
|
||||
public List<long> RssBytesSamples { get; } = new();
|
||||
public List<int> NpgsqlConnectionSamples { get; } = new();
|
||||
public List<int> FileDescriptorSamples { get; } = new();
|
||||
public List<DateTime> SampleTimestamps { get; } = new();
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
if (!Enabled)
|
||||
{
|
||||
SkipReason = "COMPOSE_RESTART_ENABLED!=1 — docker CLI primitives unavailable";
|
||||
return;
|
||||
}
|
||||
if (!CommandAvailable("docker"))
|
||||
{
|
||||
SkipReason = "docker CLI not on PATH in this consumer image";
|
||||
return;
|
||||
}
|
||||
|
||||
using var http = new HttpClient { BaseAddress = new Uri(TestEnvironment.MissionsBaseUrl) };
|
||||
var token = await new TokenMinter(TestEnvironment.JwksMockBaseUrl + "/sign").MintDefaultAsync();
|
||||
http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.Jwt);
|
||||
|
||||
var cancel = new CancellationTokenSource();
|
||||
var endpoints = new[] { "/vehicles", "/missions", "/missions?page=1&pageSize=20" };
|
||||
long requestsSent = 0;
|
||||
|
||||
var loadTask = Task.Run(async () =>
|
||||
{
|
||||
var ix = 0;
|
||||
while (!cancel.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = endpoints[ix++ % endpoints.Length];
|
||||
using var resp = await http.GetAsync(url, cancel.Token);
|
||||
Interlocked.Increment(ref requestsSent);
|
||||
}
|
||||
catch (HttpRequestException) { /* surfaces via SutExitedDuringWindow below */ }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}, cancel.Token);
|
||||
|
||||
var samplingDeadline = DateTime.UtcNow.AddSeconds(LoadDurationSeconds);
|
||||
while (DateTime.UtcNow < samplingDeadline)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(SampleIntervalSeconds), cancel.Token);
|
||||
if (!ContainerIsRunning(ContainerName))
|
||||
{
|
||||
SutExitedDuringWindow = true;
|
||||
break;
|
||||
}
|
||||
SampleTimestamps.Add(DateTime.UtcNow);
|
||||
RssBytesSamples.Add(ReadRssBytes(ContainerName));
|
||||
NpgsqlConnectionSamples.Add(ReadNpgsqlConnectionCount());
|
||||
FileDescriptorSamples.Add(ReadFileDescriptorCount(ContainerName));
|
||||
}
|
||||
|
||||
cancel.Cancel();
|
||||
try { await loadTask; } catch (OperationCanceledException) { }
|
||||
|
||||
// Sustained 50 RPS over 300s = 15000 requests; allow 10% slack for
|
||||
// CI variance / connection-refused retries.
|
||||
LoadGeneratorMetTargetRps =
|
||||
requestsSent >= (long)(TargetRps * LoadDurationSeconds * 0.9);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
private static long ReadRssBytes(string containerName)
|
||||
{
|
||||
// `docker stats --no-stream --format '{{.MemUsage}}'` prints e.g.
|
||||
// "187.4MiB / 7.7GiB". We need the LHS in bytes.
|
||||
var raw = Run("docker",
|
||||
$"stats --no-stream --format '{{{{.MemUsage}}}}' {containerName}");
|
||||
var lhs = raw.Split('/')[0].Trim().Trim('\'');
|
||||
return ParseHumanBytes(lhs);
|
||||
}
|
||||
|
||||
private static int ReadNpgsqlConnectionCount()
|
||||
{
|
||||
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
|
||||
conn.Open();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
SELECT count(*)::INTEGER FROM pg_stat_activity
|
||||
WHERE application_name LIKE 'Npgsql%'
|
||||
OR (usename = 'postgres' AND backend_type = 'client backend');
|
||||
""";
|
||||
return Convert.ToInt32(cmd.ExecuteScalar());
|
||||
}
|
||||
|
||||
private static int ReadFileDescriptorCount(string containerName)
|
||||
{
|
||||
// `pgrep` is not guaranteed in the runtime image; we walk /proc
|
||||
// directly. `/proc/1/comm` is the entrypoint process name; for the
|
||||
// ASP.NET Core SDK image this is `dotnet`.
|
||||
var stdout = Run("docker",
|
||||
$"exec {containerName} sh -c 'ls /proc/1/fd | wc -l'");
|
||||
return int.Parse(stdout.Trim(), CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static long ParseHumanBytes(string text)
|
||||
{
|
||||
// "187.4MiB" / "1.2GiB" / "234KiB" / "987B"
|
||||
var unitIx = text.IndexOfAny(new[] { 'K', 'M', 'G', 'T', 'B' });
|
||||
if (unitIx < 0) return long.Parse(text, CultureInfo.InvariantCulture);
|
||||
var num = double.Parse(text.Substring(0, unitIx), CultureInfo.InvariantCulture);
|
||||
var unit = text.Substring(unitIx);
|
||||
return unit switch
|
||||
{
|
||||
"B" => (long)num,
|
||||
"KiB" or "KB" or "K" => (long)(num * 1024),
|
||||
"MiB" or "MB" or "M" => (long)(num * 1024 * 1024),
|
||||
"GiB" or "GB" or "G" => (long)(num * 1024 * 1024 * 1024),
|
||||
"TiB" or "TB" or "T" => (long)(num * 1024L * 1024 * 1024 * 1024),
|
||||
_ => throw new FormatException($"unknown human-bytes unit in '{text}'")
|
||||
};
|
||||
}
|
||||
|
||||
private static bool ContainerIsRunning(string containerName)
|
||||
{
|
||||
try
|
||||
{
|
||||
var stdout = Run("docker",
|
||||
$"inspect --format '{{{{.State.Running}}}}' {containerName}");
|
||||
return stdout.Trim().Trim('\'').Equals("true", StringComparison.Ordinal);
|
||||
}
|
||||
catch (InvalidOperationException) { return false; }
|
||||
}
|
||||
|
||||
private static bool CommandAvailable(string command)
|
||||
{
|
||||
try
|
||||
{
|
||||
var psi = new ProcessStartInfo(command, "--version")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
using var p = Process.Start(psi);
|
||||
if (p is null) return false;
|
||||
p.WaitForExit();
|
||||
return p.ExitCode == 0;
|
||||
}
|
||||
catch (System.ComponentModel.Win32Exception) { return false; }
|
||||
}
|
||||
|
||||
private static string Run(string file, string args)
|
||||
{
|
||||
var psi = new ProcessStartInfo(file, args)
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
using var p = Process.Start(psi)
|
||||
?? throw new InvalidOperationException($"failed to launch `{file} {args}`");
|
||||
var stdout = p.StandardOutput.ReadToEnd();
|
||||
var stderr = p.StandardError.ReadToEnd();
|
||||
p.WaitForExit();
|
||||
if (p.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"`{file} {args}` exited {p.ExitCode}: {stderr}");
|
||||
return stdout;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace Azaion.Missions.E2E.Helpers;
|
||||
|
||||
/// <summary>
|
||||
/// Median + percentile helper for the NFT-PERF-* and NFT-RES-LIM-01
|
||||
/// scenarios. Inputs are wall-clock latency samples (or RSS samples)
|
||||
/// in any orderable numeric type; the helper sorts a defensive copy
|
||||
/// and uses the "nearest-rank" definition of percentile (matching the
|
||||
/// percentile defaults used in `docker stats` and most CI dashboards).
|
||||
/// </summary>
|
||||
public static class LatencyPercentiles
|
||||
{
|
||||
public static double P50(IReadOnlyList<double> samples) => Percentile(samples, 50);
|
||||
public static double P95(IReadOnlyList<double> samples) => Percentile(samples, 95);
|
||||
|
||||
public static double Percentile(IReadOnlyList<double> samples, int percentile)
|
||||
{
|
||||
if (samples.Count == 0)
|
||||
throw new ArgumentException("samples must contain at least one value", nameof(samples));
|
||||
if (percentile < 0 || percentile > 100)
|
||||
throw new ArgumentOutOfRangeException(nameof(percentile), "percentile must be in [0, 100]");
|
||||
|
||||
var sorted = samples.ToArray();
|
||||
Array.Sort(sorted);
|
||||
|
||||
// Nearest-rank: rank = ceil(p/100 * N); index = rank - 1.
|
||||
var rank = (int)Math.Ceiling(percentile / 100.0 * sorted.Length);
|
||||
if (rank < 1) rank = 1;
|
||||
if (rank > sorted.Length) rank = sorted.Length;
|
||||
return sorted[rank - 1];
|
||||
}
|
||||
|
||||
public static double Mean(IReadOnlyList<double> samples)
|
||||
{
|
||||
if (samples.Count == 0)
|
||||
throw new ArgumentException("samples must contain at least one value", nameof(samples));
|
||||
return samples.Average();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Azaion.Missions.E2E.Helpers;
|
||||
|
||||
/// <summary>
|
||||
/// Appends one row per NFT-PERF / NFT-RES-LIM scenario to a side-channel
|
||||
/// CSV referenced by an environment variable. The Reporting.Cli converter
|
||||
/// only knows about compile-time <c>[Trait]</c> data — runtime measurements
|
||||
/// (P50/P95, MAX_FD, P95_RSS_MiB, etc.) need this separate file so
|
||||
/// deployment planning + trend dashboards can read them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// File schema (idempotent header written on first append):
|
||||
/// <code>Timestamp,Category,Scenario,Result,Traces,ErrorMessage</code>
|
||||
/// The Traces column carries the dynamic key=value pairs the spec requires
|
||||
/// (e.g., <c>"AC-3.6; P50_MS=23.4; P95_MS=41.8"</c>); the recorder just
|
||||
/// joins them with semicolons — callers compose the right shape.
|
||||
/// </remarks>
|
||||
public sealed class MetricCsvRecorder
|
||||
{
|
||||
private readonly string? _path;
|
||||
private static readonly object Lock = new();
|
||||
|
||||
/// <param name="envVar">name of the env var that carries the target CSV path
|
||||
/// (e.g., <c>PERF_RESULTS_FILE</c> for NFT-PERF, <c>RESLIM_RESULTS_FILE</c>
|
||||
/// for NFT-RES-LIM). When the env var is missing or whitespace, every
|
||||
/// <see cref="Record"/> call is a no-op — the recorder is intentionally
|
||||
/// silent inside the standard CI run.</param>
|
||||
public MetricCsvRecorder(string envVar)
|
||||
{
|
||||
var v = Environment.GetEnvironmentVariable(envVar);
|
||||
_path = string.IsNullOrWhiteSpace(v) ? null : v;
|
||||
}
|
||||
|
||||
public bool IsEnabled => _path is not null;
|
||||
|
||||
public void Record(string category, string scenario, string result, string traces, string? errorMessage = null)
|
||||
{
|
||||
if (_path is null) return;
|
||||
lock (Lock)
|
||||
{
|
||||
var dir = Path.GetDirectoryName(_path);
|
||||
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
|
||||
var newFile = !File.Exists(_path);
|
||||
using var sw = new StreamWriter(_path, append: true);
|
||||
if (newFile)
|
||||
sw.WriteLine("Timestamp,Category,Scenario,Result,Traces,ErrorMessage");
|
||||
sw.WriteLine(
|
||||
$"{DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)},"
|
||||
+ $"{Csv(category)},{Csv(scenario)},{Csv(result)},{Csv(traces)},{Csv(errorMessage ?? "")}");
|
||||
}
|
||||
}
|
||||
|
||||
private static string Csv(string value) =>
|
||||
value.Contains(',') || value.Contains('"') || value.Contains('\n')
|
||||
? "\"" + value.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""
|
||||
: value;
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Net.Http.Headers;
|
||||
using Azaion.Missions.E2E.Fixtures;
|
||||
using Azaion.Missions.E2E.Helpers;
|
||||
using Npgsql;
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Tests.Performance;
|
||||
|
||||
/// <summary>
|
||||
/// NFT-PERF-01..04 — wall-clock latency observations against the dockerised
|
||||
/// <c>missions</c> service. Excluded from the default CI gate via
|
||||
/// <c>--filter "Category!=Perf"</c> in <c>entrypoint.sh</c>; run via
|
||||
/// <c>scripts/run-performance-tests.sh</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each scenario follows the same shape: seed deterministic data, warm-up
|
||||
/// 5 calls (excluded from the percentile), run N measured sequential calls
|
||||
/// recording <see cref="Stopwatch"/> wall-clock, compute P50 + P95, record
|
||||
/// them to the runtime CSV referenced by <c>PERF_RESULTS_FILE</c>, then
|
||||
/// assert against the documented gate. Sequential single-client execution
|
||||
/// keeps HTTP/1.1 connection-reuse and JIT warm-up deterministic.
|
||||
/// </remarks>
|
||||
[Collection("Perf")]
|
||||
[Trait("Category", "Perf")]
|
||||
public sealed class PerformanceTests : TestBase, IClassFixture<DbResetFixture>
|
||||
{
|
||||
private static readonly MetricCsvRecorder Csv = new("PERF_RESULTS_FILE");
|
||||
private const int WarmupCalls = 5;
|
||||
|
||||
[Fact(Timeout = 60_000)]
|
||||
[Trait("Traces", "AC-3.6")]
|
||||
[Trait("max_ms", "30000")]
|
||||
public async Task NFT_PERF_01_minimal_cascade_delete_p50_within_50ms()
|
||||
{
|
||||
// Arrange — 105 missions (100 measured + 5 warmup), each with one waypoint.
|
||||
DbResetFixture.ResetDatabase(TestEnvironment.DbSideChannel);
|
||||
Seeds.Apply(Seeds.OneDefaultVehicle.Sql);
|
||||
var (measured, warmup) = SeedSequentialMissions(105, waypointsPerMission: 1);
|
||||
await AttachAuthAsync();
|
||||
|
||||
await WarmupDeletesAsync(warmup);
|
||||
|
||||
// Act
|
||||
var latenciesMs = await MeasureSequentialDeletesAsync(measured);
|
||||
var p50 = LatencyPercentiles.P50(latenciesMs);
|
||||
var p95 = LatencyPercentiles.P95(latenciesMs);
|
||||
|
||||
Csv.Record(
|
||||
category: "Perf",
|
||||
scenario: "NFT-PERF-01",
|
||||
result: p50 <= 50.0 ? "pass" : "fail",
|
||||
traces: $"AC-3.6; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
|
||||
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert
|
||||
Assert.True(p50 <= 50.0,
|
||||
$"NFT-PERF-01 P50 budget exceeded: P50={p50:F2}ms (gate=50ms), P95={p95:F2}ms");
|
||||
}
|
||||
|
||||
[Fact(Timeout = 120_000)]
|
||||
[Trait("Traces", "AC-3.1,AC-3.6")]
|
||||
[Trait("max_ms", "60000")]
|
||||
[Trait("provisional", "yes")]
|
||||
public async Task NFT_PERF_02_full_chain_cascade_delete_p50_within_200ms_provisional()
|
||||
{
|
||||
// PROVISIONAL — lock at measured + 50% on first green run.
|
||||
// Arrange — 55 F3-shaped missions (50 measured + 5 warmup).
|
||||
DbResetFixture.ResetDatabase(TestEnvironment.DbSideChannel);
|
||||
Seeds.Apply(Seeds.OneDefaultVehicle.Sql);
|
||||
StubSchema.EnsureCreated();
|
||||
var (measured, warmup) = SeedF3MissionsCascadeChains(55);
|
||||
await AttachAuthAsync();
|
||||
|
||||
await WarmupDeletesAsync(warmup);
|
||||
|
||||
// Act
|
||||
var latenciesMs = await MeasureSequentialDeletesAsync(measured);
|
||||
var p50 = LatencyPercentiles.P50(latenciesMs);
|
||||
var p95 = LatencyPercentiles.P95(latenciesMs);
|
||||
|
||||
Csv.Record(
|
||||
category: "Perf",
|
||||
scenario: "NFT-PERF-02",
|
||||
result: p50 <= 200.0 ? "pass" : "fail",
|
||||
traces: $"AC-3.1; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
|
||||
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert
|
||||
Assert.True(p50 <= 200.0,
|
||||
$"NFT-PERF-02 P50 (provisional 200ms) exceeded: P50={p50:F2}ms, P95={p95:F2}ms");
|
||||
}
|
||||
|
||||
[Fact(Timeout = 30_000)]
|
||||
[Trait("Traces", "AC-7.3")]
|
||||
[Trait("max_ms", "5000")]
|
||||
public async Task NFT_PERF_03_health_p50_within_10ms()
|
||||
{
|
||||
// Arrange — no seed needed; /health is anonymous.
|
||||
for (int i = 0; i < WarmupCalls; i++)
|
||||
{
|
||||
using var resp = await Missions.GetAsync("/health");
|
||||
resp.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
// Act
|
||||
var latenciesMs = new List<double>(100);
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
using var resp = await Missions.GetAsync("/health");
|
||||
sw.Stop();
|
||||
resp.EnsureSuccessStatusCode();
|
||||
latenciesMs.Add(sw.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
var p50 = LatencyPercentiles.P50(latenciesMs);
|
||||
var p95 = LatencyPercentiles.P95(latenciesMs);
|
||||
|
||||
Csv.Record(
|
||||
category: "Perf",
|
||||
scenario: "NFT-PERF-03",
|
||||
result: p50 <= 10.0 ? "pass" : "fail",
|
||||
traces: $"AC-7.3; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
|
||||
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert
|
||||
Assert.True(p50 <= 10.0,
|
||||
$"NFT-PERF-03 P50 budget exceeded: P50={p50:F2}ms (gate=10ms), P95={p95:F2}ms");
|
||||
}
|
||||
|
||||
[Fact(Timeout = 90_000)]
|
||||
[Trait("Traces", "AC-2.3")]
|
||||
[Trait("max_ms", "30000")]
|
||||
[Trait("provisional", "yes")]
|
||||
public async Task NFT_PERF_04_missions_list_pagination_p95_within_100ms_provisional()
|
||||
{
|
||||
// PROVISIONAL — lock at measured + 50% on first green run.
|
||||
// Arrange — 1000 missions referencing seed_one_default_vehicle.
|
||||
DbResetFixture.ResetDatabase(TestEnvironment.DbSideChannel);
|
||||
Seeds.Apply(Seeds.OneDefaultVehicle.Sql);
|
||||
SeedSequentialMissionsNoWaypoints(1000);
|
||||
await AttachAuthAsync();
|
||||
|
||||
for (int i = 0; i < WarmupCalls; i++)
|
||||
{
|
||||
using var resp = await Missions.GetAsync("/missions?page=1&pageSize=20");
|
||||
resp.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
// Act
|
||||
var latenciesMs = new List<double>(100);
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
using var resp = await Missions.GetAsync("/missions?page=1&pageSize=20");
|
||||
sw.Stop();
|
||||
resp.EnsureSuccessStatusCode();
|
||||
latenciesMs.Add(sw.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
var p50 = LatencyPercentiles.P50(latenciesMs);
|
||||
var p95 = LatencyPercentiles.P95(latenciesMs);
|
||||
|
||||
Csv.Record(
|
||||
category: "Perf",
|
||||
scenario: "NFT-PERF-04",
|
||||
result: p95 <= 100.0 ? "pass" : "fail",
|
||||
traces: $"AC-2.3; P50_MS={p50.ToString("F2", CultureInfo.InvariantCulture)}; "
|
||||
+ $"P95_MS={p95.ToString("F2", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert
|
||||
Assert.True(p95 <= 100.0,
|
||||
$"NFT-PERF-04 P95 (provisional 100ms) exceeded: P50={p50:F2}ms, P95={p95:F2}ms");
|
||||
}
|
||||
|
||||
private async Task AttachAuthAsync()
|
||||
{
|
||||
var t = await Tokens.MintDefaultAsync();
|
||||
Missions.DefaultRequestHeaders.Authorization =
|
||||
new AuthenticationHeaderValue("Bearer", t.Jwt);
|
||||
}
|
||||
|
||||
private async Task WarmupDeletesAsync(IReadOnlyList<Guid> warmupMissionIds)
|
||||
{
|
||||
foreach (var id in warmupMissionIds)
|
||||
{
|
||||
using var resp = await Missions.DeleteAsync($"/missions/{id}");
|
||||
// 200 or 204 are both acceptable; the cascade walks regardless.
|
||||
// 4xx would indicate a seed problem — fail loudly.
|
||||
if (!resp.IsSuccessStatusCode && (int)resp.StatusCode != 404)
|
||||
throw new InvalidOperationException(
|
||||
$"warmup DELETE /missions/{id} returned {(int)resp.StatusCode}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<List<double>> MeasureSequentialDeletesAsync(IReadOnlyList<Guid> missionIds)
|
||||
{
|
||||
var latencies = new List<double>(missionIds.Count);
|
||||
foreach (var id in missionIds)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
using var resp = await Missions.DeleteAsync($"/missions/{id}");
|
||||
sw.Stop();
|
||||
if (!resp.IsSuccessStatusCode && (int)resp.StatusCode != 404)
|
||||
throw new InvalidOperationException(
|
||||
$"measured DELETE /missions/{id} returned {(int)resp.StatusCode}");
|
||||
latencies.Add(sw.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
return latencies;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns (measured, warmup) where the FIRST 5 IDs are the warmup set
|
||||
/// and the remaining (count-5) IDs are the measured set. Each mission
|
||||
/// gets the requested number of waypoints with deterministic IDs.
|
||||
/// </summary>
|
||||
private static (List<Guid> Measured, List<Guid> Warmup) SeedSequentialMissions(
|
||||
int count, int waypointsPerMission)
|
||||
{
|
||||
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
|
||||
conn.Open();
|
||||
using var tx = conn.BeginTransaction();
|
||||
|
||||
var ids = new List<Guid>(count);
|
||||
var seed = new Random(98765);
|
||||
|
||||
using (var insertMission = conn.CreateCommand())
|
||||
{
|
||||
insertMission.Transaction = tx;
|
||||
insertMission.CommandText = """
|
||||
INSERT INTO missions (id, name, vehicle_id)
|
||||
VALUES (@id, @name, @vehicle_id);
|
||||
""";
|
||||
insertMission.Parameters.Add(new NpgsqlParameter("id", NpgsqlTypes.NpgsqlDbType.Uuid));
|
||||
insertMission.Parameters.Add(new NpgsqlParameter("name", NpgsqlTypes.NpgsqlDbType.Text));
|
||||
insertMission.Parameters.Add(new NpgsqlParameter("vehicle_id", NpgsqlTypes.NpgsqlDbType.Uuid));
|
||||
|
||||
using var insertWaypoint = conn.CreateCommand();
|
||||
insertWaypoint.Transaction = tx;
|
||||
insertWaypoint.CommandText = """
|
||||
INSERT INTO waypoints (id, mission_id, lat, lon, mgrs, order_num)
|
||||
VALUES (@id, @mission_id, 50.45, 30.52, '36UYA1234567', @order_num);
|
||||
""";
|
||||
insertWaypoint.Parameters.Add(new NpgsqlParameter("id", NpgsqlTypes.NpgsqlDbType.Uuid));
|
||||
insertWaypoint.Parameters.Add(new NpgsqlParameter("mission_id", NpgsqlTypes.NpgsqlDbType.Uuid));
|
||||
insertWaypoint.Parameters.Add(new NpgsqlParameter("order_num", NpgsqlTypes.NpgsqlDbType.Integer));
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
var id = NewDeterministicGuid(seed);
|
||||
ids.Add(id);
|
||||
insertMission.Parameters["id"].Value = id;
|
||||
insertMission.Parameters["name"].Value = $"perf-mission-{i:D4}";
|
||||
insertMission.Parameters["vehicle_id"].Value = Seeds.OneDefaultVehicle.Id;
|
||||
insertMission.ExecuteNonQuery();
|
||||
|
||||
for (int w = 0; w < waypointsPerMission; w++)
|
||||
{
|
||||
insertWaypoint.Parameters["id"].Value = NewDeterministicGuid(seed);
|
||||
insertWaypoint.Parameters["mission_id"].Value = id;
|
||||
insertWaypoint.Parameters["order_num"].Value = w;
|
||||
insertWaypoint.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tx.Commit();
|
||||
var warmup = ids.Take(WarmupCalls).ToList();
|
||||
var measured = ids.Skip(WarmupCalls).ToList();
|
||||
return (measured, warmup);
|
||||
}
|
||||
|
||||
private static void SeedSequentialMissionsNoWaypoints(int count)
|
||||
{
|
||||
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
|
||||
conn.Open();
|
||||
using var tx = conn.BeginTransaction();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.Transaction = tx;
|
||||
cmd.CommandText = """
|
||||
INSERT INTO missions (id, name, vehicle_id)
|
||||
VALUES (@id, @name, @vehicle_id);
|
||||
""";
|
||||
cmd.Parameters.Add(new NpgsqlParameter("id", NpgsqlTypes.NpgsqlDbType.Uuid));
|
||||
cmd.Parameters.Add(new NpgsqlParameter("name", NpgsqlTypes.NpgsqlDbType.Text));
|
||||
cmd.Parameters.Add(new NpgsqlParameter("vehicle_id", NpgsqlTypes.NpgsqlDbType.Uuid));
|
||||
|
||||
var seed = new Random(13579);
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
cmd.Parameters["id"].Value = NewDeterministicGuid(seed);
|
||||
cmd.Parameters["name"].Value = $"list-perf-{i:D4}";
|
||||
cmd.Parameters["vehicle_id"].Value = Seeds.OneDefaultVehicle.Id;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
tx.Commit();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeds <paramref name="count"/> missions, each with the F3 cascade shape:
|
||||
/// 3 map_objects + 2 waypoints + (per waypoint: 2 media → 2 annotations → 2 detection).
|
||||
/// </summary>
|
||||
private static (List<Guid> Measured, List<Guid> Warmup) SeedF3MissionsCascadeChains(int count)
|
||||
{
|
||||
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
|
||||
conn.Open();
|
||||
var ids = new List<Guid>(count);
|
||||
var seed = new Random(24680);
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
using var tx = conn.BeginTransaction();
|
||||
var missionId = NewDeterministicGuid(seed);
|
||||
ids.Add(missionId);
|
||||
|
||||
ExecScalar(conn, tx, """
|
||||
INSERT INTO missions (id, name, vehicle_id) VALUES (@id, @name, @vid);
|
||||
""", ("id", missionId), ("name", $"f3-perf-{i:D4}"),
|
||||
("vid", Seeds.OneDefaultVehicle.Id));
|
||||
|
||||
for (int m = 0; m < 3; m++)
|
||||
ExecScalar(conn, tx, """
|
||||
INSERT INTO map_objects (id, mission_id, h3_index, mgrs)
|
||||
VALUES (@id, @mid, '8a2a1072b59ffff', '36UYA1234567');
|
||||
""", ("id", NewDeterministicGuid(seed)), ("mid", missionId));
|
||||
|
||||
for (int w = 0; w < 2; w++)
|
||||
{
|
||||
var wpId = NewDeterministicGuid(seed);
|
||||
ExecScalar(conn, tx, """
|
||||
INSERT INTO waypoints (id, mission_id, lat, lon, mgrs, order_num)
|
||||
VALUES (@id, @mid, 50.45, 30.52, '36UYA1234567', @ord);
|
||||
""", ("id", wpId), ("mid", missionId), ("ord", w));
|
||||
|
||||
for (int md = 0; md < 2; md++)
|
||||
{
|
||||
var mediaId = $"media-{Guid.NewGuid():N}";
|
||||
ExecScalar(conn, tx, """
|
||||
INSERT INTO media (id, waypoint_id) VALUES (@id, @wid);
|
||||
""", ("id", mediaId), ("wid", wpId));
|
||||
|
||||
var annId = $"ann-{Guid.NewGuid():N}";
|
||||
ExecScalar(conn, tx, """
|
||||
INSERT INTO annotations (id, media_id) VALUES (@id, @mid);
|
||||
""", ("id", annId), ("mid", mediaId));
|
||||
|
||||
ExecScalar(conn, tx, """
|
||||
INSERT INTO detection (id, annotation_id) VALUES (@id, @aid);
|
||||
""", ("id", NewDeterministicGuid(seed)), ("aid", annId));
|
||||
}
|
||||
}
|
||||
tx.Commit();
|
||||
}
|
||||
|
||||
var warmup = ids.Take(WarmupCalls).ToList();
|
||||
var measured = ids.Skip(WarmupCalls).ToList();
|
||||
return (measured, warmup);
|
||||
}
|
||||
|
||||
private static void ExecScalar(NpgsqlConnection conn, NpgsqlTransaction tx, string sql,
|
||||
params (string Name, object Value)[] args)
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.Transaction = tx;
|
||||
cmd.CommandText = sql;
|
||||
foreach (var (name, value) in args)
|
||||
cmd.Parameters.AddWithValue(name, value);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
private static Guid NewDeterministicGuid(Random rng)
|
||||
{
|
||||
var bytes = new byte[16];
|
||||
rng.NextBytes(bytes);
|
||||
// Force version 4 + variant 1 so the value is a valid UUID Postgres accepts.
|
||||
bytes[7] = (byte)((bytes[7] & 0x0F) | 0x40);
|
||||
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
|
||||
return new Guid(bytes);
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Tests.Performance;
|
||||
|
||||
/// <summary>
|
||||
/// Discovery-only smoke test for the Performance category. Real Performance
|
||||
/// scenarios (NFT-PERF-01..04) land in AZ-586.
|
||||
/// </summary>
|
||||
public sealed class Sanity
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "Perf")]
|
||||
[Trait("Traces", "AC-3")]
|
||||
public void Discovery_smoke_test_runs()
|
||||
{
|
||||
// Arrange
|
||||
const int sentinel = 1;
|
||||
// Act
|
||||
var result = sentinel + 0;
|
||||
// Assert
|
||||
Assert.Equal(1, result);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using Azaion.Missions.E2E.Helpers;
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Tests.ResourceLimits;
|
||||
|
||||
/// <summary>
|
||||
/// NFT-RES-LIM-04 — cold-start RSS. Driven independently from the
|
||||
/// steady-state window because it requires a fresh container start; lives
|
||||
/// in the <c>MigratorRestart</c> collection so it serialises with the
|
||||
/// other docker-compose-restarting tests rather than racing them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The 30-second wait between health-OK and the measurement is the spec's
|
||||
/// way of letting the JIT and the JWKS prefetch settle without doing any
|
||||
/// real work — measuring at health-OK alone would conflate the genuine cold
|
||||
/// baseline with bootstrap noise.
|
||||
/// </remarks>
|
||||
[Collection("MigratorRestart")]
|
||||
[Trait("Category", "ResLim")]
|
||||
public sealed class ColdStartRssTests
|
||||
{
|
||||
private static readonly MetricCsvRecorder Csv = new("RESLIM_RESULTS_FILE");
|
||||
private const long ProvisionalColdRssCapMiB = 200;
|
||||
private const string ComposeFile = "/workspace/docker-compose.test.yml";
|
||||
|
||||
[SkippableFact]
|
||||
[Trait("Traces", "H1|H3")]
|
||||
[Trait("max_ms", "120000")]
|
||||
public async Task NFT_RES_LIM_04_cold_start_rss_within_provisional_200_MiB()
|
||||
{
|
||||
Skip.IfNot(Environment.GetEnvironmentVariable("COMPOSE_RESTART_ENABLED") == "1",
|
||||
"COMPOSE_RESTART_ENABLED!=1 — docker compose restart unavailable in this consumer image");
|
||||
Skip.IfNot(MissionsContainerHelper.Enabled,
|
||||
"MissionsContainerHelper disabled — docker CLI unavailable");
|
||||
|
||||
// Arrange — bring missions down hard and start it fresh. The
|
||||
// surrounding "MigratorRestart" collection serialises us against
|
||||
// any other test that touches the SUT.
|
||||
DockerCompose("stop missions");
|
||||
DockerCompose("rm -f missions");
|
||||
DockerCompose("up -d missions");
|
||||
|
||||
await WaitForHealthOkAsync(TimeSpan.FromSeconds(60));
|
||||
|
||||
// Act — wait 30s after health-OK so JIT/JWKS settle, then measure.
|
||||
await Task.Delay(TimeSpan.FromSeconds(30));
|
||||
var rssBytes = ReadRssBytes("missions-sut");
|
||||
var rssMiB = rssBytes / (double)(1024 * 1024);
|
||||
|
||||
var pass = rssMiB <= ProvisionalColdRssCapMiB;
|
||||
Csv.Record(
|
||||
category: "ResLim",
|
||||
scenario: "NFT-RES-LIM-04",
|
||||
result: pass ? "pass" : "fail",
|
||||
traces: $"H1|H3; COLD_RSS_MiB={rssMiB.ToString("F1", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert — provisional gate.
|
||||
Assert.True(pass,
|
||||
$"cold-start RSS {rssMiB:F1} MiB exceeds provisional {ProvisionalColdRssCapMiB} MiB gate");
|
||||
}
|
||||
|
||||
private static async Task WaitForHealthOkAsync(TimeSpan timeout)
|
||||
{
|
||||
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(2) };
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
var healthUrl = new Uri(TestEnvironment.MissionsBaseUrl + "/health");
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var resp = await http.GetAsync(healthUrl);
|
||||
if (resp.StatusCode == HttpStatusCode.OK) return;
|
||||
}
|
||||
catch (HttpRequestException) { /* not yet listening */ }
|
||||
catch (TaskCanceledException) { /* slow first response */ }
|
||||
await Task.Delay(500);
|
||||
}
|
||||
throw new TimeoutException(
|
||||
$"missions did not become healthy within {timeout.TotalSeconds:F0}s of cold start");
|
||||
}
|
||||
|
||||
private static long ReadRssBytes(string containerName)
|
||||
{
|
||||
var raw = Run("docker",
|
||||
$"stats --no-stream --format '{{{{.MemUsage}}}}' {containerName}");
|
||||
var lhs = raw.Split('/')[0].Trim().Trim('\'');
|
||||
return ParseHumanBytes(lhs);
|
||||
}
|
||||
|
||||
private static long ParseHumanBytes(string text)
|
||||
{
|
||||
var unitIx = text.IndexOfAny(new[] { 'K', 'M', 'G', 'T', 'B' });
|
||||
if (unitIx < 0) return long.Parse(text, CultureInfo.InvariantCulture);
|
||||
var num = double.Parse(text.Substring(0, unitIx), CultureInfo.InvariantCulture);
|
||||
var unit = text.Substring(unitIx);
|
||||
return unit switch
|
||||
{
|
||||
"B" => (long)num,
|
||||
"KiB" or "KB" or "K" => (long)(num * 1024),
|
||||
"MiB" or "MB" or "M" => (long)(num * 1024 * 1024),
|
||||
"GiB" or "GB" or "G" => (long)(num * 1024 * 1024 * 1024),
|
||||
"TiB" or "TB" or "T" => (long)(num * 1024L * 1024 * 1024 * 1024),
|
||||
_ => throw new FormatException($"unknown human-bytes unit in '{text}'")
|
||||
};
|
||||
}
|
||||
|
||||
private static void DockerCompose(string subcommand) =>
|
||||
Run("docker", $"compose -f {ComposeFile} {subcommand}");
|
||||
|
||||
private static string Run(string file, string args)
|
||||
{
|
||||
var psi = new ProcessStartInfo(file, args)
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
using var p = Process.Start(psi)
|
||||
?? throw new InvalidOperationException($"failed to launch `{file} {args}`");
|
||||
var stdout = p.StandardOutput.ReadToEnd();
|
||||
var stderr = p.StandardError.ReadToEnd();
|
||||
p.WaitForExit();
|
||||
if (p.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"`{file} {args}` exited {p.ExitCode}: {stderr}");
|
||||
return stdout;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Tests.ResourceLimits;
|
||||
|
||||
/// <summary>
|
||||
/// Discovery-only smoke test for the ResourceLimits category. Real
|
||||
/// ResourceLimits scenarios (NFT-RES-LIM-01..04) land in AZ-585.
|
||||
/// </summary>
|
||||
public sealed class Sanity
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", "ResLim")]
|
||||
[Trait("Traces", "AC-3")]
|
||||
public void Discovery_smoke_test_runs()
|
||||
{
|
||||
// Arrange
|
||||
const int sentinel = 1;
|
||||
// Act
|
||||
var result = sentinel + 0;
|
||||
// Assert
|
||||
Assert.Equal(1, result);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
using System.Globalization;
|
||||
using Azaion.Missions.E2E.Fixtures;
|
||||
using Azaion.Missions.E2E.Helpers;
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Tests.ResourceLimits;
|
||||
|
||||
/// <summary>
|
||||
/// NFT-RES-LIM-01..03 — three observations on a SINGLE 5-minute sustained
|
||||
/// load window. The window itself lives in
|
||||
/// <see cref="SteadyStateLoadFixture"/> (class-scoped, runs once); each
|
||||
/// test asserts one metric against its provisional gate.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The fixture skips itself when docker primitives are unavailable; the
|
||||
/// tests detect that via <see cref="SteadyStateLoadFixture.SkipReason"/>
|
||||
/// and surface the same reason through <c>Skip.IfNot</c>. The fixture
|
||||
/// also flips <see cref="SteadyStateLoadFixture.SutExitedDuringWindow"/>
|
||||
/// if the SUT crashes mid-window — every test fails fast with a clear
|
||||
/// message rather than reporting a misleading metric.
|
||||
/// </remarks>
|
||||
[Collection("ResLimSteadyState")]
|
||||
[Trait("Category", "ResLim")]
|
||||
public sealed class SteadyStateLoadTests : TestBase, IClassFixture<SteadyStateLoadFixture>
|
||||
{
|
||||
private static readonly MetricCsvRecorder Csv = new("RESLIM_RESULTS_FILE");
|
||||
private const long ProvisionalRssCapMiB = 250;
|
||||
private const int ProvisionalConnectionCap = 100;
|
||||
private const int ProvisionalFdCap = 1024;
|
||||
|
||||
private readonly SteadyStateLoadFixture _load;
|
||||
|
||||
public SteadyStateLoadTests(SteadyStateLoadFixture load) => _load = load;
|
||||
|
||||
[SkippableFact]
|
||||
[Trait("Traces", "H1|H6|O10")]
|
||||
[Trait("max_ms", "360000")]
|
||||
public void NFT_RES_LIM_01_steady_state_rss_within_provisional_gate_and_no_leak()
|
||||
{
|
||||
Skip.If(_load.SkipReason is not null, _load.SkipReason);
|
||||
Skip.IfNot(_load.LoadGeneratorMetTargetRps,
|
||||
"runner cannot sustain target load (NFR Reliability — not a SUT defect)");
|
||||
Assert.False(_load.SutExitedDuringWindow, "SUT exited during measurement window");
|
||||
|
||||
// Arrange
|
||||
var samplesMiB = _load.RssBytesSamples.Select(b => b / (double)(1024 * 1024)).ToList();
|
||||
Assert.True(samplesMiB.Count >= 30,
|
||||
$"expected ≥ 30 RSS samples over 5-min window, got {samplesMiB.Count}");
|
||||
|
||||
// Act
|
||||
var p95 = LatencyPercentiles.P95(samplesMiB);
|
||||
var finalMiB = samplesMiB[^1];
|
||||
|
||||
var leakRatio = Math.Abs(finalMiB - p95) / Math.Max(p95, 1.0);
|
||||
var withinCap = p95 <= ProvisionalRssCapMiB;
|
||||
var noLeak = leakRatio <= 0.20;
|
||||
var pass = withinCap && noLeak;
|
||||
|
||||
Csv.Record(
|
||||
category: "ResLim",
|
||||
scenario: "NFT-RES-LIM-01",
|
||||
result: pass ? "pass" : "fail",
|
||||
traces: $"H1|H6|O10; "
|
||||
+ $"P95_RSS_MiB={p95.ToString("F1", CultureInfo.InvariantCulture)}; "
|
||||
+ $"FINAL_RSS_MiB={finalMiB.ToString("F1", CultureInfo.InvariantCulture)}; "
|
||||
+ $"LEAK_RATIO={leakRatio.ToString("F2", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert — provisional gate; lock at measured + 50% after first green run.
|
||||
Assert.True(withinCap,
|
||||
$"P95 RSS {p95:F1} MiB exceeds provisional {ProvisionalRssCapMiB} MiB gate");
|
||||
Assert.True(noLeak,
|
||||
$"final RSS {finalMiB:F1} MiB diverges {leakRatio:P0} from P95 {p95:F1} MiB (gate 20%)");
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
[Trait("Traces", "O10")]
|
||||
[Trait("max_ms", "360000")]
|
||||
public void NFT_RES_LIM_02_npgsql_connection_pool_within_100_no_unbounded_growth()
|
||||
{
|
||||
Skip.If(_load.SkipReason is not null, _load.SkipReason);
|
||||
Skip.IfNot(_load.LoadGeneratorMetTargetRps,
|
||||
"runner cannot sustain target load (NFR Reliability — not a SUT defect)");
|
||||
Assert.False(_load.SutExitedDuringWindow, "SUT exited during measurement window");
|
||||
|
||||
var samples = _load.NpgsqlConnectionSamples;
|
||||
Assert.True(samples.Count >= 30,
|
||||
$"expected ≥ 30 connection samples over 5-min window, got {samples.Count}");
|
||||
|
||||
// Act
|
||||
var max = samples.Max();
|
||||
var firstMinuteSampleCount = 60 / SteadyStateLoadFixture.SampleIntervalSeconds;
|
||||
var firstMinute = samples.Take(firstMinuteSampleCount).ToList();
|
||||
var firstMinuteMean = firstMinute.Average();
|
||||
var finalCount = samples[^1];
|
||||
|
||||
var withinCap = max <= ProvisionalConnectionCap;
|
||||
var noUnboundedGrowth = finalCount <= 1.3 * Math.Max(firstMinuteMean, 1.0);
|
||||
var pass = withinCap && noUnboundedGrowth;
|
||||
|
||||
Csv.Record(
|
||||
category: "ResLim",
|
||||
scenario: "NFT-RES-LIM-02",
|
||||
result: pass ? "pass" : "fail",
|
||||
traces: $"O10; MAX_NPGSQL_CONNS={max}; "
|
||||
+ $"FINAL_CONNS={finalCount}; "
|
||||
+ $"MINUTE1_MEAN={firstMinuteMean.ToString("F1", CultureInfo.InvariantCulture)}");
|
||||
|
||||
// Assert
|
||||
Assert.True(withinCap,
|
||||
$"max Npgsql connections {max} exceeds provisional cap {ProvisionalConnectionCap}");
|
||||
Assert.True(noUnboundedGrowth,
|
||||
$"final connection count {finalCount} > 1.3 × first-minute mean {firstMinuteMean:F1}");
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
[Trait("Traces", "H6|O10")]
|
||||
[Trait("max_ms", "360000")]
|
||||
public void NFT_RES_LIM_03_file_descriptors_within_1024_no_leak()
|
||||
{
|
||||
Skip.If(_load.SkipReason is not null, _load.SkipReason);
|
||||
Skip.IfNot(_load.LoadGeneratorMetTargetRps,
|
||||
"runner cannot sustain target load (NFR Reliability — not a SUT defect)");
|
||||
Assert.False(_load.SutExitedDuringWindow, "SUT exited during measurement window");
|
||||
|
||||
var samples = _load.FileDescriptorSamples;
|
||||
Assert.True(samples.Count >= 30,
|
||||
$"expected ≥ 30 FD samples over 5-min window, got {samples.Count}");
|
||||
|
||||
// Act
|
||||
var max = samples.Max();
|
||||
var minuteOneSampleCount = 60 / SteadyStateLoadFixture.SampleIntervalSeconds;
|
||||
// The spec calls out "count at t=1min" — anchor on the sample whose
|
||||
// timestamp is closest to (start + 60s).
|
||||
var minuteOneIx = Math.Min(minuteOneSampleCount - 1, samples.Count - 1);
|
||||
var minuteOneCount = samples[minuteOneIx];
|
||||
var finalCount = samples[^1];
|
||||
|
||||
var withinCap = max <= ProvisionalFdCap;
|
||||
var noLeak = finalCount <= 1.3 * Math.Max(minuteOneCount, 1);
|
||||
var pass = withinCap && noLeak;
|
||||
|
||||
Csv.Record(
|
||||
category: "ResLim",
|
||||
scenario: "NFT-RES-LIM-03",
|
||||
result: pass ? "pass" : "fail",
|
||||
traces: $"H6|O10; MAX_FD={max}; "
|
||||
+ $"FINAL_FD={finalCount}; MINUTE1_FD={minuteOneCount}");
|
||||
|
||||
// Assert
|
||||
Assert.True(withinCap,
|
||||
$"max FD count {max} exceeds provisional cap {ProvisionalFdCap}");
|
||||
Assert.True(noLeak,
|
||||
$"final FD count {finalCount} > 1.3 × minute-1 count {minuteOneCount}");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -14,9 +14,16 @@ set -eu
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
set +e
|
||||
## Performance scenarios (Category=Perf) are excluded from the default gate
|
||||
## per AZ-586. They are invoked from scripts/run-performance-tests.sh which
|
||||
## passes its own --filter Category=Perf. ResLim tests (Category=ResLim) stay
|
||||
## in the default gate because their docker-CLI gate causes them to skip
|
||||
## with an explicit reason when COMPOSE_RESTART_ENABLED is not set.
|
||||
TEST_FILTER="${TEST_FILTER:-Category!=Perf}"
|
||||
dotnet test /src/Azaion.Missions.E2E.Tests.csproj \
|
||||
--no-build \
|
||||
--configuration Release \
|
||||
--filter "$TEST_FILTER" \
|
||||
--logger "trx;LogFileName=results.trx" \
|
||||
--logger "console;verbosity=normal" \
|
||||
--results-directory "$RESULTS_DIR"
|
||||
|
||||
Reference in New Issue
Block a user