mirror of
https://github.com/azaion/missions.git
synced 2026-06-22 20:21:07 +00:00
[AZ-585] [AZ-586] ResLim+Perf NFT tests; close test cycle 1
Batch 4 of test implementation cycle 1 (existing-code Step 6, final batch).
- AZ-585 SteadyStateLoadTests + ColdStartRssTests: NFT-RES-LIM-01..04.
SteadyStateLoadFixture runs one 5-min sustained-load window and samples
RSS (docker stats), Npgsql conns (pg_stat_activity), and FDs
(/proc/1/fd) every 5s; three test methods assert independently. All
SkippableFact-gated on docker primitives.
- AZ-586 PerformanceTests: NFT-PERF-01..04. Sequential single-client,
5 warm-ups + N measured calls, P50+P95 via LatencyPercentiles, recorded
to PERF_RESULTS_FILE. Tagged Category=Perf so default gate excludes them.
Infrastructure:
- entrypoint.sh now applies --filter "${TEST_FILTER:-Category!=Perf}"
per AZ-586 (default CI gate excludes performance).
- MetricCsvRecorder: idempotent CSV appender keyed on env var, used by
both Perf and ResLim categories.
Step 6 (Implement Tests) is complete. Final report at
_docs/03_implementation/implementation_report_tests.md handoffs the
full-suite gate to test-run/SKILL.md (Step 7).
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Net.Http.Headers;
|
||||
using Azaion.Missions.E2E.Helpers;
|
||||
using Npgsql;
|
||||
using Xunit;
|
||||
|
||||
namespace Azaion.Missions.E2E.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Shared 5-minute steady-state load fixture for NFT-RES-LIM-01 / -02 / -03.
|
||||
/// Runs the load generator once, samples RSS / connection count / FD count
|
||||
/// every 5s, and exposes the time series + sentinel "did the SUT exit" flag.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>The fixture is class-scoped (xUnit <see cref="IClassFixture{TFixture}"/>)
|
||||
/// and shared across all three NFT-RES-LIM-01/02/03 tests so the 5-minute
|
||||
/// window runs once per CI invocation.</para>
|
||||
/// <para>Disabled when <c>COMPOSE_RESTART_ENABLED != 1</c> or docker CLI
|
||||
/// is missing. Disabled state is observable via <see cref="Enabled"/>; tests
|
||||
/// must call <see cref="Xunit.Skip.IfNot(bool, string)"/> at the top of the
|
||||
/// method body — initialising the fixture without docker would throw inside
|
||||
/// <see cref="InitializeAsync"/> and surface as a hard failure instead of
|
||||
/// the explicit skip the spec requires.</para>
|
||||
/// </remarks>
|
||||
public sealed class SteadyStateLoadFixture : IAsyncLifetime
|
||||
{
|
||||
public const int SampleIntervalSeconds = 5;
|
||||
public const int LoadDurationSeconds = 300;
|
||||
public const int TargetRps = 50;
|
||||
public const string ContainerName = "missions-sut";
|
||||
|
||||
public bool Enabled =>
|
||||
Environment.GetEnvironmentVariable("COMPOSE_RESTART_ENABLED") == "1";
|
||||
|
||||
public bool LoadGeneratorMetTargetRps { get; private set; }
|
||||
public bool SutExitedDuringWindow { get; private set; }
|
||||
public string? SkipReason { get; private set; }
|
||||
|
||||
public List<long> RssBytesSamples { get; } = new();
|
||||
public List<int> NpgsqlConnectionSamples { get; } = new();
|
||||
public List<int> FileDescriptorSamples { get; } = new();
|
||||
public List<DateTime> SampleTimestamps { get; } = new();
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
if (!Enabled)
|
||||
{
|
||||
SkipReason = "COMPOSE_RESTART_ENABLED!=1 — docker CLI primitives unavailable";
|
||||
return;
|
||||
}
|
||||
if (!CommandAvailable("docker"))
|
||||
{
|
||||
SkipReason = "docker CLI not on PATH in this consumer image";
|
||||
return;
|
||||
}
|
||||
|
||||
using var http = new HttpClient { BaseAddress = new Uri(TestEnvironment.MissionsBaseUrl) };
|
||||
var token = await new TokenMinter(TestEnvironment.JwksMockBaseUrl + "/sign").MintDefaultAsync();
|
||||
http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.Jwt);
|
||||
|
||||
var cancel = new CancellationTokenSource();
|
||||
var endpoints = new[] { "/vehicles", "/missions", "/missions?page=1&pageSize=20" };
|
||||
long requestsSent = 0;
|
||||
|
||||
var loadTask = Task.Run(async () =>
|
||||
{
|
||||
var ix = 0;
|
||||
while (!cancel.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = endpoints[ix++ % endpoints.Length];
|
||||
using var resp = await http.GetAsync(url, cancel.Token);
|
||||
Interlocked.Increment(ref requestsSent);
|
||||
}
|
||||
catch (HttpRequestException) { /* surfaces via SutExitedDuringWindow below */ }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}, cancel.Token);
|
||||
|
||||
var samplingDeadline = DateTime.UtcNow.AddSeconds(LoadDurationSeconds);
|
||||
while (DateTime.UtcNow < samplingDeadline)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(SampleIntervalSeconds), cancel.Token);
|
||||
if (!ContainerIsRunning(ContainerName))
|
||||
{
|
||||
SutExitedDuringWindow = true;
|
||||
break;
|
||||
}
|
||||
SampleTimestamps.Add(DateTime.UtcNow);
|
||||
RssBytesSamples.Add(ReadRssBytes(ContainerName));
|
||||
NpgsqlConnectionSamples.Add(ReadNpgsqlConnectionCount());
|
||||
FileDescriptorSamples.Add(ReadFileDescriptorCount(ContainerName));
|
||||
}
|
||||
|
||||
cancel.Cancel();
|
||||
try { await loadTask; } catch (OperationCanceledException) { }
|
||||
|
||||
// Sustained 50 RPS over 300s = 15000 requests; allow 10% slack for
|
||||
// CI variance / connection-refused retries.
|
||||
LoadGeneratorMetTargetRps =
|
||||
requestsSent >= (long)(TargetRps * LoadDurationSeconds * 0.9);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
private static long ReadRssBytes(string containerName)
|
||||
{
|
||||
// `docker stats --no-stream --format '{{.MemUsage}}'` prints e.g.
|
||||
// "187.4MiB / 7.7GiB". We need the LHS in bytes.
|
||||
var raw = Run("docker",
|
||||
$"stats --no-stream --format '{{{{.MemUsage}}}}' {containerName}");
|
||||
var lhs = raw.Split('/')[0].Trim().Trim('\'');
|
||||
return ParseHumanBytes(lhs);
|
||||
}
|
||||
|
||||
private static int ReadNpgsqlConnectionCount()
|
||||
{
|
||||
using var conn = new NpgsqlConnection(TestEnvironment.DbSideChannel);
|
||||
conn.Open();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
SELECT count(*)::INTEGER FROM pg_stat_activity
|
||||
WHERE application_name LIKE 'Npgsql%'
|
||||
OR (usename = 'postgres' AND backend_type = 'client backend');
|
||||
""";
|
||||
return Convert.ToInt32(cmd.ExecuteScalar());
|
||||
}
|
||||
|
||||
private static int ReadFileDescriptorCount(string containerName)
|
||||
{
|
||||
// `pgrep` is not guaranteed in the runtime image; we walk /proc
|
||||
// directly. `/proc/1/comm` is the entrypoint process name; for the
|
||||
// ASP.NET Core SDK image this is `dotnet`.
|
||||
var stdout = Run("docker",
|
||||
$"exec {containerName} sh -c 'ls /proc/1/fd | wc -l'");
|
||||
return int.Parse(stdout.Trim(), CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static long ParseHumanBytes(string text)
|
||||
{
|
||||
// "187.4MiB" / "1.2GiB" / "234KiB" / "987B"
|
||||
var unitIx = text.IndexOfAny(new[] { 'K', 'M', 'G', 'T', 'B' });
|
||||
if (unitIx < 0) return long.Parse(text, CultureInfo.InvariantCulture);
|
||||
var num = double.Parse(text.Substring(0, unitIx), CultureInfo.InvariantCulture);
|
||||
var unit = text.Substring(unitIx);
|
||||
return unit switch
|
||||
{
|
||||
"B" => (long)num,
|
||||
"KiB" or "KB" or "K" => (long)(num * 1024),
|
||||
"MiB" or "MB" or "M" => (long)(num * 1024 * 1024),
|
||||
"GiB" or "GB" or "G" => (long)(num * 1024 * 1024 * 1024),
|
||||
"TiB" or "TB" or "T" => (long)(num * 1024L * 1024 * 1024 * 1024),
|
||||
_ => throw new FormatException($"unknown human-bytes unit in '{text}'")
|
||||
};
|
||||
}
|
||||
|
||||
private static bool ContainerIsRunning(string containerName)
|
||||
{
|
||||
try
|
||||
{
|
||||
var stdout = Run("docker",
|
||||
$"inspect --format '{{{{.State.Running}}}}' {containerName}");
|
||||
return stdout.Trim().Trim('\'').Equals("true", StringComparison.Ordinal);
|
||||
}
|
||||
catch (InvalidOperationException) { return false; }
|
||||
}
|
||||
|
||||
private static bool CommandAvailable(string command)
|
||||
{
|
||||
try
|
||||
{
|
||||
var psi = new ProcessStartInfo(command, "--version")
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
using var p = Process.Start(psi);
|
||||
if (p is null) return false;
|
||||
p.WaitForExit();
|
||||
return p.ExitCode == 0;
|
||||
}
|
||||
catch (System.ComponentModel.Win32Exception) { return false; }
|
||||
}
|
||||
|
||||
private static string Run(string file, string args)
|
||||
{
|
||||
var psi = new ProcessStartInfo(file, args)
|
||||
{
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false
|
||||
};
|
||||
using var p = Process.Start(psi)
|
||||
?? throw new InvalidOperationException($"failed to launch `{file} {args}`");
|
||||
var stdout = p.StandardOutput.ReadToEnd();
|
||||
var stderr = p.StandardError.ReadToEnd();
|
||||
p.WaitForExit();
|
||||
if (p.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"`{file} {args}` exited {p.ExitCode}: {stderr}");
|
||||
return stdout;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user