make thumbnail generating multithread

fix the bug with open video add class distribution chart
2026-04-22 10:56:31 +00:00 · 2024-09-25 21:46:07 +03:00
parent 742f1ffee9
commit 22d4493d86
11 changed files with 215 additions and 50 deletions
@@ -0,0 +1,67 @@
+namespace Azaion.Annotator.Extensions;
+
+public class ParallelOptions
+{
+    public int ProgressUpdateInterval { get; set; } = 100;
+    public Func<int, Task> ProgressFn { get; set; } = null!;
+    public double CpuUtilPercent { get; set; } = 100;
+}
+
+public class ParallelExt
+{
+    public static async Task ForEachAsync<T>(ICollection<T> source,
+        Func<T, CancellationToken, ValueTask> processFn,
+        ParallelOptions? parallelOptions = null,
+        CancellationToken cancellationToken = default)
+    {
+        parallelOptions ??= new ParallelOptions
+        {
+            CpuUtilPercent = 100,
+            ProgressUpdateInterval = 100,
+            ProgressFn = i =>
+            {
+                Console.WriteLine($"Processed {i} item by Task {Task.CurrentId}%");
+                return Task.CompletedTask;
+            }
+        };
+        var threadsCount = (int)(Environment.ProcessorCount * parallelOptions.CpuUtilPercent / 100.0);
+
+        var processedCount = 0;
+        var chunkSize = Math.Max(1, (int)(source.Count / (decimal)threadsCount));
+        var chunks = source.Chunk(chunkSize).ToList();
+        if (chunks.Count > threadsCount)
+        {
+            chunks[^2] = chunks[^2].Concat(chunks.Last()).ToArray();
+            chunks.RemoveAt(chunks.Count - 1);
+        }
+        var progressUpdateLock = new SemaphoreSlim(1);
+
+        var tasks = new List<Task>();
+        foreach (var chunk in chunks)
+        {
+            tasks.Add(await Task.Factory.StartNew(async () =>
+            {
+                foreach (var item in chunk)
+                {
+                    await processFn(item, cancellationToken);
+                    Interlocked.Increment(ref processedCount);
+                    if (processedCount % parallelOptions.ProgressUpdateInterval == 0 && parallelOptions.ProgressFn != null)
+                        _ = Task.Run(async () =>
+                        {
+                            await progressUpdateLock.WaitAsync(cancellationToken);
+                            try
+                            {
+                                await parallelOptions.ProgressFn(processedCount);
+                            }
+                            finally
+                            {
+                                progressUpdateLock.Release();
+                            }
+                        }, cancellationToken);
+                }
+
+            }, TaskCreationOptions.LongRunning));
+        }
+        await Task.WhenAll(tasks);
+    }
+}