azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-23 02:46:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

add autodetection

Browse Source
This commit is contained in:
Alex Bezdieniezhnykh
2024-11-02 13:09:00 +02:00
parent b5b77d9492
commit 418a2116b7
19 changed files with 545 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Annotator/Extensions/VLCFrameExtractor.cs
+54 -49
View File
@@ -1,31 +1,27 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Azaion.Annotator.DTO;
using LibVLCSharp.Shared;
using SixLabors.ImageSharp.Drawing;
using SkiaSharp;
namespace Azaion.Annotator.Extensions;
public class VLCFrameExtractor(LibVLC libVLC, MainWindow mainWindow)
public class VLCFrameExtractor(LibVLC libVLC)
{
private const uint RGBA_BYTES = 4;
private const int PLAYBACK_RATE = 3;
private const uint DEFAULT_WIDTH = 1280;
private const int PLAYBACK_RATE = 4;
private uint _pitch; // Number of bytes per "line", aligned to x32.
private uint _lines; // Number of lines in the buffer, aligned to x32.
private uint _width; // Thumbnail width
private uint _height; // Thumbnail height
private uint _videoFPS;
private Func<Stream,Task> _frameProcessFn = null!;
private MediaPlayer _mediaPlayer = null!;
private TimeSpan _lastFrameTimestamp;
private long _lastFrame;
private static uint Align32(uint size)
{
if (size % 32 == 0)
@@ -34,76 +30,66 @@ public class VLCFrameExtractor(LibVLC libVLC, MainWindow mainWindow)
}
private static SKBitmap? _currentBitmap;
private static readonly ConcurrentQueue<SKBitmap?> FilesToProcess = new();
private static readonly ConcurrentQueue<FrameInfo> FramesQueue = new();
private static long _frameCounter;
public async Task Start(Func<Stream, Task> frameProcessFn)
public async IAsyncEnumerable<(TimeSpan Time, Stream Stream)> ExtractFrames(string mediaPath,
[EnumeratorCancellation] CancellationToken manualCancellationToken = default)
{
_frameProcessFn = frameProcessFn;
var processingCancellationTokenSource = new CancellationTokenSource();
var videoFinishedCancellationToken = new CancellationTokenSource();
_mediaPlayer = new MediaPlayer(libVLC);
_mediaPlayer.Stopped += (s, e) => processingCancellationTokenSource.CancelAfter(1);
_mediaPlayer.Stopped += (s, e) => videoFinishedCancellationToken.CancelAfter(1);
using var media = new Media(libVLC, ((MediaFileInfo)mainWindow.LvFiles.SelectedItem).Path);
await media.Parse(cancellationToken: processingCancellationTokenSource.Token);
using var media = new Media(libVLC, mediaPath);
await media.Parse(cancellationToken: videoFinishedCancellationToken.Token);
var videoTrack = media.Tracks.FirstOrDefault(x => x.Data.Video.Width != 0);
_width = videoTrack.Data.Video.Width;
_height = videoTrack.Data.Video.Height;
_videoFPS = videoTrack.Data.Video.FrameRateNum;
//rescaling to DEFAULT_WIDTH
_height = (uint)(DEFAULT_WIDTH * _height / (double)_width);
_width = DEFAULT_WIDTH;
//TODO: probably rescaling is not necessary, should be checked
//_width = DEFAULT_WIDTH;
//_height = (uint)(DEFAULT_WIDTH * _height / (double)_width);
_pitch = Align32(_width * RGBA_BYTES);
_lines = Align32(_height);
_mediaPlayer.SetRate(PLAYBACK_RATE);
media.AddOption(":no-audio");
_mediaPlayer.SetVideoFormat("RV32", _width, _height, _pitch);
_mediaPlayer.SetVideoCallbacks(Lock, null, Display);
_mediaPlayer.Play(media);
_mediaPlayer.SetRate(3);
try
{
media.AddOption(":no-audio");
_mediaPlayer.SetVideoFormat("RV32", _width, _height, _pitch);
_mediaPlayer.SetVideoCallbacks(Lock, null, Display);
await ProcessThumbnailsAsync(processingCancellationTokenSource.Token);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
_mediaPlayer.Stop();
_mediaPlayer.Dispose();
}
}
private async Task ProcessThumbnailsAsync(CancellationToken token)
{
_frameCounter = 0;
var surface = SKSurface.Create(new SKImageInfo((int) _width, (int) _height));
while (!token.IsCancellationRequested)
var token = videoFinishedCancellationToken.Token;
while (!(FramesQueue.IsEmpty && token.IsCancellationRequested) && !manualCancellationToken.IsCancellationRequested)
{
if (FilesToProcess.TryDequeue(out var bitmap))
if (FramesQueue.TryDequeue(out var frameInfo))
{
if (bitmap == null)
if (frameInfo.Bitmap == null)
continue;
surface.Canvas.DrawBitmap(bitmap, 0, 0); // Effectively crops the original bitmap to get only the visible area
surface.Canvas.DrawBitmap(frameInfo.Bitmap, 0, 0); // Effectively crops the original bitmap to get only the visible area
using var outputImage = surface.Snapshot();
using var data = outputImage.Encode(SKEncodedImageFormat.Jpeg, 85);
using var ms = new MemoryStream();
data.SaveTo(ms);
if (_frameProcessFn != null)
await _frameProcessFn(ms);
Console.WriteLine($"Time: {TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} Queue size: {FilesToProcess.Count}");
bitmap.Dispose();
yield return (frameInfo.Time, ms);
Console.WriteLine($"Queue size: {FramesQueue.Count}");
frameInfo.Bitmap?.Dispose();
}
else
{
await Task.Delay(TimeSpan.FromSeconds(1), token);
}
}
_mediaPlayer.Stop();
_mediaPlayer.Dispose();
}
@@ -116,12 +102,31 @@ public class VLCFrameExtractor(LibVLC libVLC, MainWindow mainWindow)
private void Display(IntPtr opaque, IntPtr picture)
{
if (_frameCounter % (int)(_videoFPS / 3.0) == 0)
FilesToProcess.Enqueue(_currentBitmap);
var playerTime = TimeSpan.FromMilliseconds(_mediaPlayer.Time);
if (_lastFrameTimestamp != playerTime)
{
_lastFrame = _frameCounter;
_lastFrameTimestamp = playerTime;
}
if (_frameCounter > 20 && _frameCounter % 10 == 0)
{
var msToAdd = (_frameCounter - _lastFrame) * (_lastFrameTimestamp.TotalMilliseconds / _lastFrame);
var time = _lastFrameTimestamp.Add(TimeSpan.FromMilliseconds(msToAdd));
FramesQueue.Enqueue(new FrameInfo(time, _currentBitmap));
}
else
{
_currentBitmap?.Dispose();
}
_currentBitmap = null;
_frameCounter++;
}
}
public class FrameInfo(TimeSpan time, SKBitmap? bitmap)
{
public TimeSpan Time { get; set; } = time;
public SKBitmap? Bitmap { get; set; } = bitmap;
}