annotations/Azaion.Annotator/Extensions/VLCFrameExtractor.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Runtime.InteropServices;
using Azaion.Annotator.DTO;
using LibVLCSharp.Shared;
using SixLabors.ImageSharp.Drawing;
using SkiaSharp;

namespace Azaion.Annotator.Extensions;

public class VLCFrameExtractor(LibVLC libVLC, MainWindow mainWindow)
{
    private const uint RGBA_BYTES = 4;
    private const int PLAYBACK_RATE = 3;
    private const uint DEFAULT_WIDTH = 1280;

    private uint _pitch; // Number of bytes per "line", aligned to x32.
    private uint _lines; // Number of lines in the buffer, aligned to x32.
    private uint _width; // Thumbnail width
    private uint _height; // Thumbnail height
    private uint _videoFPS;
    private Func<Stream,Task> _frameProcessFn = null!;

    private MediaPlayer _mediaPlayer = null!;

    private static uint Align32(uint size)
    {
        if (size % 32 == 0)
            return size;
        return (size / 32 + 1) * 32;// Align on the next multiple of 32
    }

    private static SKBitmap? _currentBitmap;
    private static readonly ConcurrentQueue<SKBitmap?> FilesToProcess = new();
    private static long _frameCounter;

    public async Task Start(Func<Stream, Task> frameProcessFn)
    {
        _frameProcessFn = frameProcessFn;
        var processingCancellationTokenSource = new CancellationTokenSource();

        _mediaPlayer = new MediaPlayer(libVLC);
        _mediaPlayer.Stopped += (s, e) => processingCancellationTokenSource.CancelAfter(1);

        using var media = new Media(libVLC, ((MediaFileInfo)mainWindow.LvFiles.SelectedItem).Path);
        await media.Parse(cancellationToken: processingCancellationTokenSource.Token);
        var videoTrack = media.Tracks.FirstOrDefault(x => x.Data.Video.Width != 0);
        _width = videoTrack.Data.Video.Width;
        _height = videoTrack.Data.Video.Height;
        _videoFPS = videoTrack.Data.Video.FrameRateNum;

        //rescaling to DEFAULT_WIDTH
        _height = (uint)(DEFAULT_WIDTH * _height / (double)_width);
        _width = DEFAULT_WIDTH;

        _pitch = Align32(_width * RGBA_BYTES);
        _lines = Align32(_height);
        _mediaPlayer.Play(media);
        _mediaPlayer.SetRate(3);

        try
        {
            media.AddOption(":no-audio");
            _mediaPlayer.SetVideoFormat("RV32", _width, _height, _pitch);
            _mediaPlayer.SetVideoCallbacks(Lock, null, Display);
            await ProcessThumbnailsAsync(processingCancellationTokenSource.Token);
        }
        catch (Exception e)
        {
            Console.WriteLine(e.Message);
            _mediaPlayer.Stop();
            _mediaPlayer.Dispose();
        }
    }

    private async Task ProcessThumbnailsAsync(CancellationToken token)
    {
        _frameCounter = 0;
        var surface = SKSurface.Create(new SKImageInfo((int) _width, (int) _height));
        while (!token.IsCancellationRequested)
        {
            if (FilesToProcess.TryDequeue(out var bitmap))
            {
                if (bitmap == null)
                    continue;

                surface.Canvas.DrawBitmap(bitmap, 0, 0); // Effectively crops the original bitmap to get only the visible area

                using var outputImage = surface.Snapshot();
                using var data = outputImage.Encode(SKEncodedImageFormat.Jpeg, 85);
                using var ms = new MemoryStream();
                data.SaveTo(ms);
                if (_frameProcessFn != null)
                    await _frameProcessFn(ms);

                Console.WriteLine($"Time: {TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} Queue size: {FilesToProcess.Count}");
                bitmap.Dispose();
            }
            else
            {
                await Task.Delay(TimeSpan.FromSeconds(1), token);
            }
        }
        _mediaPlayer.Dispose();
    }

    private IntPtr Lock(IntPtr opaque, IntPtr planes)
    {
        _currentBitmap = new SKBitmap(new SKImageInfo((int)(_pitch / RGBA_BYTES), (int)_lines, SKColorType.Bgra8888));
        Marshal.WriteIntPtr(planes, _currentBitmap.GetPixels());
        return IntPtr.Zero;
    }

    private void Display(IntPtr opaque, IntPtr picture)
    {
        if (_frameCounter % (int)(_videoFPS / 3.0) == 0)
            FilesToProcess.Enqueue(_currentBitmap);
        else
            _currentBitmap?.Dispose();

        _currentBitmap = null;
        _frameCounter++;
    }
}