add manual Tile Processor

zoom on video on pause (temp image)
This commit is contained in:
Oleksandr Bezdieniezhnykh
2025-07-28 12:39:52 +03:00
parent fefd054ea0
commit fc6e5db795
34 changed files with 716 additions and 209 deletions
@@ -1,4 +1,5 @@
using System.IO;
using System.Diagnostics;
using System.IO;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.Extensions;
@@ -11,9 +12,10 @@ namespace Azaion.Common;
public class Constants
{
public const string CONFIG_PATH = "config.json";
private const string DEFAULT_API_URL = "https://api.azaion.com";
public const string LOADER_CONFIG_PATH = "loaderconfig.json";
public const string DEFAULT_API_URL = "https://api.azaion.com";
public const string AZAION_SUITE_EXE = "Azaion.Suite.exe";
#region ExternalClientsConfig
private const string DEFAULT_ZMQ_LOADER_HOST = "127.0.0.1";
@@ -103,14 +105,16 @@ public class Constants
TrackingDistanceConfidence = TRACKING_DISTANCE_CONFIDENCE,
TrackingProbabilityIncrease = TRACKING_PROBABILITY_INCREASE,
TrackingIntersectionThreshold = TRACKING_INTERSECTION_THRESHOLD,
BigImageTileOverlapPercent = DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT,
FramePeriodRecognition = DEFAULT_FRAME_PERIOD_RECOGNITION
};
public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2;
public const double TRACKING_DISTANCE_CONFIDENCE = 0.15;
public const double TRACKING_PROBABILITY_INCREASE = 15;
public const double TRACKING_INTERSECTION_THRESHOLD = 0.8;
public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4;
public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2;
public const double TRACKING_DISTANCE_CONFIDENCE = 0.15;
public const double TRACKING_PROBABILITY_INCREASE = 15;
public const double TRACKING_INTERSECTION_THRESHOLD = 0.8;
public const int DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT = 20;
public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4;
# endregion AIRecognitionConfig
@@ -251,4 +255,12 @@ public class Constants
return DefaultInitConfig;
}
}
public static Version GetLocalVersion()
{
var localFileInfo = FileVersionInfo.GetVersionInfo(AZAION_SUITE_EXE);
if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion))
throw new Exception($"Can't find {AZAION_SUITE_EXE} and its version");
return new Version(localFileInfo.FileVersion!);
}
}
+16 -9
View File
@@ -6,6 +6,7 @@ using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Shapes;
using Azaion.Common.DTO;
using Azaion.Common.Events;
using MediatR;
using Color = System.Windows.Media.Color;
using Image = System.Windows.Controls.Image;
@@ -34,10 +35,10 @@ public class CanvasEditor : Canvas
private Point _panStartPoint;
private bool _isZoomedIn;
private const int MIN_SIZE = 20;
private const int MIN_SIZE = 12;
private readonly TimeSpan _viewThreshold = TimeSpan.FromMilliseconds(400);
private Image _backgroundImage { get; set; } = new() { Stretch = Stretch.Fill };
public Image BackgroundImage { get; set; } = new() { Stretch = Stretch.Uniform };
public IMediator Mediator { get; set; } = null!;
public static readonly DependencyProperty GetTimeFuncProp =
@@ -113,7 +114,7 @@ public class CanvasEditor : Canvas
MouseUp += CanvasMouseUp;
SizeChanged += CanvasResized;
Cursor = Cursors.Cross;
Children.Insert(0, _backgroundImage);
Children.Insert(0, BackgroundImage);
Children.Add(_newAnnotationRect);
Children.Add(_horizontalLine);
Children.Add(_verticalLine);
@@ -127,7 +128,7 @@ public class CanvasEditor : Canvas
public void SetBackground(ImageSource? source)
{
SetZoom();
_backgroundImage.Source = source;
BackgroundImage.Source = source;
}
private void SetZoom(Matrix? matrix = null)
@@ -142,8 +143,8 @@ public class CanvasEditor : Canvas
_matrixTransform.Matrix = matrix.Value;
_isZoomedIn = true;
}
foreach (var detection in CurrentDetections)
detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11);
// foreach (var detection in CurrentDetections)
// detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11);
}
private void CanvasWheel(object sender, MouseWheelEventArgs e)
@@ -175,6 +176,8 @@ public class CanvasEditor : Canvas
private void CanvasMouseDown(object sender, MouseButtonEventArgs e)
{
ClearSelections();
if (e.LeftButton != MouseButtonState.Pressed)
return;
if (Keyboard.Modifiers == ModifierKeys.Control && _isZoomedIn)
{
_panStartPoint = e.GetPosition(this);
@@ -182,11 +185,13 @@ public class CanvasEditor : Canvas
}
else
NewAnnotationStart(sender, e);
(sender as UIElement)?.CaptureMouse();
}
private void CanvasMouseMove(object sender, MouseEventArgs e)
{
var pos = e.GetPosition(this);
Mediator.Publish(new SetStatusTextEvent($"Mouse Coordinates: {pos.X}, {pos.Y}"));
_horizontalLine.Y1 = _horizontalLine.Y2 = pos.Y;
_verticalLine.X1 = _verticalLine.X2 = pos.X;
SetLeft(_classNameHint, pos.X + 10);
@@ -216,11 +221,14 @@ public class CanvasEditor : Canvas
var matrix = _matrixTransform.Matrix;
matrix.Translate(delta.X, delta.Y);
_matrixTransform.Matrix = matrix;
Mediator.Publish(new SetStatusTextEvent(_matrixTransform.Matrix.ToString()));
}
private void CanvasMouseUp(object sender, MouseButtonEventArgs e)
{
(sender as UIElement)?.ReleaseMouseCapture();
if (SelectionState == SelectionState.NewAnnCreating)
{
var endPos = e.GetPosition(this);
@@ -279,8 +287,8 @@ public class CanvasEditor : Canvas
{
_horizontalLine.X2 = e.NewSize.Width;
_verticalLine.Y2 = e.NewSize.Height;
_backgroundImage.Width = e.NewSize.Width;
_backgroundImage.Height = e.NewSize.Height;
BackgroundImage.Width = e.NewSize.Width;
BackgroundImage.Height = e.NewSize.Height;
}
#region Annotation Resizing & Moving
@@ -383,7 +391,6 @@ public class CanvasEditor : Canvas
private void NewAnnotationStart(object sender, MouseButtonEventArgs e)
{
_newAnnotationStartPos = e.GetPosition(this);
SetLeft(_newAnnotationRect, _newAnnotationStartPos.X);
SetTop(_newAnnotationRect, _newAnnotationStartPos.Y);
_newAnnotationRect.MouseMove += NewAnnotationCreatingMove;
+19 -23
View File
@@ -12,15 +12,15 @@ namespace Azaion.Common.Controls;
public class DetectionControl : Border
{
private readonly Action<object, MouseButtonEventArgs> _resizeStart;
private const double RESIZE_RECT_SIZE = 12;
private const double RESIZE_RECT_SIZE = 10;
private readonly Grid _grid;
private readonly Label _detectionLabel;
private readonly DetectionLabelPanel _detectionLabelPanel;
//private readonly Label _detectionLabel;
public readonly Canvas DetectionLabelContainer;
public TimeSpan Time { get; set; }
private readonly double _confidence;
private List<Rectangle> _resizedRectangles = new();
private readonly List<Rectangle> _resizedRectangles = new();
private DetectionClass _detectionClass = null!;
public DetectionClass DetectionClass
@@ -33,9 +33,8 @@ public class DetectionControl : Border
BorderThickness = new Thickness(3);
foreach (var rect in _resizedRectangles)
rect.Stroke = brush;
_detectionLabel.Background = new SolidColorBrush(value.Color.ToConfidenceColor(_confidence));
_detectionLabel.Content = _detectionLabelText(value.UIName);
_detectionLabelPanel.DetectionClass = value;
_detectionClass = value;
}
}
@@ -78,10 +77,7 @@ public class DetectionControl : Border
DetectionLabelContainer.VerticalAlignment = value.Vertical;
}
}
private string _detectionLabelText(string detectionClassName) =>
_confidence >= 0.995 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%"; //double
public DetectionControl(DetectionClass detectionClass, TimeSpan time, Action<object,
MouseButtonEventArgs> resizeStart, CanvasLabel canvasLabel)
{
@@ -89,7 +85,6 @@ public class DetectionControl : Border
Height = canvasLabel.Height;
Time = time;
_resizeStart = resizeStart;
_confidence = canvasLabel.Confidence;
DetectionLabelContainer = new Canvas
{
@@ -97,16 +92,16 @@ public class DetectionControl : Border
VerticalAlignment = VerticalAlignment.Top,
ClipToBounds = false,
};
_detectionLabel = new Label
_detectionLabelPanel = new DetectionLabelPanel
{
Content = _detectionLabelText(detectionClass.Name),
FontSize = 16,
Visibility = Visibility.Visible
Confidence = canvasLabel.Confidence
};
DetectionLabelContainer.Children.Add(_detectionLabel);
DetectionLabelContainer.Children.Add(_detectionLabelPanel);
_selectionFrame = new Rectangle
{
Margin = new Thickness(-3),
HorizontalAlignment = HorizontalAlignment.Stretch,
VerticalAlignment = VerticalAlignment.Stretch,
Stroke = new SolidColorBrush(Colors.Black),
@@ -146,12 +141,13 @@ public class DetectionControl : Border
var rect = new Rectangle() // small rectangles at the corners and sides
{
ClipToBounds = false,
Margin = new Thickness(-RESIZE_RECT_SIZE * 0.7),
Margin = new Thickness(-RESIZE_RECT_SIZE),
HorizontalAlignment = ha,
VerticalAlignment = va,
Width = RESIZE_RECT_SIZE,
Height = RESIZE_RECT_SIZE,
Stroke = new SolidColorBrush(Color.FromArgb(230, 20, 20, 20)), // small rectangles color
StrokeThickness = 0.8,
Fill = new SolidColorBrush(Color.FromArgb(150, 80, 80, 80)),
Cursor = crs,
Name = name,
@@ -160,9 +156,9 @@ public class DetectionControl : Border
return rect;
}
public YoloLabel GetLabel(Size canvasSize, Size? videoSize = null)
{
var label = new CanvasLabel(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height);
return new YoloLabel(label, canvasSize, videoSize);
}
public CanvasLabel ToCanvasLabel() =>
new(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height);
public YoloLabel ToYoloLabel(Size canvasSize, Size? videoSize = null) =>
new(ToCanvasLabel(), canvasSize, videoSize);
}
@@ -0,0 +1,59 @@
<UserControl x:Class="Azaion.Common.Controls.DetectionLabelPanel"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
mc:Ignorable="d">
<UserControl.Resources>
<!-- Friendly (Light Blue Square) -->
<DrawingImage x:Key="Friendly">
<DrawingImage.Drawing>
<DrawingGroup ClipGeometry="M0,0 V320 H320 V0 H0 Z">
<GeometryDrawing Brush="LightBlue" Geometry="M25,50 l150,0 0,100 -150,0 z">
<GeometryDrawing.Pen>
<Pen Brush="Black" Thickness="8"/>
</GeometryDrawing.Pen>
</GeometryDrawing>
</DrawingGroup>
</DrawingImage.Drawing>
</DrawingImage>
<!-- Hostile (Red Diamond) -->
<DrawingImage x:Key="Hostile">
<DrawingImage.Drawing>
<DrawingGroup ClipGeometry="M0,0 V320 H320 V0 H0 Z">
<GeometryDrawing Brush="Red" Geometry="M 100,28 L172,100 100,172 28,100 100,28 Z">
<GeometryDrawing.Pen>
<Pen Brush="Black" Thickness="8"/>
</GeometryDrawing.Pen>
</GeometryDrawing>
</DrawingGroup>
</DrawingImage.Drawing>
</DrawingImage>
<!-- Unknown (Yellow Quatrefoil) -->
<DrawingImage x:Key="Unknown">
<DrawingImage.Drawing>
<DrawingGroup ClipGeometry="M0,0 V320 H320 V0 H0 Z">
<GeometryDrawing Brush="Yellow" Geometry="M63,63 C63,20 137,20 137,63 C180,63 180,137 137,137 C137,180
63,180 63,137 C20,137 20,63 63,63 Z">
<GeometryDrawing.Pen>
<Pen Brush="Black" Thickness="8"/>
</GeometryDrawing.Pen>
</GeometryDrawing>
</DrawingGroup>
</DrawingImage.Drawing>
</DrawingImage>
</UserControl.Resources>
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="28"></ColumnDefinition>
<ColumnDefinition Width="Auto"></ColumnDefinition>
</Grid.ColumnDefinitions>
<Image Grid.Column="0" x:Name="AffiliationImage">
</Image>
<Label Grid.Column="1" FontSize="16"></Label>
</Grid>
</UserControl>
@@ -0,0 +1,55 @@
using System.Windows.Media;
using Azaion.Common.DTO;
namespace Azaion.Common.Controls
{
public partial class DetectionLabelPanel
{
private AffiliationEnum _affiliation = AffiliationEnum.None;
private double _confidence;
public AffiliationEnum Affiliation
{
get => _affiliation;
set
{
_affiliation = value;
UpdateAffiliationImage();
}
}
public DetectionClass DetectionClass { get; set; }
public double Confidence
{
get => _confidence;
set
{
_confidence = value;
}
}
public DetectionLabelPanel()
{
InitializeComponent();
}
private string _detectionLabelText(string detectionClassName) =>
_confidence >= 0.98 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%";
private void UpdateAffiliationImage()
{
if (_affiliation == AffiliationEnum.None)
{
AffiliationImage.Source = null;
return;
}
if (TryFindResource(_affiliation.ToString()) is DrawingImage drawingImage)
AffiliationImage.Source = drawingImage;
else
AffiliationImage.Source = null;
}
}
}
+9
View File
@@ -0,0 +1,9 @@
namespace Azaion.Common.DTO;
public enum AffiliationEnum
{
None = 0,
Friendly = 10,
Hostile = 20,
Unknown = 30
}
@@ -12,6 +12,7 @@ public class AIRecognitionConfig
[Key("t_dc")] public double TrackingDistanceConfidence { get; set; }
[Key("t_pi")] public double TrackingProbabilityIncrease { get; set; }
[Key("t_it")] public double TrackingIntersectionThreshold { get; set; }
[Key("ov_p")] public double BigImageTileOverlapPercent { get; set; }
[Key("d")] public byte[] Data { get; set; } = null!;
[Key("p")] public List<string> Paths { get; set; } = null!;
+2 -2
View File
@@ -6,10 +6,10 @@ namespace Azaion.Common.DTO;
public class FormState
{
public MediaFileInfo? CurrentMedia { get; set; }
public string VideoName => CurrentMedia?.FName ?? "";
public string MediaName => CurrentMedia?.FName ?? "";
public string CurrentMrl { get; set; } = null!;
public Size CurrentVideoSize { get; set; }
public Size CurrentMediaSize { get; set; }
public TimeSpan CurrentVideoLength { get; set; }
public TimeSpan? BackgroundTime { get; set; }
+34 -5
View File
@@ -22,16 +22,36 @@ public abstract class Label
public class CanvasLabel : Label
{
public double X { get; set; }
public double Y { get; set; }
public double X { get; set; } //left
public double Y { get; set; } //top
public double Width { get; set; }
public double Height { get; set; }
public double Confidence { get; set; }
public CanvasLabel()
public double Bottom
{
get => Y + Height;
set => Height = value - Y;
}
public double Right
{
get => X + Width;
set => Width = value - X;
}
public CanvasLabel() { }
public CanvasLabel(double left, double right, double top, double bottom)
{
X = left;
Y = top;
Width = right - left;
Height = bottom - top;
Confidence = 1;
ClassNumber = -1;
}
public CanvasLabel(int classNumber, double x, double y, double width, double height, double confidence = 1) : base(classNumber)
{
X = x;
@@ -77,6 +97,13 @@ public class CanvasLabel : Label
}
Confidence = confidence;
}
public CanvasLabel ReframeToSmall(CanvasLabel smallTile) =>
new(ClassNumber, X - smallTile.X, Y - smallTile.Y, Width, Height, Confidence);
public CanvasLabel ReframeFromSmall(CanvasLabel smallTile) =>
new(ClassNumber, X + smallTile.X, Y + smallTile.Y, Width, Height, Confidence);
}
[MessagePackObject]
@@ -193,13 +220,15 @@ public class Detection : YoloLabel
{
[JsonProperty(PropertyName = "an")][Key("an")] public string AnnotationName { get; set; } = null!;
[JsonProperty(PropertyName = "p")][Key("p")] public double Confidence { get; set; }
[JsonProperty(PropertyName = "dn")][Key("dn")] public string Description { get; set; }
//For db & serialization
public Detection(){}
public Detection(string annotationName, YoloLabel label, double confidence = 1)
public Detection(string annotationName, YoloLabel label, string description = "", double confidence = 1)
{
AnnotationName = annotationName;
Description = description;
ClassNumber = label.ClassNumber;
CenterX = label.CenterX;
CenterY = label.CenterY;
+3 -1
View File
@@ -60,8 +60,10 @@ public class DbFactory : IDbFactory
if (!File.Exists(_annConfig.AnnotationsDbFile))
SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile);
RecreateTables();
_fileConnection.Open();
using var db = new AnnotationsDb(_fileDataOptions);
SchemaMigrator.EnsureSchemaUpdated(db, typeof(Annotation), typeof(Detection));
_fileConnection.BackupDatabase(_memoryConnection, "main", "main", -1, null, -1);
}
+94
View File
@@ -0,0 +1,94 @@
using System.Data;
using LinqToDB.Data;
using LinqToDB.Mapping;
namespace Azaion.Common.Database;
public static class SchemaMigrator
{
public static void EnsureSchemaUpdated(DataConnection dbConnection, params Type[] entityTypes)
{
var connection = dbConnection.Connection;
var mappingSchema = dbConnection.MappingSchema;
if (connection.State == ConnectionState.Closed)
{
connection.Open();
}
foreach (var type in entityTypes)
{
var entityDescriptor = mappingSchema.GetEntityDescriptor(type);
var tableName = entityDescriptor.Name.Name;
var existingColumns = GetTableColumns(connection, tableName);
foreach (var column in entityDescriptor.Columns)
{
if (existingColumns.Contains(column.ColumnName, StringComparer.OrdinalIgnoreCase))
continue;
var columnDefinition = GetColumnDefinition(column);
dbConnection.Execute($"ALTER TABLE {tableName} ADD COLUMN {columnDefinition}");
}
}
}
private static HashSet<string> GetTableColumns(IDbConnection connection, string tableName)
{
var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
using var cmd = connection.CreateCommand();
cmd.CommandText = $"PRAGMA table_info({tableName})";
using var reader = cmd.ExecuteReader();
while (reader.Read())
columns.Add(reader.GetString(1)); // "name" is in the second column
return columns;
}
private static string GetColumnDefinition(ColumnDescriptor column)
{
var type = column.MemberType;
var underlyingType = Nullable.GetUnderlyingType(type) ?? type;
var sqliteType = GetSqliteType(underlyingType);
var defaultClause = GetSqlDefaultValue(type, underlyingType);
return $"\"{column.ColumnName}\" {sqliteType} {defaultClause}";
}
private static string GetSqliteType(Type type) =>
type switch
{
_ when type == typeof(int)
|| type == typeof(long)
|| type == typeof(bool)
|| type.IsEnum
=> "INTEGER",
_ when type == typeof(double)
|| type == typeof(float)
|| type == typeof(decimal)
=> "REAL",
_ when type == typeof(byte[])
=> "BLOB",
_ => "TEXT"
};
private static string GetSqlDefaultValue(Type originalType, Type underlyingType)
{
var isNullable = originalType.IsClass || Nullable.GetUnderlyingType(originalType) != null;
if (isNullable)
return "NULL";
var defaultValue = Activator.CreateInstance(underlyingType);
if (underlyingType == typeof(bool))
return $"NOT NULL DEFAULT {(Convert.ToBoolean(defaultValue) ? 1 : 0)}";
if (underlyingType.IsValueType && defaultValue is IFormattable f)
return $"NOT NULL DEFAULT {f.ToString(null, System.Globalization.CultureInfo.InvariantCulture)}";
return $"NOT NULL DEFAULT '{defaultValue}'";
}
}
+82
View File
@@ -0,0 +1,82 @@
using System.Windows;
using System.Windows.Media.Imaging;
using Azaion.Common.DTO;
namespace Azaion.Common.Services;
public class TileResult
{
public CanvasLabel Tile { get; set; }
public List<CanvasLabel> Detections { get; set; }
public TileResult(CanvasLabel tile, List<CanvasLabel> detections)
{
Tile = tile;
Detections = detections;
}
}
public static class TileProcessor
{
private const int MaxTileWidth = 1280;
private const int MaxTileHeight = 1280;
private const int Border = 10;
public static List<TileResult> Split(Size originalSize, List<CanvasLabel> detections, CancellationToken cancellationToken)
{
var results = new List<TileResult>();
var processingDetectionList = new List<CanvasLabel>(detections);
while (processingDetectionList.Count > 0 && !cancellationToken.IsCancellationRequested)
{
var topMostDetection = processingDetectionList
.OrderBy(d => d.Y)
.First();
var result = GetDetectionsInTile(originalSize, topMostDetection, processingDetectionList);
processingDetectionList.RemoveAll(x => result.Detections.Contains(x));
results.Add(result);
}
return results;
}
private static TileResult GetDetectionsInTile(Size originalSize, CanvasLabel startDet, List<CanvasLabel> allDetections)
{
var tile = new CanvasLabel(
left: Math.Max(startDet.X - Border, 0),
right: Math.Min(startDet.Right + Border, originalSize.Width),
top: Math.Max(startDet.Y - Border, 0),
bottom: Math.Min(startDet.Bottom + Border, originalSize.Height));
var selectedDetections = new List<CanvasLabel>{startDet};
foreach (var det in allDetections)
{
if (det == startDet)
continue;
var commonTile = new CanvasLabel(
left: Math.Max(Math.Min(tile.X, det.X) - Border, 0),
right: Math.Min(Math.Max(tile.Right, det.Right) + Border, originalSize.Width),
top: Math.Max(Math.Min(tile.Y, det.Y) - Border, 0),
bottom: Math.Min(Math.Max(tile.Bottom, det.Bottom) + Border, originalSize.Height)
);
if (commonTile.Width > MaxTileWidth || commonTile.Height > MaxTileHeight)
continue;
tile = commonTile;
selectedDetections.Add(det);
}
//normalization, width and height should be at least half of 1280px
tile.Width = Math.Max(tile.Width, MaxTileWidth / 2.0);
tile.Height = Math.Max(tile.Height, MaxTileHeight / 2.0);
//boundaries check after normalization
tile.Right = Math.Min(tile.Right, originalSize.Width);
tile.Bottom = Math.Min(tile.Bottom, originalSize.Height);
return new TileResult(tile, selectedDetections);
}
}