From 7807f5bc90131d022e5a8175a62c7c63286c2d06 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Fri, 26 Jul 2024 14:11:29 +0300 Subject: [PATCH] add quartz for jobs configure auto-scan folder and create hls files job WIP --- .../Azaion.Repository.csproj | 2 + Web/Azaion.Web/Azaion.Repository/Constants.cs | 3 -- .../DTO/Configs/ConnectionStrings.cs | 6 +++ .../DTO/Configs/FoldersConfig.cs | 8 ++++ .../DTO/ConnectionStrings.cs | 7 ---- .../Azaion.Repository/Entities/Media.cs | 12 ++++-- .../Extensions/QuartzJobConfigHelper.cs | 27 +++++++++++++ .../Azaion.Repository/Jobs/BaseJob.cs | 28 ++++++++++++++ .../Azaion.Repository/sql/01 initial.sql | 15 +++++--- .../Azaion.Video/Azaion.Video.csproj | 4 ++ Web/Azaion.Web/Azaion.Video/IFfmpegManager.cs | 38 +++++++++++++++++++ .../{IVideoManager.cs => IVideoRepository.cs} | 2 +- .../{VideoManager.cs => VideoRepository.cs} | 6 ++- .../Azaion.WebService.csproj | 2 + .../Controllers/VideoController.cs | 11 ++++-- Web/Azaion.Web/Azaion.WebService/Program.cs | 21 +++++++++- .../appsettings.Development.json | 5 +++ .../Azaion.WebService/appsettings.json | 10 ++++- 18 files changed, 179 insertions(+), 28 deletions(-) create mode 100644 Web/Azaion.Web/Azaion.Repository/DTO/Configs/ConnectionStrings.cs create mode 100644 Web/Azaion.Web/Azaion.Repository/DTO/Configs/FoldersConfig.cs delete mode 100644 Web/Azaion.Web/Azaion.Repository/DTO/ConnectionStrings.cs create mode 100644 Web/Azaion.Web/Azaion.Repository/Extensions/QuartzJobConfigHelper.cs create mode 100644 Web/Azaion.Web/Azaion.Repository/Jobs/BaseJob.cs create mode 100644 Web/Azaion.Web/Azaion.Video/IFfmpegManager.cs rename Web/Azaion.Web/Azaion.Video/{IVideoManager.cs => IVideoRepository.cs} (97%) rename Web/Azaion.Web/Azaion.Video/{VideoManager.cs => VideoRepository.cs} (86%) diff --git a/Web/Azaion.Web/Azaion.Repository/Azaion.Repository.csproj b/Web/Azaion.Web/Azaion.Repository/Azaion.Repository.csproj index 51e0b98..526d80f 100644 --- a/Web/Azaion.Web/Azaion.Repository/Azaion.Repository.csproj +++ b/Web/Azaion.Web/Azaion.Repository/Azaion.Repository.csproj @@ -8,7 +8,9 @@ + + diff --git a/Web/Azaion.Web/Azaion.Repository/Constants.cs b/Web/Azaion.Web/Azaion.Repository/Constants.cs index a4a3b08..0c89627 100644 --- a/Web/Azaion.Web/Azaion.Repository/Constants.cs +++ b/Web/Azaion.Web/Azaion.Repository/Constants.cs @@ -2,9 +2,6 @@ namespace Azaion.Repository; public class Constants { - public const string MEDIA_HLS_FOLDER = "/azaion-media/hls"; public const string M3_U8_EXT = "m3u8"; public const string TS_EXT = "ts"; - public const string FFMPEG_FILE = "/opt/homebrew/bin/ffmpeg"; - } \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Repository/DTO/Configs/ConnectionStrings.cs b/Web/Azaion.Web/Azaion.Repository/DTO/Configs/ConnectionStrings.cs new file mode 100644 index 0000000..4ea3030 --- /dev/null +++ b/Web/Azaion.Web/Azaion.Repository/DTO/Configs/ConnectionStrings.cs @@ -0,0 +1,6 @@ +namespace Azaion.Repository.DTO.Configs; + +public class ConnectionStrings +{ + public string? AzaionDb { get; set; } +} \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Repository/DTO/Configs/FoldersConfig.cs b/Web/Azaion.Web/Azaion.Repository/DTO/Configs/FoldersConfig.cs new file mode 100644 index 0000000..5301d80 --- /dev/null +++ b/Web/Azaion.Web/Azaion.Repository/DTO/Configs/FoldersConfig.cs @@ -0,0 +1,8 @@ +namespace Azaion.Repository.DTO.Configs; + +public class FoldersConfig +{ + public string VideosFolder { get; set; } = null!; + public string HlsFolder { get; set; } = null!; + public string FfmpegExecutable { get; set; } = null!; +} \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Repository/DTO/ConnectionStrings.cs b/Web/Azaion.Web/Azaion.Repository/DTO/ConnectionStrings.cs deleted file mode 100644 index 5cb592a..0000000 --- a/Web/Azaion.Web/Azaion.Repository/DTO/ConnectionStrings.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Azaion.Repository.DTO; - -public class ConnectionStrings -{ - public string? FraudDb { get; set; } - public string? FraudDbMsSql { get; set; } -} \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Repository/Entities/Media.cs b/Web/Azaion.Web/Azaion.Repository/Entities/Media.cs index bb759f0..aef5f79 100644 --- a/Web/Azaion.Web/Azaion.Repository/Entities/Media.cs +++ b/Web/Azaion.Web/Azaion.Repository/Entities/Media.cs @@ -1,4 +1,6 @@ +using Azaion.Repository.DTO.Configs; using Azaion.Video.DTO; +using Microsoft.Extensions.Options; using IOPath = System.IO.Path; namespace Azaion.Repository.Entities; @@ -15,8 +17,12 @@ public class Media private string MediaName => IOPath.GetFileNameWithoutExtension(Path); - private string OutDir => Directory.CreateDirectory(IOPath.Combine(Constants.MEDIA_HLS_FOLDER, MediaName)).FullName; + private string OutDir(IOptions config) => + Directory.CreateDirectory(IOPath.Combine(config.Value.HlsFolder, MediaName)).FullName; - public string M3U8File => IOPath.Combine(OutDir, $"{MediaName}.{Constants.M3_U8_EXT}"); - public string SegmentFile => IOPath.Combine(OutDir, $"{MediaName}%03d.{Constants.TS_EXT}"); + public string M3U8File(IOptions config) => + IOPath.Combine(OutDir(config), $"{MediaName}.{Constants.M3_U8_EXT}"); + + public string SegmentFile(IOptions config) => + IOPath.Combine(OutDir(config), $"{MediaName}%03d.{Constants.TS_EXT}"); } diff --git a/Web/Azaion.Web/Azaion.Repository/Extensions/QuartzJobConfigHelper.cs b/Web/Azaion.Web/Azaion.Repository/Extensions/QuartzJobConfigHelper.cs new file mode 100644 index 0000000..6ed687b --- /dev/null +++ b/Web/Azaion.Web/Azaion.Repository/Extensions/QuartzJobConfigHelper.cs @@ -0,0 +1,27 @@ +using Azaion.Repository.Jobs; +using Quartz; + +namespace Azaion.Repository.Extensions +{ + public static class QuartzJobConfigHelper + { + public static IServiceCollectionQuartzConfigurator RegisterJob(this IServiceCollectionQuartzConfigurator q, + string cron, TimeSpan? startForDebug = null) where T: BaseJob + { + var jobName = typeof(T).Name; + var jobKey = new JobKey(jobName); + q.AddJob(jobKey); + q.AddTrigger(t => t.WithIdentity($"{jobName}_TRIGGER") + .ForJob(jobKey) +#if DEBUG + .StartAt(DateTimeOffset.Now.Add(startForDebug ?? TimeSpan.FromHours(10))) + .WithSimpleSchedule(b => b.WithIntervalInHours(5)) +#else + .StartAt(DateTimeOffset.Now.AddMinutes(1)) + .WithCronSchedule(cron, builder => builder.InTimeZone(TimeZoneInfo.Utc)) +#endif + ); + return q; + } + } +} \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Repository/Jobs/BaseJob.cs b/Web/Azaion.Web/Azaion.Repository/Jobs/BaseJob.cs new file mode 100644 index 0000000..96258c4 --- /dev/null +++ b/Web/Azaion.Web/Azaion.Repository/Jobs/BaseJob.cs @@ -0,0 +1,28 @@ +using Microsoft.Extensions.Logging; +using Quartz; + +namespace Azaion.Repository.Jobs +{ + [DisallowConcurrentExecution] + public abstract class BaseJob(ILogger logger) : IJob + { + protected abstract Task ExecuteInner(IJobExecutionContext context); + + public async Task Execute(IJobExecutionContext context) + { + logger.LogDebug($"Start {GetType().Name}"); + + try + { + await ExecuteInner(context); + } + catch (Exception e) + { + logger.LogError(e, e.Message); + throw; + } + + logger.LogDebug($"{GetType().Name} Finished"); + } + } +} \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Repository/sql/01 initial.sql b/Web/Azaion.Web/Azaion.Repository/sql/01 initial.sql index 5f0785b..8d30292 100644 --- a/Web/Azaion.Web/Azaion.Repository/sql/01 initial.sql +++ b/Web/Azaion.Web/Azaion.Repository/sql/01 initial.sql @@ -1,6 +1,11 @@ -CREATE USER 'azaion-user' IDENTIFIED BY 'Aza1on@db123' CREATE DATABASE azaion; -GRANT SELECT, INSERT, REFERENCES, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES - ON azaion.* - TO 'azaion-user'; -GRANT FILE ON *.* TO 'azaion-user'; + +CREATE USER 'azaion-admin' IDENTIFIED BY 'Aza1on@db-admin123'; +GRANT ALL ON azaion.* TO 'azaion-admin'; +GRANT FILE ON *.* TO 'azaion-admin'; + +CREATE USER 'azaion-user' IDENTIFIED BY 'Aza1on@db123'; +GRANT SELECT, INSERT, UPDATE, DELETE ON azaion.* TO 'azaion-user'; + +CREATE USER 'azaion-user-read' IDENTIFIED BY 'Az@10n-ro'; +GRANT SELECT ON azaion.* TO 'azaion-user'; \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Video/Azaion.Video.csproj b/Web/Azaion.Web/Azaion.Video/Azaion.Video.csproj index 4232095..3524887 100644 --- a/Web/Azaion.Web/Azaion.Video/Azaion.Video.csproj +++ b/Web/Azaion.Web/Azaion.Video/Azaion.Video.csproj @@ -10,4 +10,8 @@ + + + + diff --git a/Web/Azaion.Web/Azaion.Video/IFfmpegManager.cs b/Web/Azaion.Web/Azaion.Video/IFfmpegManager.cs new file mode 100644 index 0000000..8d2c47c --- /dev/null +++ b/Web/Azaion.Web/Azaion.Video/IFfmpegManager.cs @@ -0,0 +1,38 @@ +using System.Diagnostics; +using Azaion.Repository; +using Azaion.Repository.DTO.Configs; +using Microsoft.Extensions.Options; + +namespace Azaion.Video; + +public interface IFfmpegManager +{ + void ConvertToHls(string video, string segmentFile, string outFile); +} + +public class FfmpegManager(IOptions config) : IFfmpegManager +{ + public void ConvertToHls(string video, string segmentFile, string outFile) + { + string arguments = string.Concat($"-i \"{video}\" ", + "-f hls ", + "-hls_time 2 ", + "-hls_playlist_type vod ", + "-hls_flags independent_segments ", + "-hls_segment_type mpegts ", + $"-hls_segment_filename \"{segmentFile}\"", + $"\"{outFile}\""); + + var process = new Process + { + StartInfo = new() + { + FileName = config.Value.FfmpegExecutable, + Arguments = arguments, + UseShellExecute = true, + }, + EnableRaisingEvents = true, + }; + process.Start(); + } +} \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.Video/IVideoManager.cs b/Web/Azaion.Web/Azaion.Video/IVideoRepository.cs similarity index 97% rename from Web/Azaion.Web/Azaion.Video/IVideoManager.cs rename to Web/Azaion.Web/Azaion.Video/IVideoRepository.cs index 3994605..b86180e 100644 --- a/Web/Azaion.Web/Azaion.Video/IVideoManager.cs +++ b/Web/Azaion.Web/Azaion.Video/IVideoRepository.cs @@ -3,7 +3,7 @@ using Azaion.Repository.Entities; namespace Azaion.Video; -public interface IVideoManager +public interface IVideoRepository { List GetVideos(); Media Get(Guid mediaId); diff --git a/Web/Azaion.Web/Azaion.Video/VideoManager.cs b/Web/Azaion.Web/Azaion.Video/VideoRepository.cs similarity index 86% rename from Web/Azaion.Web/Azaion.Video/VideoManager.cs rename to Web/Azaion.Web/Azaion.Video/VideoRepository.cs index d57b1a0..52fd0dc 100644 --- a/Web/Azaion.Web/Azaion.Video/VideoManager.cs +++ b/Web/Azaion.Web/Azaion.Video/VideoRepository.cs @@ -1,11 +1,13 @@ using System.Diagnostics; using Azaion.Repository; using Azaion.Repository.DTO; +using Azaion.Repository.DTO.Configs; using Azaion.Repository.Entities; +using Microsoft.Extensions.Options; namespace Azaion.Video; -public class VideoManager(IDbFactory dbFactory) : IVideoManager +public class VideoRepository(IDbFactory dbFactory, IOptions foldersConfig) : IVideoRepository { public List GetVideos() { @@ -38,7 +40,7 @@ public class VideoManager(IDbFactory dbFactory) : IVideoManager { StartInfo = new() { - FileName = Constants.FFMPEG_FILE, + FileName = foldersConfig.Value.FfmpegExecutable, Arguments = arguments, UseShellExecute = true, }, diff --git a/Web/Azaion.Web/Azaion.WebService/Azaion.WebService.csproj b/Web/Azaion.Web/Azaion.WebService/Azaion.WebService.csproj index e66d464..7f4ac3e 100644 --- a/Web/Azaion.Web/Azaion.WebService/Azaion.WebService.csproj +++ b/Web/Azaion.Web/Azaion.WebService/Azaion.WebService.csproj @@ -8,6 +8,8 @@ + + diff --git a/Web/Azaion.Web/Azaion.WebService/Controllers/VideoController.cs b/Web/Azaion.Web/Azaion.WebService/Controllers/VideoController.cs index 8ab7b73..8376f34 100644 --- a/Web/Azaion.Web/Azaion.WebService/Controllers/VideoController.cs +++ b/Web/Azaion.Web/Azaion.WebService/Controllers/VideoController.cs @@ -1,19 +1,22 @@ +using Azaion.Repository.DTO.Configs; using Azaion.Video; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; namespace Azaion.WebService.Controllers; -[Route("/controller")] +[Route("/{controller}")] [ApiController] -public class VideoController(IVideoManager videoManager) : Controller +public class VideoController(IVideoRepository videoRepository, IOptions config) : Controller { [HttpGet("{guid}")] public IActionResult GetVideo(Guid guid) { - var media = videoManager.Get(guid); - var fileStream = new FileStream(media.M3U8File, FileMode.Open); + var media = videoRepository.Get(guid); + var fileStream = new FileStream(media.M3U8File(config), FileMode.Open); var fileSize = new FileInfo(media.Path).Length; Response.ContentLength = fileSize; return File(fileStream, "application/x-mpegURL", true); } + } diff --git a/Web/Azaion.Web/Azaion.WebService/Program.cs b/Web/Azaion.Web/Azaion.WebService/Program.cs index 31718ae..f26f857 100644 --- a/Web/Azaion.Web/Azaion.WebService/Program.cs +++ b/Web/Azaion.Web/Azaion.WebService/Program.cs @@ -1,7 +1,10 @@ +using System.Collections.Specialized; using Azaion.Repository; using Azaion.Repository.DTO; +using Azaion.Repository.DTO.Configs; using Azaion.Video; using Microsoft.Extensions.Options; +using Quartz; using Serilog; using Serilog.Events; @@ -22,9 +25,23 @@ builder.Services.AddControllers(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); +builder.Services.Configure(builder.Configuration.GetSection(nameof(FoldersConfig))); builder.Services.Configure(builder.Configuration.GetSection(nameof(ConnectionStrings))); builder.Services.AddSingleton(sp => new DbFactory(sp.GetService>()!.Value.FraudDb!)); -builder.Services.AddScoped(); +builder.Services.AddScoped(); + +var connStr = builder.Configuration.Get(); + +builder.Services.AddQuartz(q => +{ + q.SchedulerId = "AzaionScheduler"; + q.SchedulerName = "Azaion Scheduler"; + q.UsePersistentStore(c => + { + c.UseNewtonsoftJsonSerializer(); + c.UseMySql(connStr.FraudDbMsSql); + }); +}); var app = builder.Build(); @@ -37,4 +54,4 @@ if (app.Environment.IsDevelopment()) app.UseHttpsRedirection(); app.MapControllers(); -app.Run(); +app.Run(); \ No newline at end of file diff --git a/Web/Azaion.Web/Azaion.WebService/appsettings.Development.json b/Web/Azaion.Web/Azaion.WebService/appsettings.Development.json index 0c208ae..9119b1f 100644 --- a/Web/Azaion.Web/Azaion.WebService/appsettings.Development.json +++ b/Web/Azaion.Web/Azaion.WebService/appsettings.Development.json @@ -4,5 +4,10 @@ "Default": "Information", "Microsoft.AspNetCore": "Warning" } + }, + "FoldersConfig": { + "VideosFolder": "E:\\Azaion3\\Videos", + "HlsFolder": "/azaion-media/hls", + "FFMPEG_EXECUTABLE": "/opt/homebrew/bin/ffmpeg" } } diff --git a/Web/Azaion.Web/Azaion.WebService/appsettings.json b/Web/Azaion.Web/Azaion.WebService/appsettings.json index 10f68b8..d77c327 100644 --- a/Web/Azaion.Web/Azaion.WebService/appsettings.json +++ b/Web/Azaion.Web/Azaion.WebService/appsettings.json @@ -5,5 +5,13 @@ "Microsoft.AspNetCore": "Warning" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "ConnectionStrings": { + "AzaionDb": "" + }, + "FoldersConfig": { + "VideosFolder": "/azaion-media", + "HlsFolder": "/azaion-media/hls", + "FFMPEG_EXECUTABLE": "/opt/homebrew/bin/ffmpeg" + } }