diff --git a/README.md b/README.md index 0f3372c..327e1c2 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Usage: EthernaVideoImporter SOURCE_TYPE SOURCE_URI [OPTIONS] Source types: ytchannel YouTube channel ytvideo YouTube video + local Local videos Options: -ff Path FFmpeg (default dir: .\FFmpeg\) @@ -78,6 +79,34 @@ Options: Run 'EthernaVideoImporter.Devcon -h' to print help ``` +#### Local videos + +To import from local videos you will need a metadata descriptor file. +Metadata is a Json file with following structure. + +``` +[ + { + "Id": "myId1", + "Title": "My video 1 title", + "Description": "My video description", + "VideoFilePath": "my/source/local/video/path.mp4", + "ThumbnailFilePath": "my/optional/thumbnail/path.jpg" + }, + { + "Id": "myId2", + "Title": "My video 2 title", + ... + }, + ... +] +``` + +The `Id` field is mandatory, and is needed to trace same video through different executions. Each Id needs to be unique in the file. +The `ThumbnailFilePath` field is optional. + +The Json file path needs to be passed as source uri with the source type `local`. + # Issue reports If you've discovered a bug, or have an idea for a new feature, please report it to our issue manager based on Jira https://etherna.atlassian.net/projects/EVI. diff --git a/src/EthernaVideoImporter.Core/CommonConsts.cs b/src/EthernaVideoImporter.Core/CommonConsts.cs index 28cbec5..fd04bc3 100644 --- a/src/EthernaVideoImporter.Core/CommonConsts.cs +++ b/src/EthernaVideoImporter.Core/CommonConsts.cs @@ -15,6 +15,7 @@ using Etherna.BeeNet.Clients.DebugApi; using Etherna.BeeNet.Clients.GatewayApi; using System; +using System.IO; using System.Runtime.InteropServices; namespace Etherna.VideoImporter.Core @@ -26,6 +27,7 @@ public sealed class CommonConsts public const string BeeNodeUrl = "http://localhost/"; public const GatewayApiVersion BeeNodeGatewayVersion = GatewayApiVersion.v4_0_0; public const DebugApiVersion BeeNodeDebugVersion = DebugApiVersion.v4_0_0; + public const string DefaultFFmpegFolder = @".\FFmpeg\"; public const int DownloadMaxRetry = 3; public static readonly TimeSpan DownloadTimespanRetry = TimeSpan.FromMilliseconds(3500); public const string EthernaCreditUrl = "https://credit.etherna.io/"; @@ -50,7 +52,22 @@ public static string FFMpegBinaryName throw new InvalidOperationException("OS not supported"); } } + public static string FFProbeBinaryName + { + get + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return "ffprobe.exe"; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + return "ffprobe"; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + return "ffprobe"; + + throw new InvalidOperationException("OS not supported"); + } + } public static readonly TimeSpan GnosisBlockTime = TimeSpan.FromSeconds(5); public const string ImporterIdentifier = "EthernaImporter"; + public static DirectoryInfo TempDirectory { get; } = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), ImporterIdentifier)); } } diff --git a/src/EthernaVideoImporter.Core/EthernaVideoImporter.Core.csproj b/src/EthernaVideoImporter.Core/EthernaVideoImporter.Core.csproj index 492e87f..8c9cfde 100644 --- a/src/EthernaVideoImporter.Core/EthernaVideoImporter.Core.csproj +++ b/src/EthernaVideoImporter.Core/EthernaVideoImporter.Core.csproj @@ -23,27 +23,30 @@ - + + + + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + diff --git a/src/EthernaVideoImporter.Core/EthernaVideoImporter.cs b/src/EthernaVideoImporter.Core/EthernaVideoImporter.cs index 1a70b5e..fd77975 100644 --- a/src/EthernaVideoImporter.Core/EthernaVideoImporter.cs +++ b/src/EthernaVideoImporter.Core/EthernaVideoImporter.cs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Etherna.ServicesClient; using Etherna.ServicesClient.Clients.Index; using Etherna.VideoImporter.Core.Models.Domain; using Etherna.VideoImporter.Core.Models.Index; @@ -21,62 +22,54 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; using System.Linq; using System.Threading.Tasks; namespace Etherna.VideoImporter.Core { - public class EthernaVideoImporter + public class EthernaVideoImporter : IEthernaVideoImporter { // Fields. private readonly ICleanerVideoService cleanerVideoService; private readonly IGatewayService gatewayClient; - private readonly IUserIndexClient ethernaIndexClient; - private readonly ILinkReporterService linkReporterService; + private readonly IEthernaUserClients ethernaUserClients; private readonly IMigrationService migrationService; - private readonly DirectoryInfo tempDirectoryInfo; private readonly IVideoUploaderService videoUploaderService; private readonly IVideoProvider videoProvider; // Constructor. public EthernaVideoImporter( ICleanerVideoService cleanerVideoService, + IEthernaUserClients ethernaUserClients, IGatewayService gatewayClient, - IUserIndexClient ethernaIndexClient, - ILinkReporterService linkReporterService, + IMigrationService migrationService, IVideoProvider videoProvider, - IVideoUploaderService videoUploaderService, - IMigrationService migrationService) + IVideoUploaderService videoUploaderService) { if (cleanerVideoService is null) throw new ArgumentNullException(nameof(cleanerVideoService)); - if (linkReporterService is null) - throw new ArgumentNullException(nameof(linkReporterService)); if (videoProvider is null) throw new ArgumentNullException(nameof(videoProvider)); if (videoUploaderService is null) throw new ArgumentNullException(nameof(videoUploaderService)); this.cleanerVideoService = cleanerVideoService; + this.ethernaUserClients = ethernaUserClients; this.gatewayClient = gatewayClient; - this.ethernaIndexClient = ethernaIndexClient; - this.linkReporterService = linkReporterService; this.migrationService = migrationService; - tempDirectoryInfo = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), CommonConsts.ImporterIdentifier)); this.videoProvider = videoProvider; this.videoUploaderService = videoUploaderService; } // Public methods. public async Task RunAsync( - string userEthAddress, + bool deleteExogenousVideos, + bool deleteVideosRemovedFromSource, + bool forceVideoUpload, bool offerVideos, bool pinVideos, - bool deleteVideosRemovedFromSource, - bool deleteExogenousVideos, - bool unpinRemovedVideos, - bool forceUploadVideo) + string userEthAddress, + bool unpinRemovedVideos) { var importSummaryModelView = new ImportSummaryModelView(); @@ -92,7 +85,7 @@ public async Task RunAsync( Console.WriteLine("Get user's videos on etherna index"); var userVideosOnIndex = await GetUserVideosOnEthernaAsync(userEthAddress); - var ethernaIndexParameters = await ethernaIndexClient.SystemClient.ParametersAsync(); + var ethernaIndexParameters = await ethernaUserClients.IndexClient.SystemClient.ParametersAsync(); Console.WriteLine($"Found {userVideosOnIndex.Count()} videos already published on etherna index"); @@ -120,7 +113,7 @@ public async Task RunAsync( //try to update only manifest, or to skip if possible if (alreadyPresentVideo is not null && - !forceUploadVideo && + !forceVideoUpload && minRequiredMigrationOp is OperationType.Skip or OperationType.UpdateManifest) { Console.WriteLine("Video already uploaded on etherna"); @@ -154,10 +147,10 @@ public async Task RunAsync( var videoMetadata = new SwarmVideoMetadata( sourceMetadata.Id, + sourceMetadata.Title, sourceMetadata.Description, TimeSpan.FromSeconds(alreadyPresentVideo.LastValidManifest!.Duration), - alreadyPresentVideo.LastValidManifest!.OriginalQuality, - sourceMetadata.Title); + alreadyPresentVideo.LastValidManifest!.OriginalQuality); var videoSwarmFile = alreadyPresentVideo.LastValidManifest.Sources.Select(v => new VideoSwarmFile(v.Size, v.Quality, v.Reference)); @@ -168,7 +161,7 @@ public async Task RunAsync( updatedPermalinkHash = await videoUploaderService.UploadVideoManifestAsync(metadataVideo, pinVideos, offerVideos); // Update on index. - await ethernaIndexClient.VideosClient.VideosPutAsync( + await ethernaUserClients.IndexClient.VideosClient.VideosPutAsync( alreadyPresentVideo.IndexId, updatedPermalinkHash); } @@ -196,7 +189,7 @@ await ethernaIndexClient.VideosClient.VideosPutAsync( } // Get and encode video from source. - var video = await videoProvider.GetVideoAsync(sourceMetadata, tempDirectoryInfo); + var video = await videoProvider.GetVideoAsync(sourceMetadata); video.EthernaIndexId = alreadyPresentVideo?.IndexId; if (!video.EncodedFiles.Any()) @@ -248,15 +241,15 @@ await ethernaIndexClient.VideosClient.VideosPutAsync( try { // Clear tmp folder. - foreach (var file in tempDirectoryInfo.GetFiles()) + foreach (var file in CommonConsts.TempDirectory.GetFiles()) file.Delete(); - foreach (var dir in tempDirectoryInfo.GetDirectories()) + foreach (var dir in CommonConsts.TempDirectory.GetDirectories()) dir.Delete(true); } catch { Console.ForegroundColor = ConsoleColor.DarkRed; - Console.WriteLine($"Warning: unable to delete some files inside of {tempDirectoryInfo.FullName}."); + Console.WriteLine($"Warning: unable to delete some files inside of {CommonConsts.TempDirectory.FullName}."); Console.WriteLine($"Please remove manually after the process."); Console.ResetColor(); } @@ -269,7 +262,7 @@ await ethernaIndexClient.VideosClient.VideosPutAsync( * Report etherna references even if video is already present on index. * This handle cases where references are missing for some previous execution error. */ - await linkReporterService.SetEthernaReferencesAsync( + await videoProvider.ReportEthernaReferencesAsync( sourceMetadata.Id, updatedIndexId, updatedPermalinkHash); @@ -318,7 +311,7 @@ await linkReporterService.SetEthernaReferencesAsync( } // Helpers. - public async Task> GetUserVideosOnEthernaAsync(string userAddress) + private async Task> GetUserVideosOnEthernaAsync(string userAddress) { var videos = new List(); const int MaxForPage = 100; @@ -326,7 +319,7 @@ public async Task> GetUserVideosOnEthernaAsync(string VideoDtoPaginatedEnumerableDto? page = null; do { - page = await ethernaIndexClient.UsersClient.Videos2Async(userAddress, page is null ? 0 : page.CurrentPage + 1, MaxForPage); + page = await ethernaUserClients.IndexClient.UsersClient.Videos2Async(userAddress, page is null ? 0 : page.CurrentPage + 1, MaxForPage); videos.AddRange(page.Elements); } while (page.Elements.Any()); diff --git a/src/EthernaVideoImporter/Services/LinkReporterService.cs b/src/EthernaVideoImporter.Core/IEthernaVideoImporter.cs similarity index 58% rename from src/EthernaVideoImporter/Services/LinkReporterService.cs rename to src/EthernaVideoImporter.Core/IEthernaVideoImporter.cs index cbb80c3..7f7bfab 100644 --- a/src/EthernaVideoImporter/Services/LinkReporterService.cs +++ b/src/EthernaVideoImporter.Core/IEthernaVideoImporter.cs @@ -12,25 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Etherna.VideoImporter.Core.Services; using System.Threading.Tasks; -namespace Etherna.VideoImporter.Services +namespace Etherna.VideoImporter.Core { - public sealed class LinkReporterService : ILinkReporterService + public interface IEthernaVideoImporter { - // Constructors. - public LinkReporterService() - { - } - - // Methods. - public Task SetEthernaReferencesAsync( - string sourceVideoId, - string ethernaIndexId, - string ethernaPermalinkHash) - { - return Task.CompletedTask; - } + Task RunAsync( + bool deleteExogenousVideos, + bool deleteVideosRemovedFromSource, + bool forceVideoUpload, + bool offerVideos, + bool pinVideos, + string userEthAddress, + bool unpinRemovedVideos); } -} +} \ No newline at end of file diff --git a/src/EthernaVideoImporter.Core/Models/Domain/SwarmVideoMetadata.cs b/src/EthernaVideoImporter.Core/Models/Domain/SwarmVideoMetadata.cs index de25803..7fd0fb4 100644 --- a/src/EthernaVideoImporter.Core/Models/Domain/SwarmVideoMetadata.cs +++ b/src/EthernaVideoImporter.Core/Models/Domain/SwarmVideoMetadata.cs @@ -7,11 +7,11 @@ internal sealed class SwarmVideoMetadata : VideoMetadataBase // Constructors. internal SwarmVideoMetadata( string id, + string title, string description, TimeSpan duration, - string originVideoQualityLabel, - string title) - : base(description, duration, originVideoQualityLabel, title) + string originVideoQualityLabel) + : base(title, description, duration, originVideoQualityLabel) { Id = id; } diff --git a/src/EthernaVideoImporter.Core/Models/Domain/ThumbnailLocalFile.cs b/src/EthernaVideoImporter.Core/Models/Domain/ThumbnailLocalFile.cs index ccc1f7c..6ab3f15 100644 --- a/src/EthernaVideoImporter.Core/Models/Domain/ThumbnailLocalFile.cs +++ b/src/EthernaVideoImporter.Core/Models/Domain/ThumbnailLocalFile.cs @@ -14,7 +14,10 @@ using Blurhash.SkiaSharp; using SkiaSharp; +using System; +using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; namespace Etherna.VideoImporter.Core.Models.Domain { @@ -58,5 +61,41 @@ public ThumbnailLocalFile( /// Canvas width (in pixels). /// public int Width { get; } + + // Methods. + public async Task> GetScaledThumbnailsAsync( + DirectoryInfo importerTempDirectoryInfo) + { + if (importerTempDirectoryInfo is null) + throw new ArgumentNullException(nameof(importerTempDirectoryInfo)); + + List thumbnails = new(); + + using var thumbFileStream = File.OpenRead(FilePath); + using var thumbManagedStream = new SKManagedStream(thumbFileStream); + using var thumbBitmap = SKBitmap.Decode(thumbManagedStream); + + foreach (var responsiveWidthSize in ThumbnailResponsiveSizes) + { + var responsiveHeightSize = (int)(responsiveWidthSize / AspectRatio); + var thumbnailResizedPath = Path.Combine(importerTempDirectoryInfo.FullName, $"thumb_{responsiveWidthSize}_{responsiveHeightSize}_{Guid.NewGuid()}.jpg"); + + using (SKBitmap scaledBitmap = thumbBitmap.Resize(new SKImageInfo(responsiveWidthSize, responsiveHeightSize), SKFilterQuality.Medium)) + using (SKImage scaledImage = SKImage.FromBitmap(scaledBitmap)) + using (SKData data = scaledImage.Encode()) + using (FileStream outputFileStream = new(thumbnailResizedPath, FileMode.CreateNew)) + { + await data.AsStream().CopyToAsync(outputFileStream); + } + + thumbnails.Add(new ThumbnailLocalFile( + thumbnailResizedPath, + new FileInfo(thumbnailResizedPath).Length, + responsiveHeightSize, + responsiveWidthSize)); + } + + return thumbnails; + } } } diff --git a/src/EthernaVideoImporter.Core/Models/Domain/VideoMetadataBase.cs b/src/EthernaVideoImporter.Core/Models/Domain/VideoMetadataBase.cs index 7d94625..8880c88 100644 --- a/src/EthernaVideoImporter.Core/Models/Domain/VideoMetadataBase.cs +++ b/src/EthernaVideoImporter.Core/Models/Domain/VideoMetadataBase.cs @@ -25,10 +25,10 @@ public abstract partial class VideoMetadataBase // Constructor. protected VideoMetadataBase( + string title, string description, TimeSpan duration, - string originVideoQualityLabel, - string title) + string originVideoQualityLabel) { Description = description; Duration = duration; diff --git a/src/EthernaVideoImporter.Core/Models/Domain/YouTubeVideoMetadataBase.cs b/src/EthernaVideoImporter.Core/Models/Domain/YouTubeVideoMetadataBase.cs index 53c5e10..acca368 100644 --- a/src/EthernaVideoImporter.Core/Models/Domain/YouTubeVideoMetadataBase.cs +++ b/src/EthernaVideoImporter.Core/Models/Domain/YouTubeVideoMetadataBase.cs @@ -23,13 +23,13 @@ public abstract class YouTubeVideoMetadataBase : VideoMetadataBase { // Constructor. protected YouTubeVideoMetadataBase( + string title, string description, TimeSpan duration, string originVideoQualityLabel, Thumbnail? thumbnail, - string title, string youtubeUrl) - : base(description, duration, originVideoQualityLabel, title) + : base(title, description, duration, originVideoQualityLabel) { Thumbnail = thumbnail; YoutubeUrl = youtubeUrl; diff --git a/src/EthernaVideoImporter.Core/Options/EncoderServiceOptions.cs b/src/EthernaVideoImporter.Core/Options/EncoderServiceOptions.cs new file mode 100644 index 0000000..7588eea --- /dev/null +++ b/src/EthernaVideoImporter.Core/Options/EncoderServiceOptions.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using System.IO; + +namespace Etherna.VideoImporter.Core.Options +{ + public sealed class EncoderServiceOptions + { + // Properties. + public string FFMpegBinaryPath => Path.Combine(FFMpegFolderPath, CommonConsts.FFMpegBinaryName); + public string FFMpegFolderPath { get; set; } = CommonConsts.DefaultFFmpegFolder; + public FFMpegHWAccelerationType FFMpegHWAccelerationType { get; set; } = FFMpegHWAccelerationType.None; + public bool IncludeAudioTrack { get; set; } + public bool Skip360 { get; set; } + public bool Skip480 { get; set; } + public bool Skip720 { get; set; } + public bool Skip1080 { get; set; } + public bool Skip1440 { get; set; } + + // Methods. + public IEnumerable GetSupportedHeightResolutions() + { + var supportedHeightResolutions = new List(); + if (!Skip1440) + supportedHeightResolutions.Add(1440); + if (!Skip1080) + supportedHeightResolutions.Add(1080); + if (!Skip720) + supportedHeightResolutions.Add(720); + if (!Skip480) + supportedHeightResolutions.Add(480); + if (!Skip360) + supportedHeightResolutions.Add(360); + + return supportedHeightResolutions; + } + } +} diff --git a/src/EthernaVideoImporter.Core/Options/EncoderServiceOptionsValidation.cs b/src/EthernaVideoImporter.Core/Options/EncoderServiceOptionsValidation.cs new file mode 100644 index 0000000..7a6be77 --- /dev/null +++ b/src/EthernaVideoImporter.Core/Options/EncoderServiceOptionsValidation.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.Options; +using System.IO; + +namespace Etherna.VideoImporter.Core.Options +{ + internal sealed class EncoderServiceOptionsValidation : IValidateOptions + { + public ValidateOptionsResult Validate(string? name, EncoderServiceOptions options) + { + if (!File.Exists(options.FFMpegBinaryPath)) + return ValidateOptionsResult.Fail($"FFmpeg not found at ({options.FFMpegBinaryPath})"); + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/EthernaVideoImporter.Core/Options/VideoUploaderServiceOptions.cs b/src/EthernaVideoImporter.Core/Options/VideoUploaderServiceOptions.cs new file mode 100644 index 0000000..0fd0321 --- /dev/null +++ b/src/EthernaVideoImporter.Core/Options/VideoUploaderServiceOptions.cs @@ -0,0 +1,11 @@ +using System; + +namespace Etherna.VideoImporter.Core.Options +{ + public sealed class VideoUploaderServiceOptions + { + public bool AcceptPurchaseOfAllBatches { get; set; } + public TimeSpan TtlPostageStamp { get; set; } + public string UserEthAddr { get; set; } = default!; + } +} diff --git a/src/EthernaVideoImporter.Core/Options/VideoUploaderServiceOptionsValidation.cs b/src/EthernaVideoImporter.Core/Options/VideoUploaderServiceOptionsValidation.cs new file mode 100644 index 0000000..82b9235 --- /dev/null +++ b/src/EthernaVideoImporter.Core/Options/VideoUploaderServiceOptionsValidation.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Options; + +namespace Etherna.VideoImporter.Core.Options +{ + internal sealed class VideoUploaderServiceOptionsValidation : IValidateOptions + { + public ValidateOptionsResult Validate(string? name, VideoUploaderServiceOptions options) + { + if (string.IsNullOrWhiteSpace(options.UserEthAddr)) + return ValidateOptionsResult.Fail("Missing user's ethereum address"); + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/EthernaVideoImporter.Core/ServiceCollectionExtensions.cs b/src/EthernaVideoImporter.Core/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..ceb5490 --- /dev/null +++ b/src/EthernaVideoImporter.Core/ServiceCollectionExtensions.cs @@ -0,0 +1,75 @@ +using Etherna.BeeNet; +using Etherna.BeeNet.Clients.GatewayApi; +using Etherna.ServicesClient; +using Etherna.VideoImporter.Core.Options; +using Etherna.VideoImporter.Core.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using System; +using System.Net.Http; + +namespace Etherna.VideoImporter.Core +{ + public static class ServiceCollectionExtensions + { + private const string EthernaServicesHttpClientName = "ethernaClient"; + + public static void AddCoreServices( + this IServiceCollection services, + Action configureEncoderOptions, + Action configureVideoUploaderOptions, + bool useBeeNativeNode, + HttpMessageHandler ethernaServicesHttpMessageHandler) + { + // Configure options. + services.Configure(configureEncoderOptions); + services.AddSingleton, EncoderServiceOptionsValidation>(); + services.Configure(configureVideoUploaderOptions); + services.AddSingleton, VideoUploaderServiceOptionsValidation>(); + + // Add transient services. + services.AddTransient(); + + services.AddTransient(); + services.AddTransient(); + if (useBeeNativeNode) + services.AddTransient(); + else + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + // Add singleton services. + //etherna services user client + services.AddSingleton((sp) => + { + var httpClientFactory = sp.GetRequiredService(); + return new EthernaUserClients( + new Uri(CommonConsts.EthernaCreditUrl), + new Uri(CommonConsts.EthernaGatewayUrl), + new Uri(CommonConsts.EthernaIndexUrl), + new Uri(CommonConsts.EthernaIndexUrl), + () => httpClientFactory.CreateClient(EthernaServicesHttpClientName)); + }); + + //bee.net + services.AddSingleton((sp) => + { + var httpClientFactory = sp.GetRequiredService(); + return new BeeGatewayClient( + httpClientFactory.CreateClient(EthernaServicesHttpClientName), + new Uri(CommonConsts.EthernaGatewayUrl), + CommonConsts.BeeNodeGatewayVersion); + }); + services.AddSingleton(); + + // Add http clients. + services.AddHttpClient(EthernaServicesHttpClientName, c => + { + c.Timeout = TimeSpan.FromMinutes(30); + c.DefaultRequestHeaders.ConnectionClose = true; //fixes https://etherna.atlassian.net/browse/EVI-74 + }) + .ConfigurePrimaryHttpMessageHandler(() => ethernaServicesHttpMessageHandler); + } + } +} diff --git a/src/EthernaVideoImporter.Core/Services/CleanerVideoService.cs b/src/EthernaVideoImporter.Core/Services/CleanerVideoService.cs index c7baa32..58a31a5 100644 --- a/src/EthernaVideoImporter.Core/Services/CleanerVideoService.cs +++ b/src/EthernaVideoImporter.Core/Services/CleanerVideoService.cs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Etherna.ServicesClient.Clients.Index; +using Etherna.ServicesClient; using Etherna.VideoImporter.Core.Models.Domain; using Etherna.VideoImporter.Core.Models.Index; using Etherna.VideoImporter.Core.Models.ManifestDtos; @@ -26,15 +26,15 @@ namespace Etherna.VideoImporter.Core.Services public sealed class CleanerVideoService : ICleanerVideoService { // Fields. - private readonly IUserIndexClient ethernaIndexClient; + private readonly IEthernaUserClients ethernaUserClients; private readonly IGatewayService gatewayService; // Constructor. public CleanerVideoService( - IUserIndexClient ethernaIndexClient, + IEthernaUserClients ethernaUserClients, IGatewayService gatewayService) { - this.ethernaIndexClient = ethernaIndexClient; + this.ethernaUserClients = ethernaUserClients; this.gatewayService = gatewayService; } @@ -120,7 +120,7 @@ private async Task RemoveFromIndexAsync( bool removeSucceeded = false; try { - await ethernaIndexClient.VideosClient.VideosDeleteAsync(indexedVideo.IndexId); + await ethernaUserClients.IndexClient.VideosClient.VideosDeleteAsync(indexedVideo.IndexId); removeSucceeded = true; Console.ForegroundColor = ConsoleColor.DarkGreen; diff --git a/src/EthernaVideoImporter.Core/Services/EncoderService.cs b/src/EthernaVideoImporter.Core/Services/EncoderService.cs index 5517e5f..cd382b4 100644 --- a/src/EthernaVideoImporter.Core/Services/EncoderService.cs +++ b/src/EthernaVideoImporter.Core/Services/EncoderService.cs @@ -1,5 +1,7 @@ using Etherna.VideoImporter.Core.Models.Domain; +using Etherna.VideoImporter.Core.Options; using Medallion.Shell; +using Microsoft.Extensions.Options; using System; using System.Collections.Generic; using System.IO; @@ -8,43 +10,36 @@ namespace Etherna.VideoImporter.Core.Services { - public partial class EncoderService : IEncoderService + internal sealed class EncoderService : IEncoderService { // Fields. - private readonly List activedCommands = new(); - private readonly FFMpegHWAccelerationType ffMpegHWAccelerationType; + private readonly List activedCommands = new(); + private readonly EncoderServiceOptions options; // Constructor. public EncoderService( - string ffMpegBinaryPath, - FFMpegHWAccelerationType ffMpegHWAccelerationType) + IOptions options) { - FFMpegBinaryPath = ffMpegBinaryPath; - this.ffMpegHWAccelerationType = ffMpegHWAccelerationType; + this.options = options.Value; } // Properties. - public string FFMpegBinaryPath { get; } + public string FFMpegBinaryPath => options.FFMpegBinaryPath; // Methods. public async Task> EncodeVideosAsync( - VideoLocalFile originalVideoLocalFile, - DirectoryInfo importerTempDirectoryInfo, - IEnumerable supportedHeightResolutions, - bool includeAudioTrack) + VideoLocalFile sourceVideoFile) { - if (originalVideoLocalFile is null) - throw new ArgumentNullException(nameof(originalVideoLocalFile)); - if (importerTempDirectoryInfo is null) - throw new ArgumentNullException(nameof(importerTempDirectoryInfo)); + if (sourceVideoFile is null) + throw new ArgumentNullException(nameof(sourceVideoFile)); var videoEncodedFiles = new List(); var fileNameGuid = Guid.NewGuid(); - var resolutionRatio = (decimal)originalVideoLocalFile.Width / originalVideoLocalFile.Height; + var resolutionRatio = (decimal)sourceVideoFile.Width / sourceVideoFile.Height; - foreach (var heightResolution in supportedHeightResolutions.Union(new List { originalVideoLocalFile.Height })) + foreach (var heightResolution in options.GetSupportedHeightResolutions().Union(new List { sourceVideoFile.Height })) { - if (originalVideoLocalFile.Height < heightResolution) + if (sourceVideoFile.Height < heightResolution) continue; Console.WriteLine($"Encoding resolution {heightResolution}..."); @@ -60,20 +55,27 @@ public async Task> EncodeVideosAsync( _ => throw new InvalidOperationException() }; - var fileName = $"{importerTempDirectoryInfo.FullName}/{fileNameGuid}_{heightResolution}.mp4"; + var fileName = $"{CommonConsts.TempDirectory.FullName}/{fileNameGuid}_{heightResolution}.mp4"; var args = new string[] { - $"-i {originalVideoLocalFile.FilePath}", - "-c:a aac", - "-c:v libx264", - "-movflags faststart", - ffMpegHWAccelerationType == FFMpegHWAccelerationType.None - ? "libx264" : "h264_nvenc", - ffMpegHWAccelerationType == FFMpegHWAccelerationType.None - ? "" : "-hwaccel cuda -hwaccel_output_format cuda", - $"-vf scale={roundedScaledWidth}:{heightResolution}", - "-loglevel info"}; - - var command = Command.Run(FFMpegBinaryPath, args.SelectMany(arg => arg.Split(' ')).Append(fileName)); + "-i", sourceVideoFile.FilePath, + "-c:a", "aac", + "-c:v", ffMpegHWAccelerationType switch + { + FFMpegHWAccelerationType.None => "libx264", + FFMpegHWAccelerationType.Cuda => "h264_nvenc" + }, + ffMpegHWAccelerationType switch + { + FFMpegHWAccelerationType.None => "", + FFMpegHWAccelerationType.Cuda => "-hwaccel cuda -hwaccel_output_format cuda" + }, + "-movflags", "faststart", + "-vf", $"scale={roundedScaledWidth}:{heightResolution}", + "-loglevel", "info", + fileName + }; + + var command = Command.Run(FFMpegBinaryPath, args); activedCommands.Add(command); Console.CancelKeyPress += new ConsoleCancelEventHandler(ManageInterrupted); diff --git a/src/EthernaVideoImporter.Core/Services/EthernaGatewayService.cs b/src/EthernaVideoImporter.Core/Services/EthernaGatewayService.cs index 8e2f7d3..485999a 100644 --- a/src/EthernaVideoImporter.Core/Services/EthernaGatewayService.cs +++ b/src/EthernaVideoImporter.Core/Services/EthernaGatewayService.cs @@ -1,4 +1,5 @@ using Etherna.BeeNet.Clients.GatewayApi; +using Etherna.ServicesClient; using Etherna.ServicesClient.Clients.Gateway; using System; using System.Collections.Generic; @@ -12,21 +13,21 @@ public class EthernaGatewayService : GatewayServiceBase private readonly TimeSpan BatchCreationTimeout = new(0, 0, 10, 0); // Fields. - private readonly IUserGatewayClient ethernaGatewayClient; + private readonly IEthernaUserClients ethernaUserClients; // Constructor. public EthernaGatewayService( IBeeGatewayClient beeGatewayClient, - IUserGatewayClient ethernaGatewayClient) + IEthernaUserClients ethernaUserClients) : base(beeGatewayClient) { - this.ethernaGatewayClient = ethernaGatewayClient; + this.ethernaUserClients = ethernaUserClients; } // Methods. public override async Task CreatePostageBatchAsync(long amount, int batchDepth) { - var batchReferenceId = await ethernaGatewayClient.UsersClient.BatchesPostAsync(batchDepth, amount); + var batchReferenceId = await ethernaUserClients.GatewayClient.UsersClient.BatchesPostAsync(batchDepth, amount); // Wait until created batch is avaiable. Console.Write("Waiting for batch created... (it may take a while)"); @@ -45,7 +46,7 @@ public override async Task CreatePostageBatchAsync(long amount, int batc try { - batchId = await ethernaGatewayClient.SystemClient.PostageBatchRefAsync(batchReferenceId); + batchId = await ethernaUserClients.GatewayClient.SystemClient.PostageBatchRefAsync(batchReferenceId); } catch (GatewayApiException) { @@ -62,18 +63,18 @@ public override async Task CreatePostageBatchAsync(long amount, int batc } public override Task DeletePinAsync(string hash) => - ethernaGatewayClient.ResourcesClient.PinDeleteAsync(hash); + ethernaUserClients.GatewayClient.ResourcesClient.PinDeleteAsync(hash); public override async Task GetCurrentChainPriceAsync() => - (await ethernaGatewayClient.SystemClient.ChainstateAsync()).CurrentPrice; + (await ethernaUserClients.GatewayClient.SystemClient.ChainstateAsync()).CurrentPrice; public override async Task> GetPinnedResourcesAsync() => - await ethernaGatewayClient.UsersClient.PinnedResourcesAsync(); + await ethernaUserClients.GatewayClient.UsersClient.PinnedResourcesAsync(); public override async Task IsBatchUsableAsync(string batchId) => - (await ethernaGatewayClient.UsersClient.BatchesGetAsync(batchId)).Usable; + (await ethernaUserClients.GatewayClient.UsersClient.BatchesGetAsync(batchId)).Usable; public override Task OfferContentAsync(string hash) => - ethernaGatewayClient.ResourcesClient.OffersPostAsync(hash); + ethernaUserClients.GatewayClient.ResourcesClient.OffersPostAsync(hash); } } diff --git a/src/EthernaVideoImporter.Core/Services/IEncoderService.cs b/src/EthernaVideoImporter.Core/Services/IEncoderService.cs index d7f8002..de2e1f7 100644 --- a/src/EthernaVideoImporter.Core/Services/IEncoderService.cs +++ b/src/EthernaVideoImporter.Core/Services/IEncoderService.cs @@ -1,6 +1,5 @@ using Etherna.VideoImporter.Core.Models.Domain; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; namespace Etherna.VideoImporter.Core.Services @@ -10,9 +9,6 @@ public interface IEncoderService string FFMpegBinaryPath { get; } Task> EncodeVideosAsync( - VideoLocalFile originalVideoLocalFile, - DirectoryInfo importerTempDirectoryInfo, - IEnumerable supportedHeightResolutions, - bool includeAudioTrack); + VideoLocalFile sourceVideoFile); } } diff --git a/src/EthernaVideoImporter.Core/Services/ILinkReporterService.cs b/src/EthernaVideoImporter.Core/Services/ILinkReporterService.cs deleted file mode 100644 index c38227e..0000000 --- a/src/EthernaVideoImporter.Core/Services/ILinkReporterService.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2022-present Etherna Sagl -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Threading.Tasks; - -namespace Etherna.VideoImporter.Core.Services -{ - /// - /// Link services - /// - public interface ILinkReporterService - { - /// - /// Set etherna data in destination Uri. - /// - /// The video id on source - /// Video Id on index - /// Video hash on permalink - Task SetEthernaReferencesAsync( - string sourceVideoId, - string ethernaIndexId, - string ethernaPermalinkHash); - } -} diff --git a/src/EthernaVideoImporter.Core/Services/IVideoProvider.cs b/src/EthernaVideoImporter.Core/Services/IVideoProvider.cs index ba47442..df1d525 100644 --- a/src/EthernaVideoImporter.Core/Services/IVideoProvider.cs +++ b/src/EthernaVideoImporter.Core/Services/IVideoProvider.cs @@ -14,7 +14,6 @@ using Etherna.VideoImporter.Core.Models.Domain; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; namespace Etherna.VideoImporter.Core.Services @@ -25,7 +24,18 @@ public interface IVideoProvider string SourceName { get; } // Methods. - Task diff --git a/src/EthernaVideoImporter.Devcon/Models/Domain/MdFileVideoMetadata.cs b/src/EthernaVideoImporter.Devcon/Models/Domain/MdFileVideoMetadata.cs index 1570a41..224c3ea 100644 --- a/src/EthernaVideoImporter.Devcon/Models/Domain/MdFileVideoMetadata.cs +++ b/src/EthernaVideoImporter.Devcon/Models/Domain/MdFileVideoMetadata.cs @@ -22,16 +22,16 @@ public class MdFileVideoMetadata : YouTubeVideoMetadataBase { // Constructor. public MdFileVideoMetadata( + string title, string description, TimeSpan duration, string originVideoQualityLabel, Thumbnail? thumbnail, - string title, string mdFileRelativePath, string youtubeUrl, string? ethernaIndexUrl, string? ethernaPermalinkUrl) - : base(description, duration, originVideoQualityLabel, thumbnail, title, youtubeUrl) + : base(title, description, duration, originVideoQualityLabel, thumbnail, youtubeUrl) { EthernaIndexUrl = ethernaIndexUrl; EthernaPermalinkUrl = ethernaPermalinkUrl; diff --git a/src/EthernaVideoImporter.Devcon/Options/MdVideoProviderOptions.cs b/src/EthernaVideoImporter.Devcon/Options/MdVideoProviderOptions.cs new file mode 100644 index 0000000..f1474a1 --- /dev/null +++ b/src/EthernaVideoImporter.Devcon/Options/MdVideoProviderOptions.cs @@ -0,0 +1,7 @@ +namespace Etherna.VideoImporter.Devcon.Options +{ + internal sealed class MdVideoProviderOptions + { + public string MdSourceFolderPath { get; set; } = default!; + } +} diff --git a/src/EthernaVideoImporter.Devcon/Options/MdVideoProviderOptionsValidation.cs b/src/EthernaVideoImporter.Devcon/Options/MdVideoProviderOptionsValidation.cs new file mode 100644 index 0000000..bcd95b0 --- /dev/null +++ b/src/EthernaVideoImporter.Devcon/Options/MdVideoProviderOptionsValidation.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.Options; +using System.IO; + +namespace Etherna.VideoImporter.Devcon.Options +{ + internal sealed class MdVideoProviderOptionsValidation : IValidateOptions + { + public ValidateOptionsResult Validate(string? name, MdVideoProviderOptions options) + { + if (!Directory.Exists(options.MdSourceFolderPath)) + return ValidateOptionsResult.Fail($"Not found MD directory path at ({options.MdSourceFolderPath})"); + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/EthernaVideoImporter.Devcon/Program.cs b/src/EthernaVideoImporter.Devcon/Program.cs index 2cadbe2..88a5276 100644 --- a/src/EthernaVideoImporter.Devcon/Program.cs +++ b/src/EthernaVideoImporter.Devcon/Program.cs @@ -13,19 +13,20 @@ // limitations under the License. using Etherna.Authentication; -using Etherna.BeeNet; -using Etherna.ServicesClient; using Etherna.VideoImporter.Core; using Etherna.VideoImporter.Core.Services; using Etherna.VideoImporter.Core.SSO; +using Etherna.VideoImporter.Core.Utilities; +using Etherna.VideoImporter.Devcon.Options; using Etherna.VideoImporter.Devcon.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using System; -using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; -using System.Net.Http; using System.Threading.Tasks; +using YoutubeExplode; namespace Etherna.VideoImporter.Devcon { @@ -33,13 +34,12 @@ internal static class Program { // Consts. private const int DefaultTTLPostageStamp = 365; - private const string DefaultFFmpegFolder = @".\FFmpeg\"; private static readonly string HelpText = "\n" + "Usage:\tEthernaVideoImporter.Devcon md MD_FOLDER [OPTIONS]\n" + "\n" + "Options:\n" + - $" -ff\tPath FFmpeg (default dir: {DefaultFFmpegFolder})\n" + + $" -ff\tPath FFmpeg (default dir: {CommonConsts.DefaultFFmpegFolder})\n" + $" -hw\tUse NVIDIA CUDA hardware acceleration on FFmpeg (default: false)\n" + $" -t\tTTL (days) Postage Stamp (default value: {DefaultTTLPostageStamp} days)\n" + " -o\tOffer video downloads to everyone\n" + @@ -66,7 +66,7 @@ static async Task Main(string[] args) { // Parse arguments. string? mdSourceFolderPath = null; - string ffMpegFolderPath = DefaultFFmpegFolder; + string? customFFMpegFolderPath = null; string? ttlPostageStampStr = null; string? beeNodeApiPortStr = null; string? beeNodeDebugPortStr = null; @@ -80,9 +80,9 @@ static async Task Main(string[] args) bool deleteExogenousVideos = false; bool includeAudioTrack = false; //temporary disabled until https://etherna.atlassian.net/browse/EVI-21 bool unpinRemovedVideos = false; - bool forceUploadVideo = false; + bool forceVideoUpload = false; bool acceptPurchaseOfAllBatches = false; - bool ignoreNewVersionOfImporter = false; + bool ignoreNewVersionsOfImporter = false; bool skip1440 = false; bool skip1080 = false; bool skip720 = false; @@ -132,7 +132,7 @@ static async Task Main(string[] args) case "-ff": if (args.Length == i + 1) throw new InvalidOperationException("ffMpeg folder is missing"); - ffMpegFolderPath = args[++i]; + customFFMpegFolderPath = args[++i]; break; case "-t": if (args.Length == i + 1) @@ -167,9 +167,9 @@ static async Task Main(string[] args) case "-m": deleteVideosMissingFromSource = true; break; case "-e": deleteExogenousVideos = true; break; case "-u": unpinRemovedVideos = true; break; - case "-f": forceUploadVideo = true; break; + case "-f": forceVideoUpload = true; break; case "-y": acceptPurchaseOfAllBatches = true; break; - case "-i": ignoreNewVersionOfImporter = true; break; + case "-i": ignoreNewVersionsOfImporter = true; break; case "-skip1440": skip1440 = true; break; case "-skip1080": skip1080 = true; break; case "-skip720": skip720 = true; break; @@ -181,14 +181,6 @@ static async Task Main(string[] args) } // Input validation. - //FFmpeg - var ffMpegBinaryPath = Path.Combine(ffMpegFolderPath, CommonConsts.FFMpegBinaryName); - if (!File.Exists(ffMpegBinaryPath)) - { - Console.WriteLine($"FFmpeg not found at ({ffMpegBinaryPath})"); - return; - } - //ttl postage batch if (!string.IsNullOrEmpty(ttlPostageStampStr) && !int.TryParse(ttlPostageStampStr, CultureInfo.InvariantCulture, out ttlPostageStamp)) @@ -237,97 +229,59 @@ static async Task Main(string[] args) } Console.WriteLine($"User {authResult.User.Claims.Where(i => i.Type == EthernaClaimTypes.Username).FirstOrDefault()?.Value} autenticated"); - // Inizialize services. - using var httpClient = new HttpClient(authResult.RefreshTokenHandler) { Timeout = TimeSpan.FromMinutes(30) }; - - //index - var ethernaUserClients = new EthernaUserClients( - new Uri(CommonConsts.EthernaCreditUrl), - new Uri(CommonConsts.EthernaGatewayUrl), - new Uri(CommonConsts.EthernaIndexUrl), - new Uri(CommonConsts.EthernaSsoUrl), - () => httpClient); - - //bee - using var beeNodeClient = new BeeNodeClient( - useBeeNativeNode ? beeNodeUrl : CommonConsts.EthernaGatewayUrl, - useBeeNativeNode ? beeNodeApiPort : CommonConsts.EthernaGatewayPort, - useBeeNativeNode ? beeNodeDebugPort : null, - CommonConsts.BeeNodeGatewayVersion, - CommonConsts.BeeNodeDebugVersion, - httpClient); + // Check for new version + var newVersionAvaiable = await EthernaVersionControl.CheckNewVersionAsync(); + if (newVersionAvaiable && !ignoreNewVersionsOfImporter) + return; - //gateway - IGatewayService gatewayService = useBeeNativeNode ? - new BeeGatewayService(beeNodeClient.GatewayClient!) : - new EthernaGatewayService( - beeNodeClient.GatewayClient!, - ethernaUserClients.GatewayClient); + // Setup DI. + var services = new ServiceCollection(); - //video uploader service - var videoUploaderService = new VideoUploaderService( - gatewayService, - ethernaUserClients.IndexClient, - userEthAddr, - TimeSpan.FromDays(ttlPostageStamp), - acceptPurchaseOfAllBatches); - var encoderService = new EncoderService(ffMpegBinaryPath, ffMpegHWAccelerationType); + //configure options + services.Configure(options => + { + options.MdSourceFolderPath = mdSourceFolderPath; + }); + services.AddSingleton, MdVideoProviderOptionsValidation>(); - // Migration service. - var migrationService = new MigrationService(); + //add services + services.AddCoreServices( + encoderOptions => + { + if (customFFMpegFolderPath is not null) + encoderOptions.FFMpegFolderPath = customFFMpegFolderPath; - // Check for new version - var newVersionAvaiable = await EthernaVersionControl.CheckNewVersionAsync(); - if (newVersionAvaiable && !ignoreNewVersionOfImporter) - return; + encoderOptions.IncludeAudioTrack = includeAudioTrack; + encoderOptions.Skip1440 = skip1440; + encoderOptions.Skip1080 = skip1080; + encoderOptions.Skip720 = skip720; + encoderOptions.Skip480 = skip480; + encoderOptions.Skip360 = skip360; + }, + uploaderOptions => + { + uploaderOptions.AcceptPurchaseOfAllBatches = acceptPurchaseOfAllBatches; + uploaderOptions.TtlPostageStamp = TimeSpan.FromSeconds(ttlPostageStamp); + uploaderOptions.UserEthAddr = userEthAddr!; + }, + useBeeNativeNode, + authResult.RefreshTokenHandler); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); - // Call runner. - var importer = new EthernaVideoImporter( - new CleanerVideoService( - ethernaUserClients.IndexClient, - gatewayService), - gatewayService, - ethernaUserClients.IndexClient, - new DevconLinkReporterService(mdSourceFolderPath), - new MdVideoProvider( - mdSourceFolderPath, - encoderService, - includeAudioTrack, - GetSupportedHeightResolutions(skip1440, skip1080, skip720, skip480, skip360)), - videoUploaderService, - migrationService); + var serviceProvider = services.BuildServiceProvider(); + // Start importer. + var importer = serviceProvider.GetRequiredService(); await importer.RunAsync( - userEthAddr, + deleteExogenousVideos, + deleteVideosMissingFromSource, + forceVideoUpload, offerVideos, pinVideos, - deleteVideosMissingFromSource, - deleteExogenousVideos, - unpinRemovedVideos, - forceUploadVideo); - } - - // Helpers. - private static IEnumerable GetSupportedHeightResolutions( - bool skip1440, - bool skip1080, - bool skip720, - bool skip480, - bool skip360) - { - var supportedHeightResolutions = new List(); - if (!skip1440) - supportedHeightResolutions.Add(1440); - if (!skip1080) - supportedHeightResolutions.Add(1080); - if (!skip720) - supportedHeightResolutions.Add(720); - if (!skip480) - supportedHeightResolutions.Add(480); - if (!skip360) - supportedHeightResolutions.Add(360); - - return supportedHeightResolutions; + userEthAddr, + unpinRemovedVideos); } } } diff --git a/src/EthernaVideoImporter.Devcon/Services/DevconLinkReporterService.cs b/src/EthernaVideoImporter.Devcon/Services/DevconLinkReporterService.cs deleted file mode 100644 index d2aa9d3..0000000 --- a/src/EthernaVideoImporter.Devcon/Services/DevconLinkReporterService.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2022-present Etherna Sagl -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Etherna.VideoImporter.Core; -using Etherna.VideoImporter.Core.Services; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - -namespace Etherna.VideoImporter.Devcon.Services -{ - public sealed class DevconLinkReporterService : ILinkReporterService - { - // Fields. - private const string EthernaIndexPrefix = "ethernaIndex:"; - private const string EthernaPermalinkPrefix = "ethernaPermalink:"; - private readonly string mdSourceFolderPath; - - public DevconLinkReporterService(string mdSourceFolderPath) - { - this.mdSourceFolderPath = mdSourceFolderPath; - } - - // Methods. - public async Task SetEthernaReferencesAsync( - string sourceVideoId, - string ethernaIndexId, - string ethernaPermalinkHash) - { - var filePath = Path.Combine(mdSourceFolderPath, sourceVideoId); - var ethernaIndexUrl = CommonConsts.EthernaIndexContentUrlPrefix + ethernaIndexId; - var ethernaPermalinkUrl = CommonConsts.EthernaPermalinkContentUrlPrefix + ethernaPermalinkHash; - - // Reaad all line. - var lines = File.ReadLines(filePath).ToList(); - - // Set ethernaIndex. - var index = GetLineNumber(lines, EthernaIndexPrefix); - var ethernaIndexLine = $"{EthernaIndexPrefix} \"{ethernaIndexUrl}\""; - if (index >= 0) - lines[index] = ethernaIndexLine; - else - lines.Insert(GetIndexOfInsertLine(lines.Count), ethernaIndexLine); - - // Set ethernaPermalink. - index = GetLineNumber(lines, EthernaPermalinkPrefix); - var ethernaPermalinkLine = $"{EthernaPermalinkPrefix} \"{ethernaPermalinkUrl}\""; - if (index >= 0) - lines[index] = ethernaPermalinkLine; - else - lines.Insert(GetIndexOfInsertLine(lines.Count), ethernaPermalinkLine); - - // Save file. - await File.WriteAllLinesAsync(filePath, lines); - } - - // Helpers. - private int GetLineNumber(List lines, string prefix) - { - var lineIndex = 0; - foreach (var line in lines) - { - if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) - return lineIndex; - - lineIndex++; - } - return -1; - } - - private int GetIndexOfInsertLine(int lines) - { - // Last position. (Exclueded final ---) - return lines - 2; - } - } -} diff --git a/src/EthernaVideoImporter.Devcon/Services/MdVideoProvider.cs b/src/EthernaVideoImporter.Devcon/Services/MdVideoProvider.cs index 8df6774..1a26afb 100644 --- a/src/EthernaVideoImporter.Devcon/Services/MdVideoProvider.cs +++ b/src/EthernaVideoImporter.Devcon/Services/MdVideoProvider.cs @@ -12,125 +12,81 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Etherna.VideoImporter.Core; using Etherna.VideoImporter.Core.Models.Domain; using Etherna.VideoImporter.Core.Services; using Etherna.VideoImporter.Core.Utilities; using Etherna.VideoImporter.Devcon.Models.Domain; using Etherna.VideoImporter.Devcon.Models.MdDto; +using Etherna.VideoImporter.Devcon.Options; +using Microsoft.Extensions.Options; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; -using System.Text; -using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading.Tasks; -using YoutubeExplode; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; using YoutubeExplode.Exceptions; using YoutubeExplode.Videos.Streams; namespace Etherna.VideoImporter.Devcon.Services { - internal sealed class MdVideoProvider : IVideoProvider + internal sealed partial class MdVideoProvider : IVideoProvider { // Consts. - public static readonly string[] _keywordNames = { "IMAGE", "IMAGEURL", "EDITION", "TITLE", "DESCRIPTION", "YOUTUBEURL", "IPFSHASH", "DURATION", "EXPERTISE", "TYPE", "TRACK", "KEYWORDS", "TAGS", "SPEAKERS", "ETHERNAINDEX", "ETHERNAPERMALINK", "SOURCEID" }; - public static readonly string[] _keywordSkips = { "IMAGE", "IMAGEURL", "IPFSHASH", "EXPERTISE", "TRACK", "KEYWORDS", "TAGS", "SPEAKERS", "SOURCEID" }; + private const string EthernaIndexPrefix = "ethernaIndex:"; + private const string EthernaPermalinkPrefix = "ethernaPermalink:"; + + [GeneratedRegex("---\\r?\\n[\\s\\S]*?---")] + private static partial Regex YamlRegex(); // Fields. public static readonly string[] _keywordForArrayString = Array.Empty(); - private readonly bool includeAudioTrack; - private readonly IEnumerable supportedHeightResolutions; - private readonly string mdFolderRootPath; - private readonly YoutubeClient youtubeClient; + private readonly MdVideoProviderOptions options; private readonly IYoutubeDownloader youtubeDownloader; // Constructor. public MdVideoProvider( - string mdFolderRootPath, - IEncoderService encoderService, - bool includeAudioTrack, - IEnumerable supportedHeightResolutions) + IOptions options, + IYoutubeDownloader youtubeDownloader) { - this.mdFolderRootPath = mdFolderRootPath; - this.includeAudioTrack = includeAudioTrack; - this.supportedHeightResolutions = supportedHeightResolutions; - youtubeClient = new(); - youtubeDownloader = new YoutubeDownloader(encoderService, youtubeClient); + this.options = options.Value; + this.youtubeDownloader = youtubeDownloader; } // Properties. - public string SourceName => mdFolderRootPath; + public string SourceName => options.MdSourceFolderPath; // Methods. - public Task