diff --git a/nuget.config b/nuget.config new file mode 100644 index 0000000..dd4edcc --- /dev/null +++ b/nuget.config @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/src/EthernaVideoImporter/.editorconfig b/src/EthernaVideoImporter/.editorconfig new file mode 100644 index 0000000..276bdf5 --- /dev/null +++ b/src/EthernaVideoImporter/.editorconfig @@ -0,0 +1,114 @@ +# Remove the line below if you want to inherit .editorconfig settings from higher directories +root = true + +#### Define style #### + +# All files +[*] +indent_style = space + +# C# Project, JS and CSS files +[*.{csproj,js,ts,css,scss}] +indent_size = 2 + +#### Suppress warnings #### + +# C# files +[*.cs] + +# CA1034: Nested types should not be visible +dotnet_diagnostic.CA1034.severity = none # Asp.Net Core Pages uses nested models + +# CA1054: Uri parameters should not be strings +dotnet_diagnostic.CA1054.severity = none # Asp.Net Core Pages uses strings natively + +# CA1056: Uri properties should not be strings +dotnet_diagnostic.CA1056.severity = none # Asp.Net Core Pages uses strings natively + +# CA1303: Do not pass literals as localized parameters +dotnet_diagnostic.CA1303.severity = none # Don't need translated exceptions + +# CA1707: Identifiers should not contain underscores +dotnet_diagnostic.CA1707.severity = none # I like underscores into constants name + +# CA1812: Avoid uninstantiated internal classes +dotnet_diagnostic.CA1812.severity = none # Doing extensive use of Dependency Injection + +# CA1822: Mark members as static +dotnet_diagnostic.CA1822.severity = none # Don't like static members + +# CA2007: Consider calling ConfigureAwait on the awaited task +dotnet_diagnostic.CA2007.severity = none # Not needed with .Net Core. More info https://devblogs.microsoft.com/dotnet/configureawait-faq/ + +# CS1591: Missing XML comment for publicly visible type or member +dotnet_diagnostic.CS1591.severity = none +csharp_indent_labels = one_less_than_current +csharp_using_directive_placement = outside_namespace:silent +csharp_prefer_simple_using_statement = true:suggestion +csharp_prefer_braces = true:silent +csharp_style_namespace_declarations = block_scoped:silent +csharp_style_prefer_method_group_conversion = true:silent +csharp_style_prefer_top_level_statements = true:silent +csharp_style_expression_bodied_methods = false:silent +csharp_style_expression_bodied_constructors = false:silent +csharp_style_expression_bodied_operators = false:silent +csharp_style_expression_bodied_properties = true:silent +csharp_style_expression_bodied_indexers = true:silent +csharp_style_expression_bodied_accessors = true:silent +csharp_style_expression_bodied_lambdas = true:silent +csharp_style_expression_bodied_local_functions = false:silent +[*.{cs,vb}] +#### Naming styles #### + +# Naming rules + +dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion +dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface +dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i + +dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.types_should_be_pascal_case.symbols = types +dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case + +dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members +dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case + +# Symbol specifications + +dotnet_naming_symbols.interface.applicable_kinds = interface +dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.interface.required_modifiers = + +dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum +dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.types.required_modifiers = + +dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method +dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected +dotnet_naming_symbols.non_field_members.required_modifiers = + +# Naming styles + +dotnet_naming_style.begins_with_i.required_prefix = I +dotnet_naming_style.begins_with_i.required_suffix = +dotnet_naming_style.begins_with_i.word_separator = +dotnet_naming_style.begins_with_i.capitalization = pascal_case + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case + +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.capitalization = pascal_case +dotnet_style_operator_placement_when_wrapping = beginning_of_line +tab_width = 4 +indent_size = 4 +end_of_line = crlf +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_auto_properties = true:silent diff --git a/src/EthernaVideoImporter/Dtos/MetadataVideoDto.cs b/src/EthernaVideoImporter/Dtos/MetadataVideoDto.cs new file mode 100644 index 0000000..b41cf63 --- /dev/null +++ b/src/EthernaVideoImporter/Dtos/MetadataVideoDto.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; + +namespace Etherna.EthernaVideoImporter.Dtos +{ + internal class MetadataVideoDto + { + // Constructors. + public MetadataVideoDto( + string? batchId, + string description, + long duration, + long createdAt, + string originalQuality, + string ownerAddress, + IEnumerable sources, + SwarmImageRaw? thumbnail, + string title, + long? updatedAt) + { + BatchId = batchId; + Description = description; + Duration = duration; + CreatedAt = createdAt; + OriginalQuality = originalQuality; + OwnerAddress = ownerAddress; + Sources = sources; + Thumbnail = thumbnail; + Title = title; + UpdatedAt = updatedAt; + } + + // Properties. + public string? BatchId { get; } + public string Description { get; } + public long Duration { get; } + public long CreatedAt { get; } + public string OriginalQuality { get; } + public string OwnerAddress { get; } + public IEnumerable Sources { get; } + public SwarmImageRaw? Thumbnail { get; } + public string Title { get; } + public long? UpdatedAt { get; } + } +} diff --git a/src/EthernaVideoImporter/Dtos/MetadataVideoSource.cs b/src/EthernaVideoImporter/Dtos/MetadataVideoSource.cs new file mode 100644 index 0000000..0e65bc6 --- /dev/null +++ b/src/EthernaVideoImporter/Dtos/MetadataVideoSource.cs @@ -0,0 +1,24 @@ +namespace Etherna.EthernaVideoImporter.Dtos +{ + public class MetadataVideoSource + { + // Constructors. + public MetadataVideoSource( + int bitrate, + string quality, + string reference, + long size) + { + Bitrate = bitrate; + Quality = quality; + Reference = reference; + Size = size; + } + + // Properties. + public int Bitrate { get; } + public string Quality { get; } + public string Reference { get; } + public long Size { get; } + } +} diff --git a/src/EthernaVideoImporter/Dtos/SwarmImageRaw.cs b/src/EthernaVideoImporter/Dtos/SwarmImageRaw.cs new file mode 100644 index 0000000..1645c41 --- /dev/null +++ b/src/EthernaVideoImporter/Dtos/SwarmImageRaw.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace Etherna.EthernaVideoImporter.Dtos +{ + public class SwarmImageRaw + { + // Constructors. + public SwarmImageRaw( + float aspectRatio, + string blurhash, + IReadOnlyDictionary sources) + { + AspectRatio = aspectRatio; + Blurhash = blurhash; + Sources = sources; + } + + // Properties. + public float AspectRatio { get; } + public string Blurhash { get; } + public IReadOnlyDictionary Sources { get; } + } +} diff --git a/src/EthernaVideoImporter/Dtos/VideoDataInfoDto.cs b/src/EthernaVideoImporter/Dtos/VideoDataInfoDto.cs index 802d60c..a9e5640 100644 --- a/src/EthernaVideoImporter/Dtos/VideoDataInfoDto.cs +++ b/src/EthernaVideoImporter/Dtos/VideoDataInfoDto.cs @@ -1,4 +1,5 @@ using CsvHelper.Configuration.Attributes; +using System.Collections.Generic; namespace EthernaVideoImporter.Dtos { @@ -7,6 +8,7 @@ internal class VideoDataInfoDto public string? Description { get; set; } [Optional] public string? DownloadedFileName { get; set; } + public string? DownloadedFilePath { get; set; } public int Duration { get; set; } public int Edition { get; set; } public string? Expertise { get; set; } diff --git a/src/EthernaVideoImporter/Dtos/VideoDownloadInfo.cs b/src/EthernaVideoImporter/Dtos/VideoDownloadInfo.cs new file mode 100644 index 0000000..80f6b06 --- /dev/null +++ b/src/EthernaVideoImporter/Dtos/VideoDownloadInfo.cs @@ -0,0 +1,21 @@ +namespace Etherna.EthernaVideoImporter.Dtos +{ + public class VideoDownloadInfo + { + // Constructors. + public VideoDownloadInfo( + int bitrate, + string quality, + long size) + { + Bitrate = bitrate; + Quality = quality; + Size = size; + } + + // Properties. + public int Bitrate { get; } + public string Quality { get; } + public long Size { get; } + } +} diff --git a/src/EthernaVideoImporter/EthernaVideoImporter.csproj b/src/EthernaVideoImporter/EthernaVideoImporter.csproj index 20c9fda..83ab80e 100644 --- a/src/EthernaVideoImporter/EthernaVideoImporter.csproj +++ b/src/EthernaVideoImporter/EthernaVideoImporter.csproj @@ -1,14 +1,28 @@ - + Exe net6.0 - enable - enable + true + Etherna.EthernaVideoImporter + + Etherna Sagl + A .Net console for upload video on Etherna + + 10.0 + enable + true + AllEnabledByDefault + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/EthernaVideoImporter/Program.cs b/src/EthernaVideoImporter/Program.cs index 5f4b3dd..f592dd0 100644 --- a/src/EthernaVideoImporter/Program.cs +++ b/src/EthernaVideoImporter/Program.cs @@ -1,7 +1,18 @@ using CsvHelper; +using Etherna.BeeNet; +using Etherna.BeeNet.Clients.DebugApi; +using Etherna.BeeNet.Clients.GatewayApi; +using Etherna.EthernaVideoImporter.Dtos; +using Etherna.EthernaVideoImporter.Services; using EthernaVideoImporter.Dtos; using EthernaVideoImporter.Services; +using System; +using System.Collections.Generic; using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading.Tasks; using YoutubeDownloader.Clients; internal class Program @@ -11,7 +22,7 @@ static async Task Main(string[] args) string tmpFolder = "tmpData"; if (args is null || - args.Length < 0) + args.Length < 1) { Console.WriteLine("Missing read path"); return; @@ -21,6 +32,28 @@ static async Task Main(string[] args) Console.WriteLine($"File not found {args[0]}"); return; } + if (args.Length < 2) + { + Console.WriteLine("Missing beenode url"); + return; + } + if (args.Length < 3) + { + Console.WriteLine("Missing beenode port"); + return; + } + if (args.Length < 4) + { + Console.WriteLine("Missing beenode version"); + return; + } + var beeNodeUrl = args[1]; +#pragma warning disable CA1305 // Specify IFormatProvider + var beeNodePort = Convert.ToInt32(args[2]); +#pragma warning restore CA1305 // Specify IFormatProvider + var beeNodeDebugPort = beeNodePort + 1; + var beeNodeVersion = GatewayApiVersion.v3_0_2; + var beeNodeDebugVersion = DebugApiVersion.v3_0_2; if (!Directory.Exists(tmpFolder)) Directory.CreateDirectory(tmpFolder); @@ -36,8 +69,9 @@ static async Task Main(string[] args) Console.WriteLine($"Csv with {totalVideo} items to upload"); // Call import service for each video. - var youtubeDownloadClient = new YoutubeDownloadClient(); - var videoImporterService = new VideoImporterService(youtubeDownloadClient, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, tmpFolder)); + var tmpFolderFullPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, tmpFolder); + var videoImporterService = new VideoImporterService(new YoutubeDownloadClient(), tmpFolderFullPath); + var videoUploaderService = new VideoUploaderService(new BeeNodeClient(beeNodeUrl, beeNodePort, beeNodeDebugPort, beeNodeVersion, beeNodeDebugVersion), tmpFolderFullPath); var videoCount = 0; foreach (var videoInfo in videoDataInfoDtos) @@ -45,20 +79,30 @@ static async Task Main(string[] args) try { Console.WriteLine($"Start processing video {++videoCount} of {totalVideo}"); - await videoImporterService.Start(videoInfo); + + // Download from youtube. + var downloadInfo = await videoImporterService.Start(videoInfo); + + if (downloadInfo != null) + // Upload on bee. + await videoUploaderService.Start(videoInfo, downloadInfo); + Console.WriteLine($"Video #{videoCount} processed"); } +#pragma warning disable CA1031 // Do not catch general exception types catch (Exception ex) +#pragma warning restore CA1031 // Do not catch general exception types { Console.WriteLine($"{ex.Message} \n Unable to upload: {videoDataInfoDtos}"); videoInfo.VideoStatusNote = ex.Message; } + finally + { + // Save csv with results at every cycle. + using (var writer = new StreamWriter(args[0])) + using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture)) + await csv.WriteRecordsAsync(videoDataInfoDtos).ConfigureAwait(false); + } } - - // Save csv with results. - using (var writer = new StreamWriter(args[0])) - using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture)) - await csv.WriteRecordsAsync(videoDataInfoDtos); - } } \ No newline at end of file diff --git a/src/EthernaVideoImporter/Services/VideoImporterService.cs b/src/EthernaVideoImporter/Services/VideoImporterService.cs index a317899..2e6aea7 100644 --- a/src/EthernaVideoImporter/Services/VideoImporterService.cs +++ b/src/EthernaVideoImporter/Services/VideoImporterService.cs @@ -1,4 +1,10 @@ -using EthernaVideoImporter.Dtos; +using Etherna.EthernaVideoImporter.Dtos; +using EthernaVideoImporter.Dtos; +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading.Tasks; using YoutubeDownloader.Clients; namespace EthernaVideoImporter.Services @@ -18,45 +24,56 @@ public VideoImporterService( } // Public methods. - public async Task Start(VideoDataInfoDto videoDataInfoDto) + public async Task Start(VideoDataInfoDto videoDataInfoDto) { if (videoDataInfoDto.VideoStatus == VideoStatus.Processed) - return; + return null; if (string.IsNullOrWhiteSpace(videoDataInfoDto.YoutubeUrl)) throw new InvalidOperationException("Invalid YoutubeUrl"); - if (string.IsNullOrWhiteSpace(videoDataInfoDto.DownloadedFileName)) + try { // Take best video resolution. var videos = await youtubeDownloadClient.GetAllVideosAsync(videoDataInfoDto.YoutubeUrl); var videoWithAudio = videos - .Where(i => i.AudioBitrate != -1); + .Where(i => i.AudioBitrate != -1); var videoDownload = videoWithAudio .First(i => i.AudioBitrate == videoWithAudio.Max(j => j.AudioBitrate)); // Take best resolution Console.WriteLine($"Resolution: {videoDownload.Resolution}\tAudio Bitrate: {videoDownload.AudioBitrate}"); - //Start download + // Start download. videoDataInfoDto.VideoStatus = VideoStatus.Downloading; + videoDataInfoDto.DownloadedFilePath = Path.Combine(tmpFolder, videoDownload.FullName); await youtubeDownloadClient .CreateDownloadAsync( new Uri(videoDownload.Uri), - Path.Combine(tmpFolder, videoDownload.FullName), + videoDataInfoDto.DownloadedFilePath, new Progress>((Tuple v) => { var percent = (int)((v.Item1 * 100) / v.Item2); +#pragma warning disable CA1305 // Specify IFormatProvider Console.Write(string.Format("Downloading.. ( % {0} ) {1} / {2} MB\r", percent, (v.Item1 / (double)(1024 * 1024)).ToString("N"), (v.Item2 / (double)(1024 * 1024)).ToString("N"))); +#pragma warning restore CA1305 // Specify IFormatProvider })); Console.WriteLine(""); videoDataInfoDto.DownloadedFileName = videoDownload.FullName; videoDataInfoDto.VideoStatus = VideoStatus.Downloaded; videoDataInfoDto.VideoStatusNote = ""; - } - - // - //TODO UPLOAD - videoDataInfoDto.VideoStatus = VideoStatus.Processed; - return; + // Result VideoDownloadInfo. + var fileSize = new FileInfo(videoDataInfoDto.DownloadedFilePath).Length; + return new VideoDownloadInfo( + (int)Math.Ceiling(fileSize / (videoDataInfoDto.Duration / 60 * 0.0075)), //TODO it's OK? bitrate = file size / (number of minutes * .0075) + videoDownload.Resolution.ToString(CultureInfo.InvariantCulture) + "p", + fileSize); + } + catch (Exception) + { + videoDataInfoDto.DownloadedFileName = ""; + videoDataInfoDto.DownloadedFilePath = ""; + videoDataInfoDto.VideoStatus = VideoStatus.Downloading; + throw; + } } } } diff --git a/src/EthernaVideoImporter/Services/VideoUploaderService.cs b/src/EthernaVideoImporter/Services/VideoUploaderService.cs new file mode 100644 index 0000000..ac6363c --- /dev/null +++ b/src/EthernaVideoImporter/Services/VideoUploaderService.cs @@ -0,0 +1,74 @@ +using Etherna.BeeNet; +using Etherna.BeeNet.InputModels; +using Etherna.EthernaVideoImporter.Dtos; +using EthernaVideoImporter.Dtos; +using JavaScriptEngineSwitcher.Core.Extensions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using YoutubeDownloader.Dto; + +namespace Etherna.EthernaVideoImporter.Services +{ + internal class VideoUploaderService + { + private readonly BeeNodeClient beeNodeClient; + private readonly string tmpFolder; + + // Constractor. + public VideoUploaderService( + BeeNodeClient beeNodeClient, + string tmpFolder) + { + if (beeNodeClient is null) + throw new ArgumentNullException(nameof(beeNodeClient)); + if (string.IsNullOrWhiteSpace(tmpFolder)) + throw new ArgumentNullException(nameof(tmpFolder)); + + this.beeNodeClient = beeNodeClient; + this.tmpFolder = tmpFolder; + } + + // Public methods. + public async Task Start(VideoDataInfoDto videoDataInfoDto, VideoDownloadInfo videoDownloadInfo) + { + if (videoDataInfoDto.VideoStatus == VideoStatus.Processed) + return; + if (videoDataInfoDto.VideoStatus != VideoStatus.Downloaded) + throw new InvalidOperationException("Invalid Status"); + + + // Create batch. + var postageBatch = await beeNodeClient.DebugClient!.BuyPostageBatchAsync(1, 28); + await Task.Delay(90000); + + // Upload file. + var fileParameterInput = new FileParameterInput( + File.OpenRead(videoDataInfoDto.DownloadedFilePath!), + Path.GetFileName(videoDataInfoDto.DownloadedFilePath!), + MimeTypes.GetMimeType(Path.GetFileName(videoDataInfoDto.DownloadedFilePath!))); + var reference = await beeNodeClient.GatewayClient!.UploadFileAsync(postageBatch, files: new List { fileParameterInput }, swarmCollection: false); + + // Upload metadata. + var metadataVideoDto = new MetadataVideoDto( + postageBatch, + videoDataInfoDto.Description ?? "", + videoDataInfoDto.Duration, + -1L, //Created at + videoDownloadInfo.Quality, + "owner", + new List { new MetadataVideoSource(videoDownloadInfo.Bitrate, videoDownloadInfo.Quality, reference, videoDownloadInfo.Size) }, + null, + videoDataInfoDto.Title ?? "", + -1); //Updated at + + + videoDataInfoDto.VideoStatus = VideoStatus.Processed; + return; + } + + // Private methods. + + } +}