From c88c74cb70cfbb709d9bcd9aa98376b178826a37 Mon Sep 17 00:00:00 2001 From: Mirko Da Corte Date: Sun, 26 Feb 2023 00:29:54 +0100 Subject: [PATCH 1/2] start upload service refactor --- .../Dtos/ManifestDto.cs | 30 +++++++++++++++++-- .../Dtos/ManifestThumbnailDto.cs | 20 ++++++++++++- src/EthernaVideoImporter.Core/Models/Video.cs | 12 +++++++- .../Models/VideoMetadataBase.cs | 3 ++ .../Models/YouTubeVideoMetadataBase.cs | 3 +- .../Services/VideoUploaderService.cs | 8 ++--- .../Utilities/YoutubeDownloader.cs | 2 +- .../Models/MdFileVideoMetadata.cs | 3 +- .../Services/MdVideoProvider.cs | 10 +++++-- .../Models/YouTubeVideoMetadata.cs | 3 +- .../Services/YouTubeChannelVideoProvider.cs | 6 ++++ 11 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/EthernaVideoImporter.Core/Dtos/ManifestDto.cs b/src/EthernaVideoImporter.Core/Dtos/ManifestDto.cs index cef1023..57100ed 100644 --- a/src/EthernaVideoImporter.Core/Dtos/ManifestDto.cs +++ b/src/EthernaVideoImporter.Core/Dtos/ManifestDto.cs @@ -12,20 +12,45 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Etherna.VideoImporter.Core.Models; +using System; using System.Collections.Generic; +using System.Linq; +using System.Text.Json; namespace Etherna.VideoImporter.Core.Dtos { public sealed class ManifestDto { // Constructors. + public ManifestDto( + Video video, + string batchId, + string ownerAddress) + { + if (video is null) + throw new ArgumentNullException(nameof(video)); + + Title = video.Metadata.Title; + Description = video.Metadata.Description; + OriginalQuality = video.Metadata.OriginVideoQualityLabel; + OwnerAddress = ownerAddress; + Duration = (long)video.Metadata.Duration.TotalSeconds; + Thumbnail = video.ThumbnailFile is null ? null : new ManifestThumbnailDto(video.ThumbnailFile); + Sources = video.EncodedFiles.OfType().Select(vf => new ManifestVideoSourceDto(vf)); + CreatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + UpdatedAt = null; + BatchId = batchId; + PersonalData = JsonSerializer.Serialize(ManifestPersonalDataDto.BuildNew(video.Metadata.Id)); + } + public ManifestDto( string title, string description, string originalQuality, string ownerAddress, long duration, - ManifestThumbnailDto thumbnail, + ManifestThumbnailDto? thumbnail, IEnumerable sources, long createdAt, long? updatedAt, @@ -46,13 +71,12 @@ public ManifestDto( } // Properties. - //public string Id { get; } public string Title { get; } public string Description { get; } public string OriginalQuality { get; } public string OwnerAddress { get; } public long Duration { get; } - public ManifestThumbnailDto Thumbnail { get; } + public ManifestThumbnailDto? Thumbnail { get; } public IEnumerable Sources { get; } public long CreatedAt { get; } public long? UpdatedAt { get; } diff --git a/src/EthernaVideoImporter.Core/Dtos/ManifestThumbnailDto.cs b/src/EthernaVideoImporter.Core/Dtos/ManifestThumbnailDto.cs index ee44d47..ca006cc 100644 --- a/src/EthernaVideoImporter.Core/Dtos/ManifestThumbnailDto.cs +++ b/src/EthernaVideoImporter.Core/Dtos/ManifestThumbnailDto.cs @@ -12,13 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Blurhash.SkiaSharp; +using Etherna.VideoImporter.Core.Models; +using SkiaSharp; +using System; using System.Collections.Generic; +using System.IO; namespace Etherna.VideoImporter.Core.Dtos { public sealed class ManifestThumbnailDto { // Constructors. + public ManifestThumbnailDto(ThumbnailFile thumbnailFile) + { + if (thumbnailFile?.UploadedHashReference is null) + throw new ArgumentNullException(nameof(thumbnailFile)); + + using var thumbFileStream = File.OpenRead(thumbnailFile.DownloadedFilePath); + using var thumbManagedStream = new SKManagedStream(thumbFileStream); + using var thumbBitmap = SKBitmap.Decode(thumbManagedStream); + + AspectRatio = thumbnailFile.Width / thumbnailFile.Height; + Blurhash = Blurhasher.Encode(thumbBitmap, 4, 4); + Sources = new Dictionary() { {$"{thumbBitmap.Width}w", thumbnailFile.UploadedHashReference} }; + } + public ManifestThumbnailDto( float aspectRatio, string blurhash, @@ -33,6 +52,5 @@ public ManifestThumbnailDto( public float AspectRatio { get; set; } public string Blurhash { get; set; } public IDictionary Sources { get; } - public string V { get; } = "1.0"; } } diff --git a/src/EthernaVideoImporter.Core/Models/Video.cs b/src/EthernaVideoImporter.Core/Models/Video.cs index ccfc5ef..3653a16 100644 --- a/src/EthernaVideoImporter.Core/Models/Video.cs +++ b/src/EthernaVideoImporter.Core/Models/Video.cs @@ -1,4 +1,8 @@ -using System.Collections.Generic; +using Etherna.VideoImporter.Core.Dtos; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; namespace Etherna.VideoImporter.Core.Models { @@ -10,6 +14,9 @@ public Video( IEnumerable encodedFiles, ThumbnailFile? thumbnailFile) { + if (!encodedFiles.Any()) + throw new ArgumentException("Must exist at least a stream"); + EncodedFiles = encodedFiles; Metadata = metadata; ThumbnailFile = thumbnailFile; @@ -21,5 +28,8 @@ public Video( public string? EthernaPermalinkHash { get; set; } public VideoMetadataBase Metadata { get; } public ThumbnailFile? ThumbnailFile { get; } + public long TotalByteSize => EncodedFiles.Sum(f => f.ByteSize) + + ThumbnailFile?.ByteSize ?? 0 + + JsonSerializer.Serialize(new ManifestDto(this, "0000000000000000000000000000000000000000000000000000000000000000", "0x0000000000000000000000000000000000000000")).Length; } } diff --git a/src/EthernaVideoImporter.Core/Models/VideoMetadataBase.cs b/src/EthernaVideoImporter.Core/Models/VideoMetadataBase.cs index 02168b9..471c1be 100644 --- a/src/EthernaVideoImporter.Core/Models/VideoMetadataBase.cs +++ b/src/EthernaVideoImporter.Core/Models/VideoMetadataBase.cs @@ -22,10 +22,12 @@ public abstract class VideoMetadataBase protected VideoMetadataBase( string description, TimeSpan duration, + string originVideoQualityLabel, string title) { Description = description; Duration = duration; + OriginVideoQualityLabel = originVideoQualityLabel; Title = title; } @@ -33,6 +35,7 @@ protected VideoMetadataBase( public abstract string Id { get; } public string Description { get; } public TimeSpan Duration { get; } + public string OriginVideoQualityLabel { get; } public string Title { get; } } } diff --git a/src/EthernaVideoImporter.Core/Models/YouTubeVideoMetadataBase.cs b/src/EthernaVideoImporter.Core/Models/YouTubeVideoMetadataBase.cs index 1e7f3db..ce4a36c 100644 --- a/src/EthernaVideoImporter.Core/Models/YouTubeVideoMetadataBase.cs +++ b/src/EthernaVideoImporter.Core/Models/YouTubeVideoMetadataBase.cs @@ -11,10 +11,11 @@ public abstract class YouTubeVideoMetadataBase : VideoMetadataBase protected YouTubeVideoMetadataBase( string description, TimeSpan duration, + string originVideoQualityLabel, Thumbnail? thumbnail, string title, string youtubeUrl) - : base(description, duration, title) + : base(description, duration, originVideoQualityLabel, title) { Thumbnail = thumbnail; YoutubeUrl = youtubeUrl; diff --git a/src/EthernaVideoImporter.Core/Services/VideoUploaderService.cs b/src/EthernaVideoImporter.Core/Services/VideoUploaderService.cs index e6da695..63a9b66 100644 --- a/src/EthernaVideoImporter.Core/Services/VideoUploaderService.cs +++ b/src/EthernaVideoImporter.Core/Services/VideoUploaderService.cs @@ -75,14 +75,12 @@ public async Task UploadVideoAsync( { if (video is null) throw new ArgumentNullException(nameof(video)); - if (!video.EncodedFiles.Any()) - throw new ArgumentException("Must exists at least an encoded file", nameof(video)); // Create new batch. - Console.WriteLine("Create batch..."); + Console.WriteLine("Creating batch..."); - // Size of all video to upload. - var totalSize = video.EncodedFiles.Sum(v => v.ByteSize); + //calculate size of all contents to upload. + var totalSize = video.TotalByteSize; // Calculate batch deep. var batchDeep = 17; diff --git a/src/EthernaVideoImporter.Core/Utilities/YoutubeDownloader.cs b/src/EthernaVideoImporter.Core/Utilities/YoutubeDownloader.cs index 316adc9..4c34953 100644 --- a/src/EthernaVideoImporter.Core/Utilities/YoutubeDownloader.cs +++ b/src/EthernaVideoImporter.Core/Utilities/YoutubeDownloader.cs @@ -39,7 +39,7 @@ public async Task