merge hotfix

README.md

@@ -37,6 +37,10 @@ General Options:
   -a, --auto-purchase     Accept automatically purchase of all batches
   -w, --write-file        Write published videos result to a JSON file
   --dry                   Run in dry mode. Any action on swarm gateway or index is performed read-only
+  
+YouTube Options:
+  --yt-autogen-sub        Import also YouTube autogenerated subtitles
+  --yt-cookies            List of cookies to use with YouTube requests, divided by ';'
 
 Video Management Options:
   -t, --ttl               TTL (days) Postage Stamp (default: 365 days)
@@ -90,14 +94,15 @@ Tool to import videos on Etherna from Devcon archive.
     Program distributed under AGPLv3 license. Copyright since 2022 by Etherna SA.
     You can find source code at: https://github.com/Etherna/etherna-video-importer
 
-Usage:  evid MD_FOLDER [OPTIONS]
+Usage:  evid VIDEO_FOLDER [OPTIONS]
 
 General Options:
   -k, --api-key           Api Key (optional)
   -f, --ffmpeg-path       Path to FFmpeg folder (default: search to <app_dir>/FFmpeg or global install)
   -i, --ignore-update     Ignore new version of EthernaVideoImporter
   -a, --auto-purchase     Accept automatically purchase of all batches
   --dry                   Run in dry mode. Any action on swarm gateway or index is performed read-only
+  --yt-cookies            List of cookies to use with YouTube requests, divided by ';'
 
 Video Management Options:
   -t, --ttl               TTL (days) Postage Stamp (default: 365 days)

src/EthernaVideoImporter.Core/EthernaVideoImporter.Core.csproj

@@ -15,26 +15,22 @@
     <RepositoryUrl>https://github.com/Etherna/etherna-video-importer</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
-    <PackageLicenseFile>LICENSE</PackageLicenseFile>
-    <EmbedUntrackedSources>true</EmbedUntrackedSources>
-    <ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
+    <PackageLicenseFile>COPYING</PackageLicenseFile>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Blurhash.SkiaSharp" Version="2.0.0" />
-    <PackageReference Include="Etherna.YoutubeDownloader.Converter" Version="6.4.4" />
-    <PackageReference Include="EthernaSdk.Users.Gateway" Version="0.4.0-alpha.141" />
-    <PackageReference Include="EthernaSdk.Users.Index" Version="0.4.0-alpha.141" />
+    <PackageReference Include="Etherna.YoutubeDownloader.Converter" Version="6.4.5" />
+    <PackageReference Include="EthernaSdk.Users.Gateway" Version="0.4.0-alpha.154" />
+    <PackageReference Include="EthernaSdk.Users.Index" Version="0.4.0-alpha.154" />
     <PackageReference Include="GitVersion.MsBuild" Version="6.0.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Keccak256" Version="1.0.0" />
     <PackageReference Include="MedallionShell" Version="1.6.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/EthernaVideoImporter.Core/EthernaVideoImporter.cs

@@ -16,6 +16,8 @@
 using Etherna.Authentication.Native;
 using Etherna.BeeNet.Hashing;
 using Etherna.BeeNet.Models;
+using Etherna.Sdk.Tools.Video.Models;
+using Etherna.Sdk.Tools.Video.Services;
 using Etherna.Sdk.Users.Index.Clients;
 using Etherna.Sdk.Users.Index.Models;
 using Etherna.VideoImporter.Core.Models.Domain;
@@ -43,6 +45,7 @@ public class EthernaVideoImporter : IEthernaVideoImporter
         private readonly IIoService ioService;
         private readonly IMigrationService migrationService;
         private readonly IResultReporterService resultReporterService;
+        private readonly IVideoManifestService videoManifestService;
         private readonly IVideoUploaderService videoUploaderService;
         private readonly IVideoProvider videoProvider;
 
@@ -57,6 +60,7 @@ public EthernaVideoImporter(
             IIoService ioService,
             IMigrationService migrationService,
             IResultReporterService resultReporterService,
+            IVideoManifestService videoManifestService,
             IVideoProvider videoProvider,
             IVideoUploaderService videoUploaderService)
         {
@@ -73,6 +77,7 @@ public EthernaVideoImporter(
             this.ioService = ioService;
             this.migrationService = migrationService;
             this.resultReporterService = resultReporterService;
+            this.videoManifestService = videoManifestService;
             this.videoProvider = videoProvider;
             this.videoUploaderService = videoUploaderService;
         }
@@ -143,9 +148,9 @@ public async Task RunAsync(
                         .Select(id => hasher.ComputeHash(id).ToHex());
 
                     var alreadyIndexedVideo = userVideosOnIndex.FirstOrDefault(
-                        indexedVideo => indexedVideo.LastValidManifest?.Manifest?.PersonalData?.SourceVideoId is not null &&
-                                        indexedVideo.LastValidManifest.Manifest.PersonalData.SourceProviderName == videoProvider.SourceName &&
-                                        allVideoIdHashes.Contains(indexedVideo.LastValidManifest.Manifest.PersonalData.SourceVideoId));
+                        indexedVideo => indexedVideo.PersonalData?.SourceVideoId is not null &&
+                                        indexedVideo.PersonalData.SourceProviderName == videoProvider.SourceName &&
+                                        allVideoIdHashes.Contains(indexedVideo.PersonalData.SourceVideoId));
 
                     // Try to minimize operations as much as possible.
                     if (alreadyIndexedVideo is not null && !forceVideoUpload)
@@ -165,7 +170,7 @@ public async Task RunAsync(
                                 importResult = VideoImportResultSucceeded.Skipped(
                                     sourceMetadata,
                                     alreadyIndexedVideo.Id,
-                                    alreadyIndexedVideo.LastValidManifest!.Hash);
+                                    alreadyIndexedVideo.LastValidManifestHash!.Value);
                                 break;
 
                             case OperationType.UpdateManifest:
@@ -354,23 +359,35 @@ private async Task SignInUserAsync()
             bool fundDownload,
             bool fundPinning)
         {
-            if (alreadyIndexedVideo.LastValidManifest?.Manifest is null)
+            if (!alreadyIndexedVideo.LastValidManifestHash.HasValue)
                 return null;
 
+            // Try to get last valid manifest.
+            PublishedVideoManifest lastValidManifest;
+            try
+            {
+                lastValidManifest = await videoManifestService.GetPublishedVideoManifestAsync(
+                    alreadyIndexedVideo.LastValidManifestHash.Value).ConfigureAwait(false);
+            }
+            catch { return null; }
+            if (lastValidManifest.Manifest is null)
+                return null;
+
+            // Get file references.
             Video video;
             try
             {
                 // Download thumbnail.
                 List<ThumbnailFile> thumbnailFiles = [];
-                foreach (var thumbnailSource in alreadyIndexedVideo.LastValidManifest.Manifest.Thumbnail.Sources)
+                foreach (var thumbnailSource in lastValidManifest.Manifest.Thumbnail.Sources)
                     thumbnailFiles.Add(await migrationService.DownloadThumbnailFile(
-                        alreadyIndexedVideo.LastValidManifest.Hash,
+                        lastValidManifest.Hash,
                         thumbnailSource.Uri));
 
                 // Download encoded video.
                 var videoEncoding = await migrationService.DownloadVideoEncodingFromManifestAsync(
-                    alreadyIndexedVideo.LastValidManifest.Hash,
-                    alreadyIndexedVideo.LastValidManifest.Manifest);
+                    lastValidManifest.Hash,
+                    lastValidManifest.Manifest);
 
                 video = new Video(
                     sourceMetadata,
@@ -389,7 +406,7 @@ await videoUploaderService.UploadVideoAsync(
                 fundPinning,
                 fundDownload,
                 userEthAddress,
-                alreadyIndexedVideo.LastValidManifest.Manifest.BatchId);
+                lastValidManifest.Manifest.BatchId);
 
             return video.EthernaPermalinkHash;
         }

src/EthernaVideoImporter.Core/Extensions/IndexedVideoExtensions.cs

@@ -27,15 +27,10 @@ public static bool HasEqualMetadata(this IndexedVideo indexedVideo, VideoMetadat
             ArgumentNullException.ThrowIfNull(hasher, nameof(hasher));
             ArgumentNullException.ThrowIfNull(indexedVideo, nameof(indexedVideo));
             ArgumentNullException.ThrowIfNull(metadata, nameof(metadata));
-
-            if (indexedVideo.LastValidManifest?.Manifest is null)
-                return false;
-
-            var indexedManifest = indexedVideo.LastValidManifest.Manifest;
 
-            return indexedManifest.PersonalData?.SourceVideoId == hasher.ComputeHash(metadata.SourceId).ToHex() &&
-                   indexedManifest.Title == metadata.Title &&
-                   indexedManifest.Description == metadata.Description;
+            return indexedVideo.PersonalData?.SourceVideoId == hasher.ComputeHash(metadata.SourceId).ToHex() &&
+                   indexedVideo.Title == metadata.Title &&
+                   indexedVideo.Description == metadata.Description;
         }
     }
 }

src/EthernaVideoImporter.Core/Services/BeeGatewayService.cs

@@ -13,10 +13,12 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Tools;
 using Etherna.Sdk.Users.Gateway.Clients;
 using Etherna.VideoImporter.Core.Options;
 using Microsoft.Extensions.Options;
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Etherna.VideoImporter.Core.Services
@@ -31,6 +33,12 @@ public class BeeGatewayService(
         public override async Task<BzzBalance> GetChainPriceAsync() =>
              (await BeeClient.GetChainStateAsync()).CurrentPrice;
 
+        public override Task<IChunkWebSocketUploader> GetChunkUploaderWebSocketAsync(
+            PostageBatchId batchId,
+            TagId? tagId = null,
+            CancellationToken cancellationToken = default) =>
+            BeeClient.GetChunkUploaderWebSocketAsync(batchId, tagId, cancellationToken);
+
         public override async Task<bool> IsBatchUsableAsync(PostageBatchId batchId) =>
             (await BeeClient.GetPostageBatchAsync(batchId)).IsUsable;
 

src/EthernaVideoImporter.Core/Services/CleanerVideoService.cs

@@ -47,7 +47,7 @@ public async Task<int> DeleteExogenousVideosAsync(
 
             ioService.WriteLine($"Start removing videos not generated with this tool");
 
-            var exogenousVideos = indexedVideos.Where(v => v.LastValidManifest?.Manifest?.PersonalData?.ClientName != CommonConsts.ImporterIdentifier);
+            var exogenousVideos = indexedVideos.Where(v => v.PersonalData?.ClientName != CommonConsts.ImporterIdentifier);
 
             var deindexedVideos = 0;
             foreach (var video in exogenousVideos)
@@ -83,11 +83,11 @@ public async Task<int> DeleteVideosRemovedFromSourceAsync(
             // - The indexed source id is not listed into any current imported source id  (has been removed from source)
             var sourceRemovedVideos = indexedVideos
                 .Where(indexedVideo =>
-                    indexedVideo.LastValidManifest?.Manifest?.PersonalData?.ClientName == CommonConsts.ImporterIdentifier &&
-                    indexedVideo.LastValidManifest.Manifest.PersonalData.SourceProviderName == sourceProviderName &&
+                    indexedVideo.PersonalData?.ClientName == CommonConsts.ImporterIdentifier &&
+                    indexedVideo.PersonalData.SourceProviderName == sourceProviderName &&
                     !videosMetadataFromSource.Any(sourceMeta => sourceMeta.AllSourceIds
                         .Select(id => hasher.ComputeHash(id).ToHex()) //get hashed version of all source Ids
-                        .Contains(indexedVideo.LastValidManifest!.Manifest.PersonalData!.SourceVideoId)));
+                        .Contains(indexedVideo.PersonalData!.SourceVideoId)));
 
             var deindexedVideos = 0;
             foreach (var sourceRemovedVideo in sourceRemovedVideos)
@@ -131,8 +131,8 @@ private async Task RemoveFromIndexAsync(
             // Unpin manifest.
             if (removeSucceeded &&
                 unpinRemovedVideos &&
-                indexedVideo.LastValidManifest is not null)
-                await TryDefundPinningAsync(indexedVideo.LastValidManifest.Hash);
+                indexedVideo.LastValidManifestHash is not null)
+                await TryDefundPinningAsync(indexedVideo.LastValidManifestHash.Value);
         }
 
         private async Task TryDefundPinningAsync(SwarmHash hash)

src/EthernaVideoImporter.Core/Services/EthernaGatewayService.cs

@@ -13,11 +13,13 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Tools;
 using Etherna.Sdk.Gateway.GenClients;
 using Etherna.Sdk.Users.Gateway.Clients;
 using Etherna.VideoImporter.Core.Options;
 using Microsoft.Extensions.Options;
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Etherna.VideoImporter.Core.Services
@@ -30,11 +32,18 @@ public class EthernaGatewayService(
     {
         // Const.
         private readonly TimeSpan BatchCreationTimeout = new(0, 0, 10, 0);
+        private ushort ChunkBatchMaxSize = 1000;
 
         // Methods.
         public override async Task<BzzBalance> GetChainPriceAsync() =>
             (await ethernaGatewayClient.GetChainStateAsync()).CurrentPrice;
 
+        public override Task<IChunkWebSocketUploader> GetChunkUploaderWebSocketAsync(
+            PostageBatchId batchId,
+            TagId? tagId = null,
+            CancellationToken cancellationToken = default) =>
+            ethernaGatewayClient.GetChunkTurboUploaderWebSocketAsync(batchId, tagId, ChunkBatchMaxSize, cancellationToken);
+
         public override async Task<bool> IsBatchUsableAsync(PostageBatchId batchId) =>
             (await ethernaGatewayClient.GetPostageBatchAsync(batchId)).IsUsable;
 

src/EthernaVideoImporter.Core/Services/GatewayServiceBase.cs

@@ -14,6 +14,7 @@
 
 using Etherna.BeeNet;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Tools;
 using Etherna.VideoImporter.Core.Options;
 using Microsoft.Extensions.Options;
 using System;
@@ -67,11 +68,10 @@ public Task FundResourcePinningAsync(SwarmHash hash) => options.IsDryRun ?
 
         public abstract Task<BzzBalance> GetChainPriceAsync();
 
-        public Task<ChunkUploaderWebSocket> GetChunkUploaderWebSocketAsync(
+        public abstract Task<IChunkWebSocketUploader> GetChunkUploaderWebSocketAsync(
             PostageBatchId batchId,
             TagId? tagId = null,
-            CancellationToken cancellationToken = default) =>
-            BeeClient.GetChunkUploaderWebSocketAsync(batchId, tagId, cancellationToken);
+            CancellationToken cancellationToken = default);
 
         public abstract Task<bool> IsBatchUsableAsync(PostageBatchId batchId);
 

src/EthernaVideoImporter.Core/Services/IGatewayService.cs

@@ -14,6 +14,7 @@
 
 using Etherna.BeeNet;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Tools;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -53,7 +54,7 @@ public interface IGatewayService
         /// </summary>
         Task<BzzBalance> GetChainPriceAsync();
 
-        Task<ChunkUploaderWebSocket> GetChunkUploaderWebSocketAsync(
+        Task<IChunkWebSocketUploader> GetChunkUploaderWebSocketAsync(
             PostageBatchId batchId,
             TagId? tagId = null,
             CancellationToken cancellationToken = default);

src/EthernaVideoImporter.Core/Services/MigrationService.cs

@@ -40,16 +40,15 @@ public OperationType DecideOperation(IndexedVideo alreadyIndexedVideo, VideoMeta
         {
             ArgumentNullException.ThrowIfNull(alreadyIndexedVideo, nameof(alreadyIndexedVideo));
 
-            var personalData = alreadyIndexedVideo.LastValidManifest?.Manifest?.PersonalData;
-
             // If client version is missing (0.1.x or 0.2.x).
-            if (string.IsNullOrWhiteSpace(personalData?.ClientVersion))
+            if (string.IsNullOrWhiteSpace(alreadyIndexedVideo.PersonalData?.ClientVersion))
                 return OperationType.ImportAll;
 
-            return new Version(personalData.ClientVersion) switch
+            var version = new Version(alreadyIndexedVideo.PersonalData.ClientVersion);
+            return version switch
             {
                 { Major: 0, Minor: <= 2 } => OperationType.ImportAll,
-                { Major: 0, Minor: 3, Revision: < 9} => OperationType.ImportAll,
+                { Major: 0, Minor: 3, Build: <= 8} => OperationType.ImportAll,
                 _ => alreadyIndexedVideo.HasEqualMetadata(sourceMetadata, hasher) ?
                     OperationType.Skip : OperationType.UpdateManifest
             };

src/EthernaVideoImporter.Devcon/EthernaVideoImporter.Devcon.csproj

@@ -19,11 +19,7 @@
     <RepositoryUrl>https://github.com/Etherna/etherna-video-importer</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
-    <PackageLicenseFile>LICENSE</PackageLicenseFile>
-    <EmbedUntrackedSources>true</EmbedUntrackedSources>
-    <ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
+    <PackageLicenseFile>COPYING</PackageLicenseFile>
     <Configurations>Debug;Release;Debug-DevEnv</Configurations>
     <Platforms>AnyCPU</Platforms>
   </PropertyGroup>