merge from dev

README.md

@@ -27,6 +27,7 @@ Usage:  EthernaVideoImporter SOURCE_TYPE SOURCE_URI [OPTIONS]
 Source types:
   ytchannel     YouTube channel
   ytvideo       YouTube video
+  local         Local videos
 
 Options:
   -ff   Path FFmpeg (default dir: .\FFmpeg\)
@@ -78,6 +79,34 @@ Options:
 Run 'EthernaVideoImporter.Devcon -h' to print help
 ```
 
+#### Local videos
+
+To import from local videos you will need a metadata descriptor file.  
+Metadata is a Json file with following structure.
+
+```
+[
+    {
+        "Id": "myId1",
+        "Title": "My video 1 title",
+        "Description": "My video description",
+        "VideoFilePath": "my/source/local/video/path.mp4",
+        "ThumbnailFilePath": "my/optional/thumbnail/path.jpg"
+    },
+    {
+        "Id": "myId2",
+        "Title": "My video 2 title",
+        ...
+    },
+    ...
+]
+```
+
+The `Id` field is mandatory, and is needed to trace same video through different executions. Each Id needs to be unique in the file.  
+The `ThumbnailFilePath` field is optional.
+
+The Json file path needs to be passed as source uri with the source type `local`.
+
 # Issue reports
 If you've discovered a bug, or have an idea for a new feature, please report it to our issue manager based on Jira https://etherna.atlassian.net/projects/EVI.
 

src/EthernaVideoImporter.Core/CommonConsts.cs

@@ -15,6 +15,7 @@
 using Etherna.BeeNet.Clients.DebugApi;
 using Etherna.BeeNet.Clients.GatewayApi;
 using System;
+using System.IO;
 using System.Runtime.InteropServices;
 
 namespace Etherna.VideoImporter.Core
@@ -26,6 +27,7 @@ public sealed class CommonConsts
         public const string BeeNodeUrl = "http://localhost/";
         public const GatewayApiVersion BeeNodeGatewayVersion = GatewayApiVersion.v4_0_0;
         public const DebugApiVersion BeeNodeDebugVersion = DebugApiVersion.v4_0_0;
+        public const string DefaultFFmpegFolder = @".\FFmpeg\";
         public const int DownloadMaxRetry = 3;
         public static readonly TimeSpan DownloadTimespanRetry = TimeSpan.FromMilliseconds(3500);
         public const string EthernaCreditUrl = "https://credit.etherna.io/";
@@ -50,7 +52,22 @@ public static string FFMpegBinaryName
                 throw new InvalidOperationException("OS not supported");
             }
         }
+        public static string FFProbeBinaryName
+        {
+            get
+            {
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                    return "ffprobe.exe";
+                else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                    return "ffprobe";
+                else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+                    return "ffprobe";
+
+                throw new InvalidOperationException("OS not supported");
+            }
+        }
         public static readonly TimeSpan GnosisBlockTime = TimeSpan.FromSeconds(5);
         public const string ImporterIdentifier = "EthernaImporter";
+        public static DirectoryInfo TempDirectory { get; } = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), ImporterIdentifier));
     }
 }

src/EthernaVideoImporter.Core/EthernaVideoImporter.Core.csproj

@@ -23,27 +23,30 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Bee.Net" Version="0.2.0-alpha.172" />
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Bee.Net" Version="0.2.0-alpha.189" />
     <PackageReference Include="Blurhash.SkiaSharp" Version="2.0.0" />
-    <PackageReference Include="Etherna.YoutubeDownloader.Converter" Version="6.2.12" />
+    <PackageReference Include="Etherna.YoutubeDownloader.Converter" Version="6.3.3" />
     <PackageReference Include="EthernaServicesClient" Version="0.3.0-alpha.60" />
     <PackageReference Include="GitVersion.MsBuild" Version="5.12.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="IdentityModel.OidcClient" Version="5.2.1" />
     <PackageReference Include="Keccak256" Version="1.0.0" />
     <PackageReference Include="MedallionShell" Version="1.6.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="2.88.3" />
-    <PackageReference Include="IdentityModel.OidcClient" Version="5.2.1" />
-    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.28.0" />
-    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.30.0" />
   </ItemGroup>
 
   <ItemGroup>

src/EthernaVideoImporter.Core/EthernaVideoImporter.cs

@@ -12,6 +12,7 @@
 //   See the License for the specific language governing permissions and
 //   limitations under the License.
 
+using Etherna.ServicesClient;
 using Etherna.ServicesClient.Clients.Index;
 using Etherna.VideoImporter.Core.Models.Domain;
 using Etherna.VideoImporter.Core.Models.Index;
@@ -21,62 +22,54 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
-using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 
 namespace Etherna.VideoImporter.Core
 {
-    public class EthernaVideoImporter
+    public class EthernaVideoImporter : IEthernaVideoImporter
     {
         // Fields.
         private readonly ICleanerVideoService cleanerVideoService;
         private readonly IGatewayService gatewayClient;
-        private readonly IUserIndexClient ethernaIndexClient;
-        private readonly ILinkReporterService linkReporterService;
+        private readonly IEthernaUserClients ethernaUserClients;
         private readonly IMigrationService migrationService;
-        private readonly DirectoryInfo tempDirectoryInfo;
         private readonly IVideoUploaderService videoUploaderService;
         private readonly IVideoProvider videoProvider;
 
         // Constructor.
         public EthernaVideoImporter(
             ICleanerVideoService cleanerVideoService,
+            IEthernaUserClients ethernaUserClients,
             IGatewayService gatewayClient,
-            IUserIndexClient ethernaIndexClient,
-            ILinkReporterService linkReporterService,
+            IMigrationService migrationService,
             IVideoProvider videoProvider,
-            IVideoUploaderService videoUploaderService,
-            IMigrationService migrationService)
+            IVideoUploaderService videoUploaderService)
         {
             if (cleanerVideoService is null)
                 throw new ArgumentNullException(nameof(cleanerVideoService));
-            if (linkReporterService is null)
-                throw new ArgumentNullException(nameof(linkReporterService));
             if (videoProvider is null)
                 throw new ArgumentNullException(nameof(videoProvider));
             if (videoUploaderService is null)
                 throw new ArgumentNullException(nameof(videoUploaderService));
 
             this.cleanerVideoService = cleanerVideoService;
+            this.ethernaUserClients = ethernaUserClients;
             this.gatewayClient = gatewayClient;
-            this.ethernaIndexClient = ethernaIndexClient;
-            this.linkReporterService = linkReporterService;
             this.migrationService = migrationService;
-            tempDirectoryInfo = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), CommonConsts.ImporterIdentifier));
             this.videoProvider = videoProvider;
             this.videoUploaderService = videoUploaderService;
         }
 
         // Public methods.
         public async Task RunAsync(
-            string userEthAddress,
+            bool deleteExogenousVideos,
+            bool deleteVideosRemovedFromSource,
+            bool forceVideoUpload,
             bool offerVideos,
             bool pinVideos,
-            bool deleteVideosRemovedFromSource,
-            bool deleteExogenousVideos,
-            bool unpinRemovedVideos,
-            bool forceUploadVideo)
+            string userEthAddress,
+            bool unpinRemovedVideos)
         {
             var importSummaryModelView = new ImportSummaryModelView();
 
@@ -92,7 +85,7 @@ public async Task RunAsync(
             Console.WriteLine("Get user's videos on etherna index");
 
             var userVideosOnIndex = await GetUserVideosOnEthernaAsync(userEthAddress);
-            var ethernaIndexParameters = await ethernaIndexClient.SystemClient.ParametersAsync();
+            var ethernaIndexParameters = await ethernaUserClients.IndexClient.SystemClient.ParametersAsync();
 
             Console.WriteLine($"Found {userVideosOnIndex.Count()} videos already published on etherna index");
 
@@ -120,7 +113,7 @@ public async Task RunAsync(
 
                     //try to update only manifest, or to skip if possible
                     if (alreadyPresentVideo is not null &&
-                        !forceUploadVideo &&
+                        !forceVideoUpload &&
                         minRequiredMigrationOp is OperationType.Skip or OperationType.UpdateManifest)
                     {
                         Console.WriteLine("Video already uploaded on etherna");
@@ -154,10 +147,10 @@ public async Task RunAsync(
 
                             var videoMetadata = new SwarmVideoMetadata(
                                 sourceMetadata.Id,
+                                sourceMetadata.Title,
                                 sourceMetadata.Description,
                                 TimeSpan.FromSeconds(alreadyPresentVideo.LastValidManifest!.Duration),
-                                alreadyPresentVideo.LastValidManifest!.OriginalQuality,
-                                sourceMetadata.Title);
+                                alreadyPresentVideo.LastValidManifest!.OriginalQuality);
 
                             var videoSwarmFile = alreadyPresentVideo.LastValidManifest.Sources.Select(v => new VideoSwarmFile(v.Size, v.Quality, v.Reference));
 
@@ -168,7 +161,7 @@ public async Task RunAsync(
                             updatedPermalinkHash = await videoUploaderService.UploadVideoManifestAsync(metadataVideo, pinVideos, offerVideos);
 
                             // Update on index.
-                            await ethernaIndexClient.VideosClient.VideosPutAsync(
+                            await ethernaUserClients.IndexClient.VideosClient.VideosPutAsync(
                                 alreadyPresentVideo.IndexId,
                                 updatedPermalinkHash);
                         }
@@ -196,7 +189,7 @@ await ethernaIndexClient.VideosClient.VideosPutAsync(
                         }
 
                         // Get and encode video from source.
-                        var video = await videoProvider.GetVideoAsync(sourceMetadata, tempDirectoryInfo);
+                        var video = await videoProvider.GetVideoAsync(sourceMetadata);
                         video.EthernaIndexId = alreadyPresentVideo?.IndexId;
 
                         if (!video.EncodedFiles.Any())
@@ -248,15 +241,15 @@ await ethernaIndexClient.VideosClient.VideosPutAsync(
                     try
                     {
                         // Clear tmp folder.
-                        foreach (var file in tempDirectoryInfo.GetFiles())
+                        foreach (var file in CommonConsts.TempDirectory.GetFiles())
                             file.Delete();
-                        foreach (var dir in tempDirectoryInfo.GetDirectories())
+                        foreach (var dir in CommonConsts.TempDirectory.GetDirectories())
                             dir.Delete(true);
                     }
                     catch
                     {
                         Console.ForegroundColor = ConsoleColor.DarkRed;
-                        Console.WriteLine($"Warning: unable to delete some files inside of {tempDirectoryInfo.FullName}.");
+                        Console.WriteLine($"Warning: unable to delete some files inside of {CommonConsts.TempDirectory.FullName}.");
                         Console.WriteLine($"Please remove manually after the process.");
                         Console.ResetColor();
                     }
@@ -269,7 +262,7 @@ await ethernaIndexClient.VideosClient.VideosPutAsync(
                      * Report etherna references even if video is already present on index.
                      * This handle cases where references are missing for some previous execution error.
                      */
-                    await linkReporterService.SetEthernaReferencesAsync(
+                    await videoProvider.ReportEthernaReferencesAsync(
                         sourceMetadata.Id,
                         updatedIndexId,
                         updatedPermalinkHash);
@@ -318,15 +311,15 @@ await linkReporterService.SetEthernaReferencesAsync(
         }
 
         // Helpers.
-        public async Task<IEnumerable<IndexedVideo>> GetUserVideosOnEthernaAsync(string userAddress)
+        private async Task<IEnumerable<IndexedVideo>> GetUserVideosOnEthernaAsync(string userAddress)
         {
             var videos = new List<VideoDto>();
             const int MaxForPage = 100;
 
             VideoDtoPaginatedEnumerableDto? page = null;
             do
             {
-                page = await ethernaIndexClient.UsersClient.Videos2Async(userAddress, page is null ? 0 : page.CurrentPage + 1, MaxForPage);
+                page = await ethernaUserClients.IndexClient.UsersClient.Videos2Async(userAddress, page is null ? 0 : page.CurrentPage + 1, MaxForPage);
                 videos.AddRange(page.Elements);
             } while (page.Elements.Any());
 

src/EthernaVideoImporter/Services/LinkReporterService.cs → src/EthernaVideoImporter.Core/IEthernaVideoImporter.cs

@@ -12,25 +12,19 @@
 //   See the License for the specific language governing permissions and
 //   limitations under the License.
 
-using Etherna.VideoImporter.Core.Services;
 using System.Threading.Tasks;
 
-namespace Etherna.VideoImporter.Services
+namespace Etherna.VideoImporter.Core
 {
-    public sealed class LinkReporterService : ILinkReporterService
+    public interface IEthernaVideoImporter
     {
-        // Constructors.
-        public LinkReporterService()
-        {
-        }
-
-        // Methods.
-        public Task SetEthernaReferencesAsync(
-            string sourceVideoId,
-            string ethernaIndexId,
-            string ethernaPermalinkHash)
-        {
-            return Task.CompletedTask;
-        }
+        Task RunAsync(
+            bool deleteExogenousVideos,
+            bool deleteVideosRemovedFromSource,
+            bool forceVideoUpload,
+            bool offerVideos,
+            bool pinVideos,
+            string userEthAddress,
+            bool unpinRemovedVideos);
     }
-}
+}

src/EthernaVideoImporter.Core/Models/Domain/SwarmVideoMetadata.cs

@@ -7,11 +7,11 @@ internal sealed class SwarmVideoMetadata : VideoMetadataBase
         // Constructors.
         internal SwarmVideoMetadata(
             string id,
+            string title,
             string description,
             TimeSpan duration,
-            string originVideoQualityLabel,
-            string title)
-            : base(description, duration, originVideoQualityLabel, title)
+            string originVideoQualityLabel)
+            : base(title, description, duration, originVideoQualityLabel)
         {
             Id = id;
         }