Skip to content

Commit

Permalink
merge v0.3.11
Browse files Browse the repository at this point in the history
  • Loading branch information
tmm360 committed Nov 5, 2024
2 parents 1aeecf9 + ee5a464 commit 1d50196
Show file tree
Hide file tree
Showing 41 changed files with 828 additions and 498 deletions.
6 changes: 6 additions & 0 deletions .run/Publish linux arm64.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Publish linux arm64" type="DotNetFolderPublish" factoryName="Publish to folder">
<riderPublish configuration="Release" platform="Any CPU" runtime="linux-arm64" self_contained="true" target_folder="$PROJECT_DIR$/src/EthernaVideoImporter/bin/Release/net8.0/linux-arm64/publish" target_framework="net8.0" uuid_high="7733598048098471660" uuid_low="-6828675579183111726" />
<method v="2" />
</configuration>
</component>
6 changes: 6 additions & 0 deletions .run/Publish linux x64.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Publish linux x64" type="DotNetFolderPublish" factoryName="Publish to folder">
<riderPublish configuration="Release" platform="Any CPU" runtime="linux-x64" self_contained="true" target_folder="$PROJECT_DIR$/src/EthernaVideoImporter/bin/Release/net8.0/linux-x64/publish" target_framework="net8.0" uuid_high="7733598048098471660" uuid_low="-6828675579183111726" />
<method v="2" />
</configuration>
</component>
6 changes: 6 additions & 0 deletions .run/Publish osx x64.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Publish osx x64" type="DotNetFolderPublish" factoryName="Publish to folder">
<riderPublish configuration="Release" platform="Any CPU" runtime="osx-x64" self_contained="true" target_folder="$PROJECT_DIR$/src/EthernaVideoImporter/bin/Release/net8.0/osx-x64/publish" target_framework="net8.0" uuid_high="7733598048098471660" uuid_low="-6828675579183111726" />
<method v="2" />
</configuration>
</component>
6 changes: 6 additions & 0 deletions .run/Publish win x64.run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Publish win x64" type="DotNetFolderPublish" factoryName="Publish to folder">
<riderPublish configuration="Release" platform="Any CPU" runtime="win-x64" self_contained="true" target_folder="$PROJECT_DIR$/src/EthernaVideoImporter/bin/Release/net8.0/win-x64/publish" target_framework="net8.0" uuid_high="7733598048098471660" uuid_low="-6828675579183111726" />
<method v="2" />
</configuration>
</component>
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ General Options:
-a, --auto-purchase Accept automatically purchase of all batches
-w, --write-file Write published videos result to a JSON file
--dry Run in dry mode. Any action on swarm gateway or index is performed read-only
--working-dir Use custom working directory
YouTube Options:
--yt-autogen-sub Import also YouTube autogenerated subtitles
Expand Down Expand Up @@ -103,6 +104,7 @@ General Options:
-a, --auto-purchase Accept automatically purchase of all batches
--dry Run in dry mode. Any action on swarm gateway or index is performed read-only
--yt-cookies List of cookies to use with YouTube requests, divided by ';'
--working-dir Use custom working directory
Video Management Options:
-t, --ttl TTL (days) Postage Stamp (default: 365 days)
Expand Down
12 changes: 4 additions & 8 deletions src/EthernaVideoImporter.Core/CommonConsts.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ namespace Etherna.VideoImporter.Core
{
public static class CommonConsts
{
public const int BeePort = 1633;
public const string BeeNodeUrl = "http://localhost/";
public static readonly string DefaultFFmpegFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "FFmpeg");
public const int DownloadMaxRetry = 3;
public static readonly TimeSpan DownloadTimespanRetry = TimeSpan.FromMilliseconds(3500);
Expand All @@ -32,9 +30,9 @@ public static string FFmpegBinaryName
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
return "ffmpeg.exe";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
return "ffmpeg";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
return "ffmpeg";

throw new InvalidOperationException("OS not supported");
Expand All @@ -46,16 +44,14 @@ public static string FFprobeBinaryName
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
return "ffprobe.exe";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
return "ffprobe";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
return "ffprobe";

throw new InvalidOperationException("OS not supported");
}
}
public const string ImporterIdentifier = "EthernaImporter";
public static DirectoryInfo TempDirectory { get; } = Directory.CreateDirectory(
Path.Combine(Path.GetTempPath(), ImporterIdentifier, Guid.NewGuid().ToString()));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
<ItemGroup>
<PackageReference Include="Blurhash.SkiaSharp" Version="2.0.0" />
<PackageReference Include="Etherna.YoutubeDownloader.Converter" Version="6.4.5" />
<PackageReference Include="EthernaSdk.Users.Gateway" Version="0.4.0-alpha.154" />
<PackageReference Include="EthernaSdk.Users.Index" Version="0.4.0-alpha.154" />
<PackageReference Include="GitVersion.MsBuild" Version="6.0.2">
<PackageReference Include="EthernaSdk.Users.Gateway" Version="0.4.0-alpha.165" />
<PackageReference Include="EthernaSdk.Users.Index" Version="0.4.0-alpha.165" />
<PackageReference Include="GitVersion.MsBuild" Version="6.0.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
Expand Down
49 changes: 34 additions & 15 deletions src/EthernaVideoImporter.Core/EthernaVideoImporter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@
using Etherna.Sdk.Users.Index.Clients;
using Etherna.Sdk.Users.Index.Models;
using Etherna.VideoImporter.Core.Models.Domain;
using Etherna.VideoImporter.Core.Models.Domain.Directories;
using Etherna.VideoImporter.Core.Models.ModelView;
using Etherna.VideoImporter.Core.Options;
using Etherna.VideoImporter.Core.Services;
using Microsoft.Extensions.Options;
using Nethereum.Hex.HexConvertors.Extensions;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Video = Etherna.VideoImporter.Core.Models.Domain.Video;
Expand All @@ -44,6 +48,7 @@ public class EthernaVideoImporter : IEthernaVideoImporter
private readonly IHasher hasher;
private readonly IIoService ioService;
private readonly IMigrationService migrationService;
private readonly EthernaVideoImporterOptions options;
private readonly IResultReporterService resultReporterService;
private readonly IVideoManifestService videoManifestService;
private readonly IVideoUploaderService videoUploaderService;
Expand All @@ -59,12 +64,14 @@ public EthernaVideoImporter(
IHasher hasher,
IIoService ioService,
IMigrationService migrationService,
IOptions<EthernaVideoImporterOptions> options,
IResultReporterService resultReporterService,
IVideoManifestService videoManifestService,
IVideoProvider videoProvider,
IVideoUploaderService videoUploaderService)
{
ArgumentNullException.ThrowIfNull(cleanerVideoService, nameof(cleanerVideoService));
ArgumentNullException.ThrowIfNull(options, nameof(options));
ArgumentNullException.ThrowIfNull(videoProvider, nameof(videoProvider));
ArgumentNullException.ThrowIfNull(videoUploaderService, nameof(videoUploaderService));

Expand All @@ -76,6 +83,7 @@ public EthernaVideoImporter(
this.hasher = hasher;
this.ioService = ioService;
this.migrationService = migrationService;
this.options = options.Value;
this.resultReporterService = resultReporterService;
this.videoManifestService = videoManifestService;
this.videoProvider = videoProvider;
Expand Down Expand Up @@ -106,11 +114,17 @@ public async Task RunAsync(
var userName = await ethernaOpenIdConnectClient.GetUsernameAsync();

ioService.WriteLine($"User {userName} authenticated");

// Create workspace.
var workingDirectory = new WorkingDirectory(
options.CustomWorkingDirectory ??
Path.Combine(Path.GetTempPath(), "etherna-importer"));
workingDirectory.CreateDirectory();

// Get videos info.
ioService.WriteLine($"Get videos metadata from {videoProvider.SourceName}");

var sourceVideosMetadata = await videoProvider.GetVideosMetadataAsync();
var sourceVideosMetadata = await videoProvider.GetVideosMetadataAsync(workingDirectory);
var totalSourceVideo = sourceVideosMetadata.Length;

ioService.WriteLine($"Found {totalSourceVideo} valid distinct videos from source");
Expand All @@ -131,6 +145,7 @@ public async Task RunAsync(
foreach (var (sourceMetadata, i) in sourceVideosMetadata.Select((m, i) => (m, i)))
{
VideoImportResultBase? importResult = null;
var projectDirectory = workingDirectory.NewProjectDirectory();

try
{
Expand Down Expand Up @@ -181,7 +196,8 @@ public async Task RunAsync(
sourceMetadata,
userEthAddress,
fundDownload,
fundPinning);
fundPinning,
projectDirectory);

if (updatedPermalinkHash != null)
importResult = VideoImportResultSucceeded.ManifestUpdated(
Expand All @@ -204,11 +220,18 @@ public async Task RunAsync(
$"Error: Description too long, max length: {ethernaIndexParameters.VideoDescriptionMaxLength}");

// Get and encode video from source.
var video = await videoProvider.BuildVideoFromMetadataAsync(sourceMetadata);
var video = await videoProvider.BuildVideoFromMetadataAsync(
sourceMetadata,
projectDirectory);
video.EthernaIndexId = alreadyIndexedVideo?.Id;

// Upload video and all related data.
await videoUploaderService.UploadVideoAsync(video, fundPinning, fundDownload, userEthAddress);
await videoUploaderService.UploadVideoAsync(
video,
fundPinning,
fundDownload,
userEthAddress,
projectDirectory);

importResult = VideoImportResultSucceeded.FullUploaded(
sourceMetadata,
Expand All @@ -230,17 +253,14 @@ public async Task RunAsync(
{
try
{
// Clear tmp folder for next import.
foreach (var file in CommonConsts.TempDirectory.GetFiles())
file.Delete();
foreach (var dir in CommonConsts.TempDirectory.GetDirectories())
dir.Delete(true);
if (projectDirectory.Exists())
projectDirectory.Delete();
}
catch (Exception e)
{
ioService.WriteErrorLine(
$"Warning: unable to delete some files in \"{CommonConsts.TempDirectory.FullName}\".");
ioService.PrintException(e);
ioService.WriteErrorLine(
$"Warning: unable to delete some files in \"{projectDirectory.DirPath}\".");
}
}

Expand All @@ -260,9 +280,6 @@ public async Task RunAsync(
ioService.PrintException(ex);
}
}

// Delete temporary directory.
CommonConsts.TempDirectory.Delete(true);

// Clean up user channel on etherna index.
if (removeMissingVideosFromSource)
Expand Down Expand Up @@ -357,7 +374,8 @@ private async Task SignInUserAsync()
VideoMetadataBase sourceMetadata,
string userEthAddress,
bool fundDownload,
bool fundPinning)
bool fundPinning,
ProjectDirectory projectDirectory)
{
if (!alreadyIndexedVideo.LastValidManifestHash.HasValue)
return null;
Expand Down Expand Up @@ -406,6 +424,7 @@ await videoUploaderService.UploadVideoAsync(
fundPinning,
fundDownload,
userEthAddress,
projectDirectory,
lastValidManifest.Manifest.BatchId);

return video.EthernaPermalinkHash;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright 2022-present Etherna SA
// This file is part of Etherna Video Importer.
//
// Etherna Video Importer is free software: you can redistribute it and/or modify it under the terms of the
// GNU Affero General Public License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// Etherna Video Importer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License along with Etherna Video Importer.
// If not, see <https://www.gnu.org/licenses/>.

namespace Etherna.VideoImporter.Core.Models.Domain.Directories
{
public class ChunksDirectory(string parentDirPath)
: DirectoryBase(parentDirPath, DirName)
{
public const string DirName = "chunks";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2022-present Etherna SA
// This file is part of Etherna Video Importer.
//
// Etherna Video Importer is free software: you can redistribute it and/or modify it under the terms of the
// GNU Affero General Public License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// Etherna Video Importer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License along with Etherna Video Importer.
// If not, see <https://www.gnu.org/licenses/>.

using System.IO;

namespace Etherna.VideoImporter.Core.Models.Domain.Directories
{
public abstract class DirectoryBase(string dirPath)
{
// Constructors.
protected DirectoryBase(string parentDirPath, string dirName)
: this(Path.Combine(parentDirPath, dirName))
{ }

// Properties.
public string DirPath { get; } = dirPath;

// Methods.
public DirectoryInfo CreateDirectory() =>
Directory.CreateDirectory(DirPath);

public void Delete() => Directory.Delete(DirPath, true);

public bool Exists() => Directory.Exists(DirPath);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright 2022-present Etherna SA
// This file is part of Etherna Video Importer.
//
// Etherna Video Importer is free software: you can redistribute it and/or modify it under the terms of the
// GNU Affero General Public License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// Etherna Video Importer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License along with Etherna Video Importer.
// If not, see <https://www.gnu.org/licenses/>.

using System.IO;

namespace Etherna.VideoImporter.Core.Models.Domain.Directories
{
public class EncodedDirectory : DirectoryBase
{
// Consts.
public const string DirName = "encoded";

// Constructor.
public EncodedDirectory(string parentDirPath)
: base(parentDirPath, DirName)
{
SubtitlesDir = new(DirPath);
ThumbnailDir = new(DirPath);
VideoDir = new(DirPath);
}

// Properties.
public EncodedSubtitlesDirectory SubtitlesDir { get; }
public EncodedThumbnailDirectory ThumbnailDir { get; }
public EncodedVideoDirectory VideoDir { get; }
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright 2022-present Etherna SA
// This file is part of Etherna Video Importer.
//
// Etherna Video Importer is free software: you can redistribute it and/or modify it under the terms of the
// GNU Affero General Public License as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// Etherna Video Importer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License along with Etherna Video Importer.
// If not, see <https://www.gnu.org/licenses/>.

namespace Etherna.VideoImporter.Core.Models.Domain.Directories
{
public class EncodedSubtitlesDirectory(string parentDirPath)
: DirectoryBase(parentDirPath, DirName)
{
public const string DirName = "subs";
}
}
Loading

0 comments on commit 1d50196

Please sign in to comment.