Skip to content

Commit

Permalink
upload data
Browse files Browse the repository at this point in the history
  • Loading branch information
FedeC87p committed Sep 16, 2022
1 parent ef8803a commit 988091e
Show file tree
Hide file tree
Showing 11 changed files with 412 additions and 26 deletions.
9 changes: 9 additions & 0 deletions nuget.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<!--To inherit the global NuGet package sources remove the <clear/> line below -->
<clear />
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
<add key="ethernaMyget" value="https://www.myget.org/F/etherna/api/v3/index.json" />
</packageSources>
</configuration>
114 changes: 114 additions & 0 deletions src/EthernaVideoImporter/.editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true

#### Define style ####

# All files
[*]
indent_style = space

# C# Project, JS and CSS files
[*.{csproj,js,ts,css,scss}]
indent_size = 2

#### Suppress warnings ####

# C# files
[*.cs]

# CA1034: Nested types should not be visible
dotnet_diagnostic.CA1034.severity = none # Asp.Net Core Pages uses nested models

# CA1054: Uri parameters should not be strings
dotnet_diagnostic.CA1054.severity = none # Asp.Net Core Pages uses strings natively

# CA1056: Uri properties should not be strings
dotnet_diagnostic.CA1056.severity = none # Asp.Net Core Pages uses strings natively

# CA1303: Do not pass literals as localized parameters
dotnet_diagnostic.CA1303.severity = none # Don't need translated exceptions

# CA1707: Identifiers should not contain underscores
dotnet_diagnostic.CA1707.severity = none # I like underscores into constants name

# CA1812: Avoid uninstantiated internal classes
dotnet_diagnostic.CA1812.severity = none # Doing extensive use of Dependency Injection

# CA1822: Mark members as static
dotnet_diagnostic.CA1822.severity = none # Don't like static members

# CA2007: Consider calling ConfigureAwait on the awaited task
dotnet_diagnostic.CA2007.severity = none # Not needed with .Net Core. More info https://devblogs.microsoft.com/dotnet/configureawait-faq/

# CS1591: Missing XML comment for publicly visible type or member
dotnet_diagnostic.CS1591.severity = none
csharp_indent_labels = one_less_than_current
csharp_using_directive_placement = outside_namespace:silent
csharp_prefer_simple_using_statement = true:suggestion
csharp_prefer_braces = true:silent
csharp_style_namespace_declarations = block_scoped:silent
csharp_style_prefer_method_group_conversion = true:silent
csharp_style_prefer_top_level_statements = true:silent
csharp_style_expression_bodied_methods = false:silent
csharp_style_expression_bodied_constructors = false:silent
csharp_style_expression_bodied_operators = false:silent
csharp_style_expression_bodied_properties = true:silent
csharp_style_expression_bodied_indexers = true:silent
csharp_style_expression_bodied_accessors = true:silent
csharp_style_expression_bodied_lambdas = true:silent
csharp_style_expression_bodied_local_functions = false:silent
[*.{cs,vb}]
#### Naming styles ####

# Naming rules

dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion
dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface
dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i

dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.types_should_be_pascal_case.symbols = types
dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case

dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members
dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case

# Symbol specifications

dotnet_naming_symbols.interface.applicable_kinds = interface
dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.interface.required_modifiers =

dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum
dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.types.required_modifiers =

dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method
dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.non_field_members.required_modifiers =

# Naming styles

dotnet_naming_style.begins_with_i.required_prefix = I
dotnet_naming_style.begins_with_i.required_suffix =
dotnet_naming_style.begins_with_i.word_separator =
dotnet_naming_style.begins_with_i.capitalization = pascal_case

dotnet_naming_style.pascal_case.required_prefix =
dotnet_naming_style.pascal_case.required_suffix =
dotnet_naming_style.pascal_case.word_separator =
dotnet_naming_style.pascal_case.capitalization = pascal_case

dotnet_naming_style.pascal_case.required_prefix =
dotnet_naming_style.pascal_case.required_suffix =
dotnet_naming_style.pascal_case.word_separator =
dotnet_naming_style.pascal_case.capitalization = pascal_case
dotnet_style_operator_placement_when_wrapping = beginning_of_line
tab_width = 4
indent_size = 4
end_of_line = crlf
dotnet_style_coalesce_expression = true:suggestion
dotnet_style_null_propagation = true:suggestion
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
dotnet_style_prefer_auto_properties = true:silent
44 changes: 44 additions & 0 deletions src/EthernaVideoImporter/Dtos/MetadataVideoDto.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
using System.Collections.Generic;

namespace Etherna.EthernaVideoImporter.Dtos
{
internal class MetadataVideoDto
{
// Constructors.
public MetadataVideoDto(
string? batchId,
string description,
long duration,
long createdAt,
string originalQuality,
string ownerAddress,
IEnumerable<MetadataVideoSource> sources,
SwarmImageRaw? thumbnail,
string title,
long? updatedAt)
{
BatchId = batchId;
Description = description;
Duration = duration;
CreatedAt = createdAt;
OriginalQuality = originalQuality;
OwnerAddress = ownerAddress;
Sources = sources;
Thumbnail = thumbnail;
Title = title;
UpdatedAt = updatedAt;
}

// Properties.
public string? BatchId { get; }
public string Description { get; }
public long Duration { get; }
public long CreatedAt { get; }
public string OriginalQuality { get; }
public string OwnerAddress { get; }
public IEnumerable<MetadataVideoSource> Sources { get; }
public SwarmImageRaw? Thumbnail { get; }
public string Title { get; }
public long? UpdatedAt { get; }
}
}
24 changes: 24 additions & 0 deletions src/EthernaVideoImporter/Dtos/MetadataVideoSource.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
namespace Etherna.EthernaVideoImporter.Dtos
{
public class MetadataVideoSource
{
// Constructors.
public MetadataVideoSource(
int bitrate,
string quality,
string reference,
long size)
{
Bitrate = bitrate;
Quality = quality;
Reference = reference;
Size = size;
}

// Properties.
public int Bitrate { get; }
public string Quality { get; }
public string Reference { get; }
public long Size { get; }
}
}
23 changes: 23 additions & 0 deletions src/EthernaVideoImporter/Dtos/SwarmImageRaw.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
using System.Collections.Generic;

namespace Etherna.EthernaVideoImporter.Dtos
{
public class SwarmImageRaw
{
// Constructors.
public SwarmImageRaw(
float aspectRatio,
string blurhash,
IReadOnlyDictionary<string, string> sources)
{
AspectRatio = aspectRatio;
Blurhash = blurhash;
Sources = sources;
}

// Properties.
public float AspectRatio { get; }
public string Blurhash { get; }
public IReadOnlyDictionary<string, string> Sources { get; }
}
}
2 changes: 2 additions & 0 deletions src/EthernaVideoImporter/Dtos/VideoDataInfoDto.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using CsvHelper.Configuration.Attributes;
using System.Collections.Generic;

namespace EthernaVideoImporter.Dtos
{
Expand All @@ -7,6 +8,7 @@ internal class VideoDataInfoDto
public string? Description { get; set; }
[Optional]
public string? DownloadedFileName { get; set; }
public string? DownloadedFilePath { get; set; }
public int Duration { get; set; }
public int Edition { get; set; }
public string? Expertise { get; set; }
Expand Down
21 changes: 21 additions & 0 deletions src/EthernaVideoImporter/Dtos/VideoDownloadInfo.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
namespace Etherna.EthernaVideoImporter.Dtos
{
public class VideoDownloadInfo
{
// Constructors.
public VideoDownloadInfo(
int bitrate,
string quality,
long size)
{
Bitrate = bitrate;
Quality = quality;
Size = size;
}

// Properties.
public int Bitrate { get; }
public string Quality { get; }
public long Size { get; }
}
}
20 changes: 17 additions & 3 deletions src/EthernaVideoImporter/EthernaVideoImporter.csproj
Original file line number Diff line number Diff line change
@@ -1,14 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<RootNamespace>Etherna.EthernaVideoImporter</RootNamespace>

<Authors>Etherna Sagl</Authors>
<Description>A .Net console for upload video on Etherna</Description>

<LangVersion>10.0</LangVersion>
<Nullable>enable</Nullable>
<EnableNETAnalyzers>true</EnableNETAnalyzers>
<AnalysisMode>AllEnabledByDefault</AnalysisMode>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Bee.Net" Version="0.2.0-alpha.127" />
<PackageReference Include="CsvHelper" Version="28.0.1" />
<PackageReference Include="IdentityModel.OidcClient" Version="5.0.2" />
<PackageReference Include="MimeTypes" Version="2.4.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
Expand Down
64 changes: 54 additions & 10 deletions src/EthernaVideoImporter/Program.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,18 @@
using CsvHelper;
using Etherna.BeeNet;
using Etherna.BeeNet.Clients.DebugApi;
using Etherna.BeeNet.Clients.GatewayApi;
using Etherna.EthernaVideoImporter.Dtos;
using Etherna.EthernaVideoImporter.Services;
using EthernaVideoImporter.Dtos;
using EthernaVideoImporter.Services;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using YoutubeDownloader.Clients;

internal class Program
Expand All @@ -11,7 +22,7 @@ static async Task Main(string[] args)
string tmpFolder = "tmpData";

if (args is null ||
args.Length < 0)
args.Length < 1)
{
Console.WriteLine("Missing read path");
return;
Expand All @@ -21,6 +32,28 @@ static async Task Main(string[] args)
Console.WriteLine($"File not found {args[0]}");
return;
}
if (args.Length < 2)
{
Console.WriteLine("Missing beenode url");
return;
}
if (args.Length < 3)
{
Console.WriteLine("Missing beenode port");
return;
}
if (args.Length < 4)
{
Console.WriteLine("Missing beenode version");
return;
}
var beeNodeUrl = args[1];
#pragma warning disable CA1305 // Specify IFormatProvider
var beeNodePort = Convert.ToInt32(args[2]);
#pragma warning restore CA1305 // Specify IFormatProvider
var beeNodeDebugPort = beeNodePort + 1;
var beeNodeVersion = GatewayApiVersion.v3_0_2;
var beeNodeDebugVersion = DebugApiVersion.v3_0_2;

if (!Directory.Exists(tmpFolder))
Directory.CreateDirectory(tmpFolder);
Expand All @@ -36,29 +69,40 @@ static async Task Main(string[] args)
Console.WriteLine($"Csv with {totalVideo} items to upload");

// Call import service for each video.
var youtubeDownloadClient = new YoutubeDownloadClient();
var videoImporterService = new VideoImporterService(youtubeDownloadClient, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, tmpFolder));
var tmpFolderFullPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, tmpFolder);
var videoImporterService = new VideoImporterService(new YoutubeDownloadClient(), tmpFolderFullPath);
var videoUploaderService = new VideoUploaderService(new BeeNodeClient(beeNodeUrl, beeNodePort, beeNodeDebugPort, beeNodeVersion, beeNodeDebugVersion), tmpFolderFullPath);

var videoCount = 0;
foreach (var videoInfo in videoDataInfoDtos)
{
try
{
Console.WriteLine($"Start processing video {++videoCount} of {totalVideo}");
await videoImporterService.Start(videoInfo);

// Download from youtube.
var downloadInfo = await videoImporterService.Start(videoInfo);

if (downloadInfo != null)
// Upload on bee.
await videoUploaderService.Start(videoInfo, downloadInfo);

Console.WriteLine($"Video #{videoCount} processed");
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception ex)
#pragma warning restore CA1031 // Do not catch general exception types
{
Console.WriteLine($"{ex.Message} \n Unable to upload: {videoDataInfoDtos}");
videoInfo.VideoStatusNote = ex.Message;
}
finally
{
// Save csv with results at every cycle.
using (var writer = new StreamWriter(args[0]))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
await csv.WriteRecordsAsync(videoDataInfoDtos).ConfigureAwait(false);
}
}

// Save csv with results.
using (var writer = new StreamWriter(args[0]))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
await csv.WriteRecordsAsync(videoDataInfoDtos);

}
}
Loading

0 comments on commit 988091e

Please sign in to comment.