YoutubeDownloader

EthernaVideoImporter.sln

@@ -0,0 +1,31 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32819.101
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EthernaVideoImporter", "src\EthernaVideoImporter\EthernaVideoImporter.csproj", "{6B534273-4C90-4EEC-A13B-ABBD9867E1D2}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YoutubeDownloader", "src\YoutubeDownloader\YoutubeDownloader.csproj", "{4DA6944A-8048-4A33-BC18-D29C0C1A59A0}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{6B534273-4C90-4EEC-A13B-ABBD9867E1D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6B534273-4C90-4EEC-A13B-ABBD9867E1D2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6B534273-4C90-4EEC-A13B-ABBD9867E1D2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6B534273-4C90-4EEC-A13B-ABBD9867E1D2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4DA6944A-8048-4A33-BC18-D29C0C1A59A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4DA6944A-8048-4A33-BC18-D29C0C1A59A0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4DA6944A-8048-4A33-BC18-D29C0C1A59A0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4DA6944A-8048-4A33-BC18-D29C0C1A59A0}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {09CED59B-3404-49D6-ADE2-70D93A110B72}
+	EndGlobalSection
+EndGlobal

src/EthernaVideoImporter/Dtos/VideoDataInfoDto.cs

@@ -0,0 +1,25 @@
+using CsvHelper.Configuration.Attributes;
+
+namespace EthernaVideoImporter.Dtos
+{
+    internal class VideoDataInfoDto
+    {
+        public string? Description { get; set; }
+        [Optional]
+        public string? DownloadedFileName { get; set; }
+        public int Duration { get; set; }
+        public int Edition { get; set; }
+        public string? Expertise { get; set; }
+        public string? IpfsHash { get; set; }
+        public IEnumerable<string>? Keywords { get; set; }
+        [Optional]
+        public VideoStatus? VideoStatus { get; set; }
+        [Optional]
+        public string? VideoStatusNote { get; set; }
+        public IEnumerable<string>? Tags { get; set; }
+        public string? Title { get; set; }
+        public string? Type { get; set; }
+        public string? Track { get; set; }
+        public string? YoutubeUrl { get; set; }
+    }
+}

src/EthernaVideoImporter/Dtos/VideoStatus.cs

@@ -0,0 +1,11 @@
+namespace EthernaVideoImporter.Dtos
+{
+    internal enum VideoStatus
+    {
+        NotProcess,
+        Downloading,
+        Downloaded,
+        Uploading,
+        Processed
+    }
+}

src/EthernaVideoImporter/EthernaVideoImporter.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CsvHelper" Version="28.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\YoutubeDownloader\YoutubeDownloader.csproj" />
+  </ItemGroup>
+
+</Project>

src/EthernaVideoImporter/Program.cs

@@ -0,0 +1,64 @@
+using CsvHelper;
+using EthernaVideoImporter.Dtos;
+using EthernaVideoImporter.Services;
+using System.Globalization;
+using YoutubeDownloader.Clients;
+
+internal class Program
+{
+    static async Task Main(string[] args)
+    {
+        string tmpFolder = "tmpData";
+
+        if (args is null ||
+            args.Length < 0)
+        {
+            Console.WriteLine("Missing read path");
+            return;
+        }
+        if (!File.Exists(args[0]))
+        {
+            Console.WriteLine($"File not found {args[0]}");
+            return;
+        }
+
+        if (!Directory.Exists(tmpFolder))
+            Directory.CreateDirectory(tmpFolder);
+
+        // Read csv.
+        IEnumerable<VideoDataInfoDto> videoDataInfoDtos;
+        using (var reader = new StreamReader(args[0]))
+        using (var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture))
+        {
+            videoDataInfoDtos = csvReader.GetRecords<VideoDataInfoDto>().ToList();
+        }
+        var totalVideo = videoDataInfoDtos.Count();
+        Console.WriteLine($"Csv with {totalVideo} items to upload");
+
+        // Call import service for each video.
+        var youtubeDownloadClient = new YoutubeDownloadClient();
+        var videoImporterService = new VideoImporterService(youtubeDownloadClient, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, tmpFolder));
+
+        var videoCount = 0;
+        foreach (var videoInfo in videoDataInfoDtos)
+        {
+            try
+            {
+                Console.WriteLine($"Start processing video {++videoCount} of {totalVideo}");
+                await videoImporterService.Start(videoInfo);
+                Console.WriteLine($"Video #{videoCount} processed");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"{ex.Message} \n Unable to upload: {videoDataInfoDtos}");
+                videoInfo.VideoStatusNote = ex.Message;
+            }
+        }
+
+        // Save csv with results.
+        using (var writer = new StreamWriter(args[0]))
+        using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
+            await csv.WriteRecordsAsync(videoDataInfoDtos);
+
+    }
+}

src/EthernaVideoImporter/Properties/AssemblyInfo.cs

@@ -0,0 +1,17 @@
+//   Copyright 2021-present Etherna Sagl
+//
+//   Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//   Unless required by applicable law or agreed to in writing, software
+//   distributed under the License is distributed on an "AS IS" BASIS,
+//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//   See the License for the specific language governing permissions and
+//   limitations under the License.
+
+using System;
+
+[assembly: CLSCompliant(false)]

src/EthernaVideoImporter/Properties/launchSettings.json

@@ -0,0 +1,8 @@
+{
+  "profiles": {
+    "DevconArchiveVideoParser": {
+      "commandName": "Project",
+      "commandLineArgs": "D:\\Etherna\\VideoDevCon.csv"
+    }
+  }
+}

src/EthernaVideoImporter/Services/VideoImporterService.cs

@@ -0,0 +1,62 @@
+using EthernaVideoImporter.Dtos;
+using YoutubeDownloader.Clients;
+
+namespace EthernaVideoImporter.Services
+{
+    internal class VideoImporterService
+    {
+        private readonly YoutubeDownloadClient youtubeDownloadClient;
+        private readonly string tmpFolder;
+
+        // Constractor.
+        public VideoImporterService(
+            YoutubeDownloadClient youtubeDownloadClient,
+            string tmpFolder)
+        {
+            this.youtubeDownloadClient = youtubeDownloadClient;
+            this.tmpFolder = tmpFolder;
+        }
+
+        // Public methods.
+        public async Task Start(VideoDataInfoDto videoDataInfoDto)
+        {
+            if (videoDataInfoDto.VideoStatus == VideoStatus.Processed)
+                return;
+            if (string.IsNullOrWhiteSpace(videoDataInfoDto.YoutubeUrl))
+                throw new InvalidOperationException("Invalid YoutubeUrl");
+
+            if (string.IsNullOrWhiteSpace(videoDataInfoDto.DownloadedFileName))
+            {
+                // Take best video resolution.
+                var videos = await youtubeDownloadClient.GetAllVideosAsync(videoDataInfoDto.YoutubeUrl);
+                var videoWithAudio = videos
+                    .Where(i => i.AudioBitrate != -1); 
+                var videoDownload = videoWithAudio
+                    .First(i => i.AudioBitrate == videoWithAudio.Max(j => j.AudioBitrate)); // Take best resolution
+                Console.WriteLine($"Resolution: {videoDownload.Resolution}\tAudio Bitrate: {videoDownload.AudioBitrate}");
+
+                //Start download
+                videoDataInfoDto.VideoStatus = VideoStatus.Downloading;
+                await youtubeDownloadClient
+                    .CreateDownloadAsync(
+                    new Uri(videoDownload.Uri),
+                    Path.Combine(tmpFolder, videoDownload.FullName),
+                    new Progress<Tuple<long, long>>((Tuple<long, long> v) =>
+                    {
+                        var percent = (int)((v.Item1 * 100) / v.Item2);
+                        Console.Write(string.Format("Downloading.. ( % {0} ) {1} / {2} MB\r", percent, (v.Item1 / (double)(1024 * 1024)).ToString("N"), (v.Item2 / (double)(1024 * 1024)).ToString("N")));
+                    }));
+                Console.WriteLine("");
+                videoDataInfoDto.DownloadedFileName = videoDownload.FullName;
+                videoDataInfoDto.VideoStatus = VideoStatus.Downloaded;
+                videoDataInfoDto.VideoStatusNote = "";
+            }
+
+            //
+            //TODO UPLOAD
+
+            videoDataInfoDto.VideoStatus = VideoStatus.Processed;
+            return;
+        }
+    }
+}

src/YoutubeDownloader/Clients/YoutubeDownloadClient.cs

@@ -0,0 +1,75 @@
+using System.Net.Http.Headers;
+using VideoLibrary;
+using YoutubeDownloader.Handlers;
+
+namespace YoutubeDownloader.Clients
+{
+
+    public class YoutubeDownloadClient : YouTube
+    {
+        private readonly HttpClient _client = new();
+        private readonly long _chunkSize = 10_485_760;
+        private long _fileSize = 0L;
+
+        // Public Methods.
+        public async Task CreateDownloadAsync(Uri uri, string filePath, IProgress<Tuple<long, long>> progress)
+        {
+            var totalBytesCopied = 0L;
+            _fileSize = await GetContentLengthAsync(uri.AbsoluteUri) ?? 0;
+            if (_fileSize == 0)
+            {
+                throw new Exception("File has no any content !");
+            }
+            using var output = File.OpenWrite(filePath);
+            var segmentCount = (int)Math.Ceiling(1.0 * _fileSize / _chunkSize);
+            for (var i = 0; i < segmentCount; i++)
+            {
+                var from = i * _chunkSize;
+                var to = (i + 1) * _chunkSize - 1;
+                var request = new HttpRequestMessage(HttpMethod.Get, uri);
+                request.Headers.Range = new RangeHeaderValue(from, to);
+                using (request)
+                {
+                    // Download Stream
+                    var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+                    if (response.IsSuccessStatusCode)
+                        response.EnsureSuccessStatusCode();
+                    var stream = await response.Content.ReadAsStreamAsync();
+                    //File Steam
+                    var buffer = new byte[81920];
+                    int bytesCopied;
+                    do
+                    {
+                        bytesCopied = await stream.ReadAsync(buffer);
+                        output.Write(buffer, 0, bytesCopied);
+                        totalBytesCopied += bytesCopied;
+                        progress.Report(new Tuple<long, long>(totalBytesCopied, _fileSize));
+                    } while (bytesCopied > 0);
+                }
+            }
+        }
+
+        // Protected Methods.
+        protected override HttpClient MakeClient(HttpMessageHandler handler)
+        {
+            return base.MakeClient(handler);
+        }
+
+        protected override HttpMessageHandler MakeHandler()
+        {
+            return YoutubeHandler.GetHandler();
+        }
+
+        // Private Methods.
+        private async Task<long?> GetContentLengthAsync(string requestUri, bool ensureSuccess = true)
+        {
+            using var request = new HttpRequestMessage(HttpMethod.Head, requestUri);
+            var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+            if (ensureSuccess)
+                response.EnsureSuccessStatusCode();
+            return response.Content.Headers.ContentLength;
+        }
+    }
+
+
+}

src/YoutubeDownloader/Handlers/YoutubeHandler.cs

@@ -0,0 +1,19 @@
+using System.Net;
+
+namespace YoutubeDownloader.Handlers
+{
+    class YoutubeHandler
+    {
+        public static HttpMessageHandler GetHandler()
+        {
+            CookieContainer cookieContainer = new();
+            cookieContainer.Add(new Cookie("CONSENT", "YES+cb", "/", "youtube.com"));
+            return new HttpClientHandler
+            {
+                UseCookies = true,
+                CookieContainer = cookieContainer
+            };
+
+        }
+    }
+}

src/YoutubeDownloader/Properties/AssemblyInfo.cs

@@ -0,0 +1,17 @@
+//   Copyright 2021-present Etherna Sagl
+//
+//   Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//   Unless required by applicable law or agreed to in writing, software
+//   distributed under the License is distributed on an "AS IS" BASIS,
+//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//   See the License for the specific language governing permissions and
+//   limitations under the License.
+
+using System;
+
+[assembly: CLSCompliant(false)]

src/YoutubeDownloader/Properties/launchSettings.json

@@ -0,0 +1,8 @@
+{
+  "profiles": {
+    "DevconArchiveVideoParser": {
+      "commandName": "Project",
+      "commandLineArgs": "D:\\tmp\\devcon-website-archive\\src\\content\\archive\\videos testfile.csv"
+    }
+  }
+}

src/YoutubeDownloader/YoutubeDownloader.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="VideoLibrary" Version="3.2.1" />
+  </ItemGroup>
+
+</Project>