Skip to content

Commit

Permalink
- fixed a bug that prevented the user from setting a timeout
Browse files Browse the repository at this point in the history
- HttpClient.Timeout is now set to Infinite and the timeout is regulated through Cancellation Tokens
- timeout, concurrent tasks and download attempts are now saved locally in settings.json
- removed some dead code
  • Loading branch information
sentouki committed Jan 28, 2021
1 parent c4b0e32 commit df94337
Show file tree
Hide file tree
Showing 14 changed files with 222 additions and 115 deletions.
7 changes: 4 additions & 3 deletions ArtAPI/ArtStationAPI.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Threading.Tasks;
using System.Linq;
using ArtAPI.misc;
using Newtonsoft.Json.Linq;

namespace ArtAPI
Expand Down Expand Up @@ -42,15 +43,15 @@ protected override async Task GetImagesMetadataAsync(string apiUrl)
var settings = new JsonMergeSettings { MergeArrayHandling = MergeArrayHandling.Concat };
try
{
string rawResponse = await Client.GetStringAsync(apiUrl).ConfigureAwait(false);
string rawResponse = await Client.GetStringAsyncM(apiUrl).ConfigureAwait(false);
var responseJson = JObject.Parse(rawResponse);
int totalCount = Int32.Parse(responseJson["total_count"]?.ToString() ?? throw new Exception("Bad API Response"));
var pages = (totalCount / 50) + 1;

allPages = (JContainer)responseJson["data"]; // add the first page to the container
for (int page = pages; page > 1; page--) // go through the remaining pages and add them to the container
{
rawResponse = await Client.GetStringAsync(apiUrl + $"{page}").ConfigureAwait(false);
rawResponse = await Client.GetStringAsyncM(apiUrl + $"{page}").ConfigureAwait(false);
allPages.Merge((JContainer)JObject.Parse(rawResponse)["data"], settings);
}
}
Expand All @@ -67,7 +68,7 @@ protected override async Task GetImagesMetadataAsync(string apiUrl)
// get all the images from a project
private async Task GetAssets(string hash_id, string name)
{
var response = await Client.GetStringAsync(string.Format(AssetsUrl, hash_id)).ConfigureAwait(false);
var response = await Client.GetStringAsyncM(string.Format(AssetsUrl, hash_id)).ConfigureAwait(false);
var assetsJson = JObject.Parse(response);
foreach (var image in assetsJson["assets"] as JContainer)
{
Expand Down
10 changes: 6 additions & 4 deletions ArtAPI/DeviantArtAPI.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using ArtAPI.misc;
using Newtonsoft.Json.Linq;

namespace ArtAPI
Expand Down Expand Up @@ -53,7 +55,7 @@ protected override async Task GetImagesMetadataAsync(string apiUrl)

while (true)
{
var rawResponse = await Client.GetStringAsync(apiUrl + paginationOffset).ConfigureAwait(false);
var rawResponse = await Client.GetStringAsyncM(apiUrl + paginationOffset).ConfigureAwait(false);
var responseJson = JObject.Parse(rawResponse);
var Gallery = (JContainer)responseJson["results"];
if (!(Gallery.HasValues)) return; // check if the user has any images in his gallery
Expand Down Expand Up @@ -91,12 +93,12 @@ private async Task<string> GetOriginImage(string deviationID)
{
try
{
var rawResponse = await Client.GetStringAsync(string.Format(ORIGINIMAGE_URL, deviationID))
.ConfigureAwait(false);
var rawResponse = await Client.GetStringAsyncM(string.Format(ORIGINIMAGE_URL, deviationID))
.ConfigureAwait(false);
var responseJson = JObject.Parse(rawResponse);
return responseJson["src"].ToString();
}
catch (HttpRequestException)
catch (Exception)
{
return null;
}
Expand Down
11 changes: 6 additions & 5 deletions ArtAPI/PixivAPI.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using ArtAPI.misc;

namespace ArtAPI
{
Expand Down Expand Up @@ -54,15 +55,15 @@ public override async Task<bool> CheckArtistExistsAsync(string artist)
private async Task<string> GetArtistID(string artistName)
{
if (!IsLoggedIn) return null;
var response = await Client.GetStringAsync(string.Format(UserSearchUrl, artistName)).ConfigureAwait(false);
var response = await Client.GetStringAsyncM(string.Format(UserSearchUrl, artistName)).ConfigureAwait(false);
var searchResults = JObject.Parse(response);
if (!searchResults["user_previews"].HasValues) return null;
var artistID = searchResults["user_previews"][0]["user"]["id"].ToString();
return artistID;
}
private async Task<string> GetArtistName(string artistID)
{
var response = await Client.GetStringAsync(string.Format(ArtistDetails, artistID)).ConfigureAwait(false);
var response = await Client.GetStringAsyncM(string.Format(ArtistDetails, artistID)).ConfigureAwait(false);
return JObject.Parse(response)["body"]["user_details"]["user_name"].ToString();
}

Expand Down Expand Up @@ -93,7 +94,7 @@ protected override async Task GetImagesMetadataAsync(string apiUrl)
{
// to store the IDs of each project
var tasks = new List<Task>();
var rawResponse = await Client.GetStringAsync(apiUrl).ConfigureAwait(false);
var rawResponse = await Client.GetStringAsyncM(apiUrl).ConfigureAwait(false);
var responseJson = JObject.Parse(rawResponse);
if (!IsLoggedIn)
{
Expand All @@ -115,7 +116,7 @@ await GetImageURLsWithoutLoginAsync(string.Format(IllustProjectUrl, illust_id))
})
);
if (string.IsNullOrEmpty(responseJson["next_url"].ToString())) break;
rawResponse = await Client.GetStringAsync(responseJson["next_url"].ToString()).ConfigureAwait(false);
rawResponse = await Client.GetStringAsyncM(responseJson["next_url"].ToString()).ConfigureAwait(false);
responseJson = JObject.Parse(rawResponse);
}
}
Expand All @@ -130,7 +131,7 @@ await GetImageURLsWithoutLoginAsync(string.Format(IllustProjectUrl, illust_id))

private async Task GetImageURLsWithoutLoginAsync(string illustProject)
{
var response = await Client.GetStringAsync(illustProject).ConfigureAwait(false);
var response = await Client.GetStringAsyncM(illustProject).ConfigureAwait(false);
var illustDetails = JObject.Parse(response)["body"]["illust_details"];
if (Int32.Parse(illustDetails["page_count"].ToString()) > 1)
{
Expand Down
47 changes: 26 additions & 21 deletions ArtAPI/RequestArt.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using ArtAPI.misc;
using DownloadProgressChangedEventArgs = ArtAPI.misc.DownloadProgressChangedEventArgs;

namespace ArtAPI
{
Expand All @@ -18,25 +20,20 @@ private int
_concurrentTasks = 15;
private string _artistDirSavepath;
private int _progress;
private CancellationTokenSource cts;
private readonly TimeoutHandler _handler;
private CancellationTokenSource _cts;
#endregion
#region protected fields
protected HttpClient Client { get; }
protected HttpClientHandler Handler { get; }
protected virtual string Header { get; } = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36";
protected List<ImageModel> ImagesToDownload { get; } = new List<ImageModel>();
#endregion

#region public properties

public int ClientTimeout
{
get => _clientTimeout;
set
{
_clientTimeout = value > 0 ? value : 1;
Client.Timeout = new TimeSpan(0, 0, _clientTimeout);
}
set => _clientTimeout = value > 0 ? value : 1;
}
public int DownloadAttempts
{
Expand Down Expand Up @@ -81,26 +78,28 @@ protected set
#region ctor & dtor
protected RequestArt()
{
Handler = new HttpClientHandler()
_handler = new TimeoutHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
InnerHandler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
}
};
Client = new HttpClient(Handler);
Client = new HttpClient(_handler);
Client.DefaultRequestHeaders.Add("User-Agent", Header);
Client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
Client.Timeout = new TimeSpan(0, 0, _clientTimeout);
Client.Timeout = Timeout.InfiniteTimeSpan;
}
~RequestArt()
{

Client?.Dispose();
Handler?.Dispose();
_handler?.Dispose();
}
#endregion
#region Event handler
public event EventHandler<DownloadStateChangedEventArgs> DownloadStateChanged;
public event EventHandler<DownloadProgressChangedEventArgs> DownloadProgressChanged;
public event EventHandler<LoginStatusChangedEventArgs> LoginStatusChanged;
public event EventHandler<LoginStatusChangedEventArgs> LoginStatusChanged;
protected void OnDownloadProgressChanged(DownloadProgressChangedEventArgs e)
{
DownloadProgressChanged?.Invoke(this, e);
Expand Down Expand Up @@ -134,7 +133,7 @@ protected async Task DownloadImagesAsync()
}
CheckLocalImages();
OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.DownloadRunning, TotalImageCount: TotalImageCount));
cts = new CancellationTokenSource();
_cts = new CancellationTokenSource();
var ss = new SemaphoreSlim(_concurrentTasks);
var t = Task.WhenAll(ImagesToDownload.Select(image => Task.Run(async () =>
{
Expand All @@ -155,7 +154,7 @@ protected async Task DownloadImagesAsync()
}
finally
{
OnDownloadStateChanged(!cts.IsCancellationRequested
OnDownloadStateChanged(!_cts.IsCancellationRequested
? new DownloadStateChangedEventArgs(State.DownloadCompleted, FailedDownloads: FailedDownloads)
: new DownloadStateChangedEventArgs(State.DownloadCanceled, FailedDownloads: FailedDownloads));
ss.Dispose();
Expand All @@ -176,6 +175,10 @@ protected async Task DownloadAsync(ImageModel image, string savePath)
{
await TryDownloadAsync(image.Url, imageSavePath);
}
catch (TimeoutException)
{
OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.ExceptionRaised, "Timeout"));
}
catch (Exception e)
{ // notify about the exception
OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.ExceptionRaised, e.Message));
Expand All @@ -188,7 +191,9 @@ private async Task TryDownloadAsync(string imgUrl, string imageSavePath)
{ // if download fails, try to download again
try
{
using (var asyncResponse = await Client.GetAsync(imgUrl, cts.Token).ConfigureAwait(false))
using var request = new HttpRequestMessage(HttpMethod.Get, imgUrl);
request.SetTimeout(new TimeSpan(0, 0, ClientTimeout)); // set the timeout for every request separately
using (var asyncResponse = await Client.SendAsync(request, _cts.Token).ConfigureAwait(false))
{
if (asyncResponse.StatusCode == HttpStatusCode.Unauthorized)
{
Expand All @@ -205,7 +210,7 @@ private async Task TryDownloadAsync(string imgUrl, string imageSavePath)
}
catch (Exception)
{
if (i == 1 || cts.IsCancellationRequested) throw;
if (i == 1 || _cts.IsCancellationRequested) throw;
// if there's a some timeout or connection error, wait random amount of time before trying again
await Task.Delay(new Random().Next(500, 3000)).ConfigureAwait(false);
}
Expand All @@ -223,7 +228,7 @@ protected void CreateSaveDir(string artistName)
private void Clear()
{
_progress = 0;
cts?.Dispose();
_cts?.Dispose();
ImagesToDownload.Clear();
}
/// <summary>
Expand All @@ -239,7 +244,7 @@ public void CancelDownload()
{
try
{
cts?.Cancel();
_cts?.Cancel();
OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.DownloadCanceled,
FailedDownloads: FailedDownloads));
}
Expand Down
3 changes: 2 additions & 1 deletion ArtAPI/General.cs → ArtAPI/misc/General.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;

#pragma warning disable CA5351

namespace ArtAPI
namespace ArtAPI.misc
{
public static class General
{
Expand Down
71 changes: 71 additions & 0 deletions ArtAPI/misc/HttpRequestExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;

namespace ArtAPI.misc
{
/// <summary>
/// Better timeout handling with HttpClient with the ability to specify timeout on a per-request basis
/// code pieces taken from https://thomaslevesque.com/2018/02/25/better-timeout-handling-with-httpclient/
/// </summary>

public static class HttpRequestExtensions
{
private const string TimeoutPropertyKey = "RequestTimeout";

public static void SetTimeout(this HttpRequestMessage request, TimeSpan? timeout)
{
if (request == null) throw new ArgumentNullException(nameof(request));
request.Properties[TimeoutPropertyKey] = timeout;
}
public static TimeSpan? GetTimeout(this HttpRequestMessage request)
{
if (request == null) throw new ArgumentNullException(nameof(request));
if (request.Properties.TryGetValue(TimeoutPropertyKey, out var value) && value is TimeSpan timeout)
return timeout;
return null;
}

/// <summary>
/// modified version of the <see cref="HttpClient.GetStringAsync"/> method which allows to specify timeout on a per-request basis
/// </summary>
/// <param name="client"><see cref="HttpClient"/> object</param>
/// <param name="uri">request uri</param>
/// <param name="timeout_s">request timeout in seconds</param>
/// <returns></returns>
public static async Task<string> GetStringAsyncM(this HttpClient client, string uri, int timeout_s = 150)
{
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
request.SetTimeout(new TimeSpan(0, 0, timeout_s));
var response = await client.SendAsync(request).ConfigureAwait(false);
return await response.Content.ReadAsStringAsync();
}
}

public class TimeoutHandler : DelegatingHandler
{
public TimeSpan DefaultTimeout { get; set; } = TimeSpan.FromSeconds(150);

protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
using var cts = GetCancellationTokenSource(request, cancellationToken);
try
{
return await base.SendAsync(request, cts?.Token ?? cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
throw new TimeoutException();
}
}
private CancellationTokenSource GetCancellationTokenSource(HttpRequestMessage request, CancellationToken cancellationToken)
{
var timeout = request.GetTimeout() ?? DefaultTimeout;
if (timeout == Timeout.InfiniteTimeSpan) return null;
var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(timeout);
return cts;
}
}
}
19 changes: 19 additions & 0 deletions Artify/Models/ArtifyModel.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Dynamic;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
Expand Down Expand Up @@ -52,6 +53,21 @@ public string SavePath
get => settings.last_used_savepath ?? Environment.GetFolderPath((Environment.SpecialFolder.MyPictures)); // set the default dir for images
set => Platform.SavePath = settings.last_used_savepath = value;
}
public int ClientTimeout
{
get => settings.timeout > 0 ? settings.timeout : Platform.ClientTimeout;
set => Platform.ClientTimeout = settings.timeout = value;
}
public int DownloadAttempts
{
get => settings.download_attempts > 0 ? settings.download_attempts : Platform.DownloadAttempts;
set => Platform.DownloadAttempts = settings.download_attempts = value;
}
public int ConcurrentTasks
{
get => settings.concurrent_tasks > 0 ? settings.concurrent_tasks : Platform.ConcurrentTasks;
set => Platform.ConcurrentTasks = settings.concurrent_tasks = value;
}

private string _selectedPlatform;
public Settings settings = new Settings();
Expand Down Expand Up @@ -153,6 +169,9 @@ public void SelectPlatform(string platformName)
Platform = ArtPlatform[platformName](); // create object of the selected platform
_selectedPlatform = platformName;
Platform.SavePath = SavePath;
Platform.ClientTimeout = ClientTimeout;
Platform.ConcurrentTasks = ConcurrentTasks;
Platform.DownloadAttempts = DownloadAttempts;
}

public async Task<bool> Auth()
Expand Down
3 changes: 3 additions & 0 deletions Artify/Models/Settings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@ public class Settings
{
public string last_used_savepath { get; set; }
public string pixiv_refresh_token { get; set; }
public int concurrent_tasks { get; set; }
public int download_attempts { get; set; }
public int timeout { get; set; }
}
}
Loading

0 comments on commit df94337

Please sign in to comment.