Skip to content

Commit

Permalink
- minor improvements
Browse files Browse the repository at this point in the history
- (hopefully) improved download performance of deviantart
  • Loading branch information
sentouki committed Nov 27, 2020
1 parent 5d5f759 commit 086e327
Show file tree
Hide file tree
Showing 8 changed files with 43 additions and 16 deletions.
2 changes: 1 addition & 1 deletion ArtAPI/ArtStationAPI.cs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ private async Task GetAssets(string hash_id, string name)
Url = image["image_url"].ToString().Replace("/large/", "/4k/")
.Replace("/covers/", "/images/")
.Replace("/video_clips/", "/images/")
.Replace("/videos/","/images/")
.Replace("/videos/", "/images/")
.Replace("/panos/", "/images/"), // workaround to download 4k images, maybe there's a better way, I might change this later
Name = name,
ID = image["id"].ToString(),
Expand Down
28 changes: 20 additions & 8 deletions ArtAPI/DeviantArtAPI.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;

Expand All @@ -20,6 +21,8 @@ private const string

public bool IsLoggedIn { get; private set; }

private readonly Regex _scaledDownImgUrlPattern = new Regex(@"(w_[0-9]{3,4}),(h_[0-9]{3,4}),(q_[0-9]{2})", RegexOptions.Compiled);

public override Task<Uri> CreateUrlFromName(string artistName)
{
return Task.FromResult(new Uri($"https://www.deviantart.com/{artistName}"));
Expand Down Expand Up @@ -49,6 +52,7 @@ public override async Task GetImagesAsync(Uri artistUrl)
protected override async Task GetImagesMetadataAsync(string apiUrl)
{
var paginationOffset = 0;

while (true)
{
var rawResponse = await Client.GetStringAsync(apiUrl + paginationOffset).ConfigureAwait(false);
Expand All @@ -59,14 +63,23 @@ protected override async Task GetImagesMetadataAsync(string apiUrl)
{
if (image["content"] == null) return;
var deviationID = image["deviationid"].ToString();
ImagesToDownload.Add(new ImageModel()
var url = image["content"]["src"].ToString();
if (_scaledDownImgUrlPattern.IsMatch(url)) // Check if it's a URL to a scaled down image in order to avoid unnecessary api calls (too many calls may result in a 'Error 401 API token expired')
{
if (await GetOriginImage(deviationID) is { } _url) // try to get the origin image, use the scaled down image if fails
url = _url;
}
lock (ImagesToDownload)
{
Url = (await GetOriginImage(deviationID) is { } url) ? url : image["content"]["src"].ToString(), // try to get the origin image, use the scaled down image if fails
Name = image["title"].ToString(),
ID = deviationID,
FileType = image["content"]["src"].ToString().Split('?')[0].Split('/').Last()
.Split('.')[1] // maybe not the best way but surely the the easiest one
});
ImagesToDownload.Add(new ImageModel()
{
Url = url,
Name = image["title"].ToString(),
ID = deviationID,
FileType = image["content"]["src"].ToString().Split('?')[0].Split('/').Last()
.Split('.')[1] // maybe not the best way but surely the the easiest one
});
}
});
await Task.WhenAll(tasks);
if (responseJson["has_more"].ToString() == "False") return;
Expand All @@ -93,7 +106,6 @@ private async Task<string> GetOriginImage(string deviationID)

public override async Task<bool> auth(string refreshToken)
{
if (IsLoggedIn) return true;
var data = new Dictionary<string, string>()
{
{"grant_type", "client_credentials" },
Expand Down
13 changes: 10 additions & 3 deletions ArtAPI/IRequestArt.cs
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,16 @@ protected async Task DownloadAsync(ImageModel image, string savePath)
try
{
using (var asyncResponse = await Client.GetAsync(image.Url, cts.Token).ConfigureAwait(false))
await using (var fstream = new FileStream(imageSavePath, FileMode.Create))
{
await (await asyncResponse.Content.ReadAsStreamAsync()).CopyToAsync(fstream).ConfigureAwait(false);
if (asyncResponse.StatusCode == HttpStatusCode.Unauthorized)
{
// TODO: fix this later; handle this error somehow, like try to auth again
i = 1;
throw new Exception("API token expired");
}
await using var fstream = new FileStream(imageSavePath, FileMode.Create);
await (await asyncResponse.Content.ReadAsStreamAsync()).CopyToAsync(fstream)
.ConfigureAwait(false);
}
Progress++;
return;
Expand All @@ -195,7 +202,7 @@ protected async Task DownloadAsync(ImageModel image, string savePath)
{
if (i == 1 || cts.IsCancellationRequested) throw;
// if there's a some timeout or connection error, wait random amount of time before trying again
await Task.Delay(new Random().Next(500, 2000)).ConfigureAwait(false);
await Task.Delay(new Random().Next(500, 3000)).ConfigureAwait(false);
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions ArtAPI/PixivAPI.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace ArtAPI
{
public sealed class PixivAPI : RequestArt
{
private const string
private const string
AUTH_URL = @"https://oauth.secure.pixiv.net/auth/token",
LOGIN_SECRET = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c",
APIUrlWithLogin = @"https://app-api.pixiv.net/v1/user/illusts?user_id={0}",
Expand Down Expand Up @@ -72,7 +72,7 @@ public override async Task GetImagesAsync(Uri artistUrl)
OnDownloadStateChanged(new DownloadStateChangedEventArgs(State.DownloadPreparing));
var artistID = artistUrl?.AbsolutePath.Split('/')[3];
if (artistID == null) return;
if (_artistName is {})
if (_artistName is { })
{
CreateSaveDir(_artistName);
_artistName = null;
Expand Down
3 changes: 3 additions & 0 deletions Artify/ViewModels/ArtifyViewModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ private void Platform_DownloadStateChanged(object sender, DownloadStateChangedEv
Notification += $"\n{e.FailedDownloads} images couldn't be downloaded";
break;
case State.ExceptionRaised:
#if DEBUG
Debug.WriteLine($"ERR_MSG: {e.ExceptionMsg}");
#endif
break;
}
}
Expand Down
2 changes: 1 addition & 1 deletion Artify/Views/SettingsPopUp.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d"
Title="SettingsPopUp" MinHeight="450" MinWidth="400" Height="450" Width="400" MaxHeight="700" MaxWidth="900" WindowStartupLocation="CenterOwner" ShowInTaskbar="False" Background="#141414" Opacity="0.99" Style="{StaticResource ArtifyPopUpWindowStyle}">
Title="SettingsPopUp" MinHeight="450" MinWidth="400" Height="450" Width="400" MaxHeight="700" MaxWidth="900" WindowStartupLocation="CenterOwner" ShowInTaskbar="False" Background="#141414" Opacity="0.99" Style="{StaticResource ArtifyPopUpWindowStyle}" KeyDown="Window_KeyDown">
<WindowChrome.WindowChrome>
<WindowChrome CaptionHeight="30"/>
</WindowChrome.WindowChrome>
Expand Down
5 changes: 5 additions & 0 deletions Artify/Views/SettingsPopUp.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,10 @@ private void InputFieldGotFocus(object sender, RoutedEventArgs e)
{
username.Tag = password.Tag = LoginButton.IsEnabled = true;
}

private void Window_KeyDown(object sender, System.Windows.Input.KeyEventArgs e)
{
if (e.Key == System.Windows.Input.Key.Escape) Close();
}
}
}
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Press ***ESC*** to select another platform

#### Running the app
- [.NET Core 3.1 Runtime](https://dotnet.microsoft.com/download/visual-studio-sdks)
- or download the [self-contained version](https://github.com/sentouki/Artify/releases/download/v1.4.0/Artify.selfcontained.zip)
- or download the [self-contained version](https://github.com/sentouki/Artify/releases/download/v1.4.1/Artify.selfcontained.zip)

#### Building the app
- [.NET Core 3.1 SDK](https://dotnet.microsoft.com/download/visual-studio-sdks)
Expand Down

0 comments on commit 086e327

Please sign in to comment.