diff --git a/GW2PAO.API/Data/ItemsDatabaseBuilder.cs b/GW2PAO.API/Data/ItemsDatabaseBuilder.cs index 8b592d9..2675124 100644 --- a/GW2PAO.API/Data/ItemsDatabaseBuilder.cs +++ b/GW2PAO.API/Data/ItemsDatabaseBuilder.cs @@ -6,13 +6,17 @@ using System.Threading; using System.Threading.Tasks; using GW2NET; +using GW2NET.Common; +using GW2NET.Items; using GW2PAO.API.Constants; -using GW2PAO.API.Data.Enums; +using ItemRarity = GW2PAO.API.Data.Enums.ItemRarity; + using Newtonsoft.Json; using NLog; namespace GW2PAO.API.Data { + public class ItemsDatabaseBuilder { /// @@ -51,60 +55,60 @@ public IDictionary LoadFromFile(CultureInfo culture) public int RebuildItemDatabase(CultureInfo culture, Action incrementProgressAction, Action rebuildCompleteAction, CancellationToken cancelToken) { var itemService = GW2.V2.Items.ForCulture(culture); - var itemIds = itemService.Discover(); - - // We'll split this up into multiple requests - int requestSize = 200; // maybe tweak this - int totalRequests = (itemIds.Count / requestSize) + 1; + int requestSize = 200; + IPageContext ctx = itemService.FindPage(0, requestSize); // Start up a task that will kick off the requests and wait for their completion - Task.Factory.StartNew(() => - { - System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); - sw.Start(); - System.Collections.Concurrent.ConcurrentDictionary itemsDb = new System.Collections.Concurrent.ConcurrentDictionary(); - var parallelOptions = new ParallelOptions() - { - CancellationToken = cancelToken - }; - - try - { - Parallel.For(0, totalRequests, parallelOptions, (i) => + Task.Factory.StartNew( + () => { - var items = itemService.FindPage(i, requestSize); - foreach (var item in items) + var itemsDb = new Dictionary(capacity: ctx.TotalCount); + var tasks = new List>>(ctx.PageCount); + tasks.AddRange(itemService.FindAllPagesAsync(ctx.PageSize, ctx.PageCount, cancelToken)); + var buckets = Interleaved(tasks); + for (int i = 0; i < buckets.Length; i++) { if (cancelToken.IsCancellationRequested) { return; } - var entry = new ItemDBEntry(item.ItemId, item.Name, (ItemRarity)item.Rarity, item.Level); - if (!itemsDb.TryAdd(item.ItemId, entry)) + var task = buckets[i].Result; + if (task.IsCanceled) + { + logger.Info("Rebuilding the item names database was canceled."); + } + else if (task.IsFaulted) + { + logger.ErrorException("One or more errors occurred when retrieving item names.", task.Exception); + } + else { - logger.Warn("Failed to add {0} to items database", item); + foreach (var item in task.Result) + { + itemsDb[item.ItemId] = new ItemDBEntry( + item.ItemId, + item.Name, + (ItemRarity)item.Rarity, + item.Level); + } + + incrementProgressAction.Invoke(); } } - incrementProgressAction.Invoke(); - }); - } - catch (OperationCanceledException) - { - // Operation was cancelled, return - return; - } - var dbString = JsonConvert.SerializeObject(itemsDb); - File.WriteAllText(this.GetFilePath(CultureInfo.CurrentUICulture.TwoLetterISOLanguageName), dbString); - rebuildCompleteAction.Invoke(); - - sw.Stop(); - logger.Info("Item rebuild took {0}", sw.Elapsed.ToString()); + var filePath = this.GetFilePath(CultureInfo.CurrentUICulture.TwoLetterISOLanguageName); + using (var streamWriter = new StreamWriter(filePath)) + { + var serializer = new JsonSerializer(); + serializer.Serialize(streamWriter, itemsDb); + } - }, cancelToken); + rebuildCompleteAction.Invoke(); + }, + cancelToken); - return totalRequests; + return ctx.PageCount; } /// @@ -114,6 +118,32 @@ public string GetFilePath(string twoLetterIsoLangId) { return string.Format("{0}\\{1}\\{2}", Paths.LocalizationFolder, twoLetterIsoLangId, "ItemDatabase.json"); } + + // http://blogs.msdn.com/b/pfxteam/archive/2012/08/02/processing-tasks-as-they-complete.aspx + private static Task>[] Interleaved(IEnumerable> tasks) + { + var inputTasks = tasks.ToList(); + + var buckets = new TaskCompletionSource>[inputTasks.Count]; + var results = new Task>[buckets.Length]; + for (int i = 0; i < buckets.Length; i++) + { + buckets[i] = new TaskCompletionSource>(); + results[i] = buckets[i].Task; + } + + int nextTaskIndex = -1; + Action> continuation = completed => + { + var bucket = buckets[Interlocked.Increment(ref nextTaskIndex)]; + bucket.TrySetResult(completed); + }; + + foreach (var inputTask in inputTasks) + inputTask.ContinueWith(continuation, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + + return results; + } } public class ItemDBEntry diff --git a/GW2PAO/App.config b/GW2PAO/App.config index 1a31257..2920c08 100644 --- a/GW2PAO/App.config +++ b/GW2PAO/App.config @@ -1,249 +1,254 @@  - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +
+ + + + + + + + + + - - - - - - - 250 - - - 225 - - - -1 - - - -1 - - - 125 - - - 250 - - - 200 - - - 200 - - - True - - - -1 - - - -1 - - - 125 - - - 280 - - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - - - True - - - False - - - False - - - False - - - False - - - False - - - True - - - 0 - - - 29 - - - False - - - -1 - - - -1 - - - 200 - - - 225 - - - False - - - False - - - True - - - True - - - -1 - - - -1 - - - False - - - - - - 125 - - - 250 - - - -1 - - - -1 - - - False - - - - - - False - - - -1 - - - -1 - - - 110 - - - 250 - - - False - - - False - - - 125 - - - 250 - - - -1 - - - -1 - - - False - - - False - - - False - - - False - - - -1 - - - -1 - - - False - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 250 + + + 225 + + + -1 + + + -1 + + + 125 + + + 250 + + + 200 + + + 200 + + + True + + + -1 + + + -1 + + + 125 + + + 280 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + True + + + False + + + False + + + False + + + False + + + False + + + True + + + 0 + + + 29 + + + False + + + -1 + + + -1 + + + 200 + + + 225 + + + False + + + False + + + True + + + True + + + -1 + + + -1 + + + False + + + + + + 125 + + + 250 + + + -1 + + + -1 + + + False + + + + + + False + + + -1 + + + -1 + + + 110 + + + 250 + + + False + + + False + + + 125 + + + 250 + + + -1 + + + -1 + + + False + + + False + + + False + + + False + + + -1 + + + -1 + + + False + + + diff --git a/GW2PAO/App.xaml.cs b/GW2PAO/App.xaml.cs index a22523b..7ed4276 100644 --- a/GW2PAO/App.xaml.cs +++ b/GW2PAO/App.xaml.cs @@ -18,6 +18,8 @@ namespace GW2PAO { + using System.Net; + /// /// Interaction logic for App.xaml /// @@ -66,6 +68,9 @@ protected override void OnStartup(StartupEventArgs e) GW2PAO.Properties.Settings.Default.FirstTimeRun = false; GW2PAO.Properties.Settings.Default.Save(); + + // Disable client-side connection throttling + ServicePointManager.DefaultConnectionLimit = int.MaxValue; } private void InitializeSettings()