Merge pull request #89 from Etherna/improve/BHM-133-cleanup-hangfire

Add task to clean up old failed Hangfire tasks

src/Beehive.Services/Extensions/LoggerExtensions.cs

@@ -21,14 +21,26 @@ namespace Etherna.Beehive.Services.Extensions
 {
     /*
      * Always group similar log delegates by type, always use incremental event ids.
-     * Last event id is: 11
+     * Last event id is: 13
      */
     public static class LoggerExtensions
     {
         // Fields.
         //*** DEBUG LOGS ***
 
         //*** INFORMATION LOGS ***
+        private static readonly Action<ILogger, long, Exception> _cleanupOldFailedTasksTaskCompleted =
+            LoggerMessage.Define<long>(
+                LogLevel.Information,
+                new EventId(12, nameof(CleanupOldFailedTasksTaskCompleted)),
+                "Clean up failed-tasks task completed removing {RemovedFailedTasks} tasks");
+
+        private static readonly Action<ILogger, Exception> _cleanupOldFailedTasksTaskStarted =
+            LoggerMessage.Define(
+                LogLevel.Information,
+                new EventId(13, nameof(CleanupOldFailedTasksTaskStarted)),
+                "Clean up failed-tasks task started");
+
         private static readonly Action<ILogger, string, BzzBalance, IEnumerable<string>, Exception> _nodeCashedOut =
             LoggerMessage.Define<string, BzzBalance, IEnumerable<string>>(
                 LogLevel.Information,
@@ -105,6 +117,13 @@ public static class LoggerExtensions
                 "Withdraw on node {BeeNodeId} chequebook failed with {BzzAmount} BZZ");
 
         // Methods.
+        public static void CleanupOldFailedTasksTaskCompleted(
+            this ILogger logger,
+            long deletedTasks) => _cleanupOldFailedTasksTaskCompleted(logger, deletedTasks, null!);
+
+        public static void CleanupOldFailedTasksTaskStarted(
+            this ILogger logger) => _cleanupOldFailedTasksTaskStarted(logger, null!);
+
         public static void FailedToDepositBzzOnNodeChequeBook(this ILogger logger, string nodeId, BzzBalance bzzDeposit, Exception exception) =>
             _failedToDepositBzzOnNodeChequeBook(logger, nodeId, bzzDeposit, exception);
 

src/Beehive.Services/ServiceCollectionExtensions.cs

@@ -50,6 +50,7 @@ where t.GetInterfaces().Contains(typeof(IEventHandler))
 
             // Tasks.
             services.AddTransient<ICashoutAllNodesChequesTask, CashoutAllNodesChequesTask>();
+            services.AddTransient<ICleanupOldFailedTasksTask, CleanupOldFailedTasksTask>();
             services.AddTransient<INodesAddressMaintainerTask, NodesAddressMaintainerTask>();
             services.AddTransient<INodesChequebookMaintainerTask, NodesChequebookMaintainerTask>();
             services.AddTransient<IPinContentInNodeTask, PinContentInNodeTask>();

src/Beehive.Services/Tasks/CleanupOldFailedTasksTask.cs

@@ -0,0 +1,68 @@
+// Copyright 2021-present Etherna SA
+// This file is part of Beehive.
+// 
+// Beehive is free software: you can redistribute it and/or modify it under the terms of the
+// GNU Affero General Public License as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+// 
+// Beehive is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU Affero General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License along with Beehive.
+// If not, see <https://www.gnu.org/licenses/>.
+
+using Etherna.Beehive.Services.Extensions;
+using Hangfire;
+using Hangfire.Storage.Monitoring;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Threading.Tasks;
+
+namespace Etherna.Beehive.Services.Tasks
+{
+    public class CleanupOldFailedTasksTask(ILogger<CleanupOldFailedTasksTask> logger)
+        : ICleanupOldFailedTasksTask
+    {
+        // Consts.
+        public const string TaskId = "cleanupOldFailedTasksTask";
+
+        // Fields.
+        private readonly TimeSpan retentionTime = new(30, 0, 0, 0);
+
+        // Methods.
+        public Task RunAsync()
+        {
+            var api = JobStorage.Current.GetMonitoringApi();
+            var tasksToSkip = 0;
+            var deletedTasks = 0L;
+            JobList<FailedJobDto> failedJobs;
+
+            // Log begin.
+            logger.CleanupOldFailedTasksTaskStarted();
+
+            // Run clean up.
+            do
+            {
+                failedJobs = api.FailedJobs(tasksToSkip, 1000 /* limit */);
+
+                foreach (var job in failedJobs)
+                {
+                    if (job.Value.FailedAt is not null &&
+                        DateTime.UtcNow - job.Value.FailedAt > retentionTime)
+                    {
+                        BackgroundJob.Delete(job.Key);
+                        deletedTasks++;
+                    }
+                    else
+                        tasksToSkip++;
+                }
+            } while (failedJobs.Count > 0);
+
+            // Log result.
+            logger.CleanupOldFailedTasksTaskCompleted(deletedTasks);
+
+            return Task.CompletedTask;
+        }
+    }
+}

src/Beehive.Services/Tasks/ICleanupOldFailedTasksTask.cs

@@ -0,0 +1,28 @@
+// Copyright 2021-present Etherna SA
+// This file is part of Beehive.
+// 
+// Beehive is free software: you can redistribute it and/or modify it under the terms of the
+// GNU Affero General Public License as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+// 
+// Beehive is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU Affero General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License along with Beehive.
+// If not, see <https://www.gnu.org/licenses/>.
+
+using Hangfire;
+using System.Threading.Tasks;
+
+namespace Etherna.Beehive.Services.Tasks
+{
+    /// <summary>
+    /// Delete failed tasks older than a month from Hangfire
+    /// </summary>
+    public interface ICleanupOldFailedTasksTask
+    {
+        [Queue(Queues.DB_MAINTENANCE)]
+        Task RunAsync();
+    }
+}

src/Beehive.Services/Tasks/Queues.cs

@@ -16,6 +16,7 @@ namespace Etherna.Beehive.Services.Tasks
 {
     public static class Queues
     {
+        public const string DB_MAINTENANCE = "db_maintenance";
         public const string DOMAIN_MAINTENANCE = "domain_maintenance";
         public const string NODE_MAINTENANCE = "node_maintenance";
         public const string PIN_CONTENTS = "pin_contents";

src/Beehive/Program.cs

@@ -194,6 +194,7 @@ private static void ConfigureServices(WebApplicationBuilder builder)
                     options.Queues =
                     [
                         Queues.DOMAIN_MAINTENANCE,
+                        Queues.DB_MAINTENANCE,
                         Queues.PIN_CONTENTS,
                         Queues.NODE_MAINTENANCE,
                         "default"
@@ -331,6 +332,11 @@ private static void ConfigureApplication(WebApplication app)
                 CashoutAllNodesChequesTask.TaskId,
                 task => task.RunAsync(),
                 Cron.Daily(5));
+
+            RecurringJob.AddOrUpdate<ICleanupOldFailedTasksTask>(
+                CleanupOldFailedTasksTask.TaskId,
+                task => task.RunAsync(),
+                Cron.Daily);
 
             RecurringJob.AddOrUpdate<INodesAddressMaintainerTask>(
                 NodesAddressMaintainerTask.TaskId,