Merge pull request #68 from Etherna/improve/BNET-133-chunk-store-cache

Refactor chunk store classes and update usages
Etherna · Dec 9, 2024 · a554cfb · a554cfb
2 parents a9f34e7 + 50c442c
commit a554cfb
Show file tree

Hide file tree

Showing 22 changed files with 154 additions and 118 deletions.
diff --git a/src/BeeNet.Client/BeeClient.cs b/src/BeeNet.Client/BeeClient.cs
@@ -14,14 +14,15 @@
 
 using Etherna.BeeNet.Clients;
 using Etherna.BeeNet.Exceptions;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Manifest;
 using Etherna.BeeNet.Models;
 using Etherna.BeeNet.Services;
+using Etherna.BeeNet.Stores;
 using Etherna.BeeNet.Tools;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.Formats.Tar;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -38,10 +39,6 @@
 using PostageProof = Etherna.BeeNet.Models.PostageProof;
 using SocProof = Etherna.BeeNet.Clients.SocProof;
 
-#if NET7_0_OR_GREATER
-using System.Formats.Tar;
-#endif
-
 namespace Etherna.BeeNet
 {
     [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings")]
@@ -1404,7 +1401,6 @@ public async Task<SwarmHash> UploadBytesAsync(
                 body: body,
                 cancellationToken).ConfigureAwait(false)).Reference;
 
-#if NET7_0_OR_GREATER
         public async Task<SwarmHash> UploadDirectoryAsync(
             PostageBatchId batchId,
             string directoryPath,
@@ -1445,7 +1441,6 @@ public async Task<SwarmHash> UploadDirectoryAsync(
                 swarm_act_history_address: swarmActHistoryAddress,
                 cancellationToken: cancellationToken).ConfigureAwait(false)).Reference;
         }
-#endif
 
         public async Task<SwarmHash> UploadFileAsync(
             PostageBatchId batchId,

diff --git a/...eNet.Core/Tools/ChunkUploaderWebSocket.cs → ...eNet.Core/Tools/ChunkWebSocketUploader.cs b/...eNet.Core/Tools/ChunkUploaderWebSocket.cs → ...eNet.Core/Tools/ChunkWebSocketUploader.cs
diff --git a/src/BeeNet.Util/Hashing/Pipeline/ChunkStoreWriterPipelineStage.cs b/src/BeeNet.Util/Hashing/Pipeline/ChunkStoreWriterPipelineStage.cs
@@ -13,15 +13,15 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Hashing.Postage;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using System;
 using System.Threading.Tasks;
 
 namespace Etherna.BeeNet.Hashing.Pipeline
 {
     internal sealed class ChunkStoreWriterPipelineStage(
-        IChunkStore chunkStore,
+        ChunkStoreBase chunkStore,
         IPostageStamper postageStamper,
         IHasherPipelineStage? nextStage)
         : IHasherPipelineStage
@@ -47,7 +47,7 @@ public async Task FeedAsync(HasherPipelineFeedArgs args)
             // Store chunk.
             var chunk = SwarmChunk.BuildFromSpanAndData(args.Hash.Value, args.Data.Span);
             chunk.PostageStamp = stamp;
-            await chunkStore.AddAsync(chunk).ConfigureAwait(false);
+            await chunkStore.AddAsync(chunk, true).ConfigureAwait(false);
 
             if (nextStage is not null)
                 await nextStage.FeedAsync(args).ConfigureAwait(false);

diff --git a/src/BeeNet.Util/Hashing/Pipeline/HasherPipelineBuilder.cs b/src/BeeNet.Util/Hashing/Pipeline/HasherPipelineBuilder.cs
@@ -13,8 +13,8 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Hashing.Postage;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using System;
 using System.Diagnostics.CodeAnalysis;
 
@@ -40,7 +40,7 @@ public static IHasherPipeline BuildNewHasherPipeline(
                 chunkConcurrency);
 
         public static IHasherPipeline BuildNewHasherPipeline(
-            IChunkStore chunkStore,
+            ChunkStoreBase chunkStore,
             IPostageStamper postageStamper,
             RedundancyLevel redundancyLevel,
             bool isEncrypted,

diff --git a/src/BeeNet.Util/Hashing/Postage/FakePostageStamper.cs b/src/BeeNet.Util/Hashing/Postage/FakePostageStamper.cs
@@ -13,8 +13,8 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Hashing.Signer;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using System;
 
 namespace Etherna.BeeNet.Hashing.Postage

diff --git a/src/BeeNet.Util/Hashing/Postage/IPostageStamper.cs b/src/BeeNet.Util/Hashing/Postage/IPostageStamper.cs
@@ -13,8 +13,8 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Hashing.Signer;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 
 namespace Etherna.BeeNet.Hashing.Postage
 {

diff --git a/src/BeeNet.Util/Hashing/Postage/PostageStamper.cs b/src/BeeNet.Util/Hashing/Postage/PostageStamper.cs
@@ -14,8 +14,8 @@
 
 using Etherna.BeeNet.Extensions;
 using Etherna.BeeNet.Hashing.Signer;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using Org.BouncyCastle.Crypto.Digests;
 using System;
 

diff --git a/src/BeeNet.Util/Hashing/Store/IReadOnlyChunkStore.cs b/src/BeeNet.Util/Hashing/Store/IReadOnlyChunkStore.cs
diff --git a/src/BeeNet.Util/IBeeClient.cs b/src/BeeNet.Util/IBeeClient.cs
@@ -865,7 +865,6 @@ Task<SwarmHash> UploadBytesAsync(
             RedundancyLevel swarmRedundancyLevel = RedundancyLevel.None,
             CancellationToken cancellationToken = default);
 
-#if NET7_0_OR_GREATER
         /// <summary>Upload a directory</summary>
         /// <param name="batchId">ID of Postage Batch that is used to upload data with</param>
         /// <param name="directoryPath">The directory path</param>
@@ -894,7 +893,6 @@ Task<SwarmHash> UploadDirectoryAsync(
             bool? swarmAct = null,
             string? swarmActHistoryAddress = null,
             CancellationToken cancellationToken = default);
-#endif
 
         /// <summary>Upload a file</summary>
         /// <param name="batchId">ID of Postage Batch that is used to upload data with</param>

diff --git a/src/BeeNet.Util/Manifest/ReferencedMantarayManifest.cs b/src/BeeNet.Util/Manifest/ReferencedMantarayManifest.cs
@@ -12,27 +12,23 @@
 // You should have received a copy of the GNU Lesser General Public License along with Bee.Net.
 // If not, see <https://www.gnu.org/licenses/>.
 
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
 namespace Etherna.BeeNet.Manifest
 {
-    public class ReferencedMantarayManifest : IReadOnlyMantarayManifest
+    public class ReferencedMantarayManifest(
+        ReadOnlyChunkStoreBase chunkStore,
+        SwarmHash rootHash,
+        bool useChunksCache = true)
+        : IReadOnlyMantarayManifest
     {
         // Fields.
-        private readonly ReferencedMantarayNode _rootNode;
+        private readonly ReferencedMantarayNode _rootNode = new(chunkStore, rootHash, null, NodeType.Edge, useChunksCache);
 
-        // Constructors.
-        public ReferencedMantarayManifest(
-            IReadOnlyChunkStore chunkStore,
-            SwarmHash rootHash)
-        {
-            _rootNode = new ReferencedMantarayNode(chunkStore, rootHash, null, NodeType.Edge);
-        }
-
         // Properties.
         public IReadOnlyMantarayNode RootNode => _rootNode;
 

diff --git a/src/BeeNet.Util/Manifest/ReferencedMantarayNode.cs b/src/BeeNet.Util/Manifest/ReferencedMantarayNode.cs
@@ -12,8 +12,8 @@
 // You should have received a copy of the GNU Lesser General Public License along with Bee.Net.
 // If not, see <https://www.gnu.org/licenses/>.
 
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using Newtonsoft.Json;
 using System;
 using System.Buffers.Binary;
@@ -26,20 +26,24 @@ namespace Etherna.BeeNet.Manifest
     public class ReferencedMantarayNode : IReadOnlyMantarayNode
     {
         // Fields.
-        private readonly IReadOnlyChunkStore chunkStore;
+        private readonly ReadOnlyChunkStoreBase chunkStore;
+        private readonly bool useChunksCache;
+
         private SwarmHash? _entryHash;
         private readonly Dictionary<char, ReferencedMantarayNodeFork> _forks = new();
         private readonly Dictionary<string, string> _metadata;
         private XorEncryptKey? _obfuscationKey;
 
         // Constructor.
         public ReferencedMantarayNode(
-            IReadOnlyChunkStore chunkStore,
+            ReadOnlyChunkStoreBase chunkStore,
             SwarmHash chunkHash,
             Dictionary<string, string>? metadata,
-            NodeType nodeTypeFlags)
+            NodeType nodeTypeFlags,
+            bool useChunksCache)
         {
             this.chunkStore = chunkStore ?? throw new ArgumentNullException(nameof(chunkStore));
+            this.useChunksCache = useChunksCache;
             Hash = chunkHash;
             _metadata = metadata ?? new Dictionary<string, string>();
             NodeTypeFlags = nodeTypeFlags;
@@ -74,7 +78,10 @@ public async Task DecodeFromChunkAsync(SwarmHash manifestHash)
             if (IsDecoded)
                 return;
 
-            var chunk = await chunkStore.GetAsync(Hash).ConfigureAwait(false);
+            var chunk = await chunkStore.GetAsync(
+                Hash,
+                !useChunksCache,
+                !useChunksCache).ConfigureAwait(false);
 
             var data = chunk.Data.ToArray();
             var readIndex = 0;
@@ -238,7 +245,12 @@ private void DecodeVersion02(ReadOnlySpan<byte> data)
                 //add fork
                 _forks[key] = new ReferencedMantarayNodeFork(
                     prefix,
-                    new ReferencedMantarayNode(chunkStore, childNodeHash, childNodeMetadata, childNodeTypeFlags));
+                    new ReferencedMantarayNode(
+                        chunkStore,
+                        childNodeHash,
+                        childNodeMetadata,
+                        childNodeTypeFlags,
+                        useChunksCache));
             }
         }
     }

diff --git a/src/BeeNet.Util/Services/ChunkService.cs b/src/BeeNet.Util/Services/ChunkService.cs
@@ -15,9 +15,9 @@
 using Etherna.BeeNet.Hashing.Pipeline;
 using Etherna.BeeNet.Hashing.Postage;
 using Etherna.BeeNet.Hashing.Signer;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Manifest;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -37,7 +37,7 @@ public async Task<UploadEvaluationResult> EvaluateDirectoryUploadAsync(
             RedundancyLevel redundancyLevel = RedundancyLevel.None,
             IPostageStampIssuer? postageStampIssuer = null,
             int? chunkCuncorrency = null, 
-            IChunkStore? chunkStore = null)
+            ChunkStoreBase? chunkStore = null)
         {
             // Checks.
             if (indexFilename?.Contains(SwarmAddress.Separator, StringComparison.InvariantCulture) == true)
@@ -146,7 +146,7 @@ public async Task<UploadEvaluationResult> EvaluateSingleFileUploadAsync(
             RedundancyLevel redundancyLevel = RedundancyLevel.None,
             IPostageStampIssuer? postageStampIssuer = null,
             int? chunkCuncorrency = null, 
-            IChunkStore? chunkStore = null)
+            ChunkStoreBase? chunkStore = null)
         {
             using var stream = new MemoryStream(data);
             return await EvaluateSingleFileUploadAsync(
@@ -170,7 +170,7 @@ public async Task<UploadEvaluationResult> EvaluateSingleFileUploadAsync(
             RedundancyLevel redundancyLevel = RedundancyLevel.None,
             IPostageStampIssuer? postageStampIssuer = null,
             int? chunkCuncorrency = null, 
-            IChunkStore? chunkStore = null)
+            ChunkStoreBase? chunkStore = null)
         {
             chunkStore ??= new FakeChunkStore();
 
@@ -240,7 +240,7 @@ public string[] GetAllChunkFilesInDirectory(string chunkStoreDirectory) =>
 
         public async Task<IReadOnlyDictionary<string, string>> GetFileMetadataFromChunksAsync(
             SwarmAddress address,
-            IReadOnlyChunkStore chunkStore)
+            ReadOnlyChunkStoreBase chunkStore)
         {
             var rootManifest = new ReferencedMantarayManifest(
                 chunkStore,

diff --git a/src/BeeNet.Util/Services/IChunkService.cs b/src/BeeNet.Util/Services/IChunkService.cs
@@ -13,8 +13,8 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Hashing.Postage;
-using Etherna.BeeNet.Hashing.Store;
 using Etherna.BeeNet.Models;
+using Etherna.BeeNet.Stores;
 using System.Collections.Generic;
 using System.IO;
 using System.Threading;
@@ -46,7 +46,7 @@ Task<UploadEvaluationResult> EvaluateDirectoryUploadAsync(
             RedundancyLevel redundancyLevel = RedundancyLevel.None,
             IPostageStampIssuer? postageStampIssuer = null,
             int? chunkCuncorrency = null,
-            IChunkStore? chunkStore = null);
+            ChunkStoreBase? chunkStore = null);
 
         /// <summary>
         /// Evaluate the result uploading a single file
@@ -70,7 +70,7 @@ Task<UploadEvaluationResult> EvaluateSingleFileUploadAsync(
             RedundancyLevel redundancyLevel = RedundancyLevel.None,
             IPostageStampIssuer? postageStampIssuer = null,
             int? chunkCuncorrency = null,
-            IChunkStore? chunkStore = null);
+            ChunkStoreBase? chunkStore = null);
 
         /// <summary>
         /// Evaluate the result uploading a single file
@@ -94,7 +94,7 @@ Task<UploadEvaluationResult> EvaluateSingleFileUploadAsync(
             RedundancyLevel redundancyLevel = RedundancyLevel.None,
             IPostageStampIssuer? postageStampIssuer = null,
             int? chunkCuncorrency = null,
-            IChunkStore? chunkStore = null);
+            ChunkStoreBase? chunkStore = null);
 
         /// <summary>
         /// Get list of all chunk files in directory
@@ -112,7 +112,7 @@ string[] GetAllChunkFilesInDirectory(
         /// <returns>Resource metadata</returns>
         Task<IReadOnlyDictionary<string, string>> GetFileMetadataFromChunksAsync(
             SwarmAddress address,
-            IReadOnlyChunkStore chunkStore);
+            ReadOnlyChunkStoreBase chunkStore);
 
         /// <summary>
         /// Get resource stream from a directory of chunks and the resource address

diff --git a/src/BeeNet.Util/Hashing/Store/IChunkStore.cs → ...BeeNet.Util/Stores/BeeClientChunkStore.cs b/src/BeeNet.Util/Hashing/Store/IChunkStore.cs → ...BeeNet.Util/Stores/BeeClientChunkStore.cs
@@ -13,17 +13,18 @@
 // If not, see <https://www.gnu.org/licenses/>.
 
 using Etherna.BeeNet.Models;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 
-namespace Etherna.BeeNet.Hashing.Store
+namespace Etherna.BeeNet.Stores
 {
-    public interface IChunkStore : IReadOnlyChunkStore
+    public class BeeClientChunkStore(
+        IBeeClient beeClient,
+        IDictionary<SwarmHash, SwarmChunk>? chunksCache = null)
+        : ReadOnlyChunkStoreBase(chunksCache)
     {
-        /// <summary>
-        /// Add a chunk in the store
-        /// </summary>
-        /// <param name="chunk">The chuck to add</param>
-        /// <returns>True if chunk has been added, false if already existing</returns>
-        public Task<bool> AddAsync(SwarmChunk chunk);
+        // Methods.
+        protected override Task<SwarmChunk> LoadChunkAsync(SwarmHash hash) =>
+            beeClient.GetChunkAsync(hash);
     }
 }