From 90a715ad04993006f29c096d599da0b1780bdbe8 Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Tue, 5 Apr 2022 23:23:21 +0200 Subject: [PATCH 1/7] Replace file system watcher with polling --- CHANGELOG.md | 6 + DEPLOY.md | 2 +- appveyor.yml | 14 ++- src/ConfigCat.Client.Tests/OverrideTests.cs | 6 +- .../Override/LocalFileDataSource.cs | 107 +++++++++++------- 5 files changed, 82 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e72062..58a3642d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +### 6.5.0 +- Replace `FileSystemWatcher` with file polling in local file override data source. + +### 6.4.12 +- Fix various local file override data source issues. + ### 6.4.9 - Move the PollingMode option to public scope. diff --git a/DEPLOY.md b/DEPLOY.md index 15fd6d27..97f83679 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,6 +1,6 @@ # Steps to Deploy 1. Run tests -2. Set version in `appveyor.yml` (e.g: from `version: 2.5.{build}` to `version: 2.6.{build}`) +2. Set version in `appveyor.yml` (e.g: from `build_version: 6.5.0` to `build_version: 6.5.1`) 3. Update release notes in ConfigCatClient.csproj (PackageReleaseNotes) 4. Push to `master` 5. Deploy to NuGet.org diff --git a/appveyor.yml b/appveyor.yml index 77e7936c..77eda149 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,6 @@ -version: 6.4.{build} +environment: + build_version: 6.5.0 +version: $(build_version)-{build} image: Visual Studio 2022 configuration: Release skip_commits: @@ -8,11 +10,11 @@ skip_commits: dotnet_csproj: patch: true file: '**\*.csproj' - version: '{version}' - package_version: '{version}' - assembly_version: '{version}' - file_version: '{version}' - informational_version: '{version}' + version: '{build_version}' + package_version: '{build_version}' + assembly_version: '{build_version}' + file_version: '{build_version}' + informational_version: '{build_version}' install: - cmd: dotnet tool install -g InheritDocTool build_script: diff --git a/src/ConfigCat.Client.Tests/OverrideTests.cs b/src/ConfigCat.Client.Tests/OverrideTests.cs index d9c26082..52185256 100644 --- a/src/ConfigCat.Client.Tests/OverrideTests.cs +++ b/src/ConfigCat.Client.Tests/OverrideTests.cs @@ -121,7 +121,7 @@ public void LocalFile_Default_WhenErrorOccures() } [TestMethod] - public void LocalFile_Reload() + public void LocalFile_Read() { using var client = new ConfigCatClient(options => { @@ -137,7 +137,7 @@ public void LocalFile_Reload() } [TestMethod] - public async Task LocalFileAsync_Reload() + public async Task LocalFileAsync_Read() { using var client = new ConfigCatClient(options => { @@ -362,7 +362,7 @@ public async Task LocalFile_Watcher_Reload() Assert.AreEqual("initial", await client.GetValueAsync("fakeKey", string.Empty)); await WriteContent(SampleFileToCreate, "modified"); - await Task.Delay(400); + await Task.Delay(1100); Assert.AreEqual("modified", await client.GetValueAsync("fakeKey", string.Empty)); diff --git a/src/ConfigCatClient/Override/LocalFileDataSource.cs b/src/ConfigCatClient/Override/LocalFileDataSource.cs index ece7955f..d2a6ad37 100644 --- a/src/ConfigCatClient/Override/LocalFileDataSource.cs +++ b/src/ConfigCatClient/Override/LocalFileDataSource.cs @@ -11,50 +11,31 @@ namespace ConfigCat.Client.Override internal sealed class LocalFileDataSource : IOverrideDataSource { const int WAIT_TIME_FOR_UNLOCK = 200; // ms - const int MAX_WAIT_ITERATIONS = 50; + const int MAX_WAIT_ITERATIONS = 50; // ms + const int FILE_POLL_INTERVAL = 1000; // ms - private int isReading; + private DateTime fileLastWriteTime; private readonly string fullPath; private readonly ILogger logger; - private readonly FileSystemWatcher fileSystemWatcher; private readonly TaskCompletionSource asyncInit = new(); private readonly ManualResetEvent syncInit = new(false); + private readonly CancellationTokenSource pollerCancellationTokenSource = new(); private volatile IDictionary overrideValues; public LocalFileDataSource(string filePath, bool autoReload, ILogger logger) { - this.fullPath = Path.GetFullPath(filePath); - if (autoReload) + if (!File.Exists(filePath)) { - var directory = Path.GetDirectoryName(this.fullPath); - if (string.IsNullOrWhiteSpace(directory)) - { - logger.Error($"Directory of {this.fullPath} not found to watch."); - } - else - { - this.fileSystemWatcher = new FileSystemWatcher(directory); - this.fileSystemWatcher.Changed += OnChanged; - this.fileSystemWatcher.Created += OnChanged; - this.fileSystemWatcher.Renamed += OnChanged; - this.fileSystemWatcher.EnableRaisingEvents = true; - logger.Information($"Watching {this.fullPath} for changes."); - } + logger.Error($"File {filePath} does not exist."); + this.SetInitialized(); + return; } + this.fullPath = Path.GetFullPath(filePath); this.logger = logger; - _ = this.ReadFileAsync(this.fullPath); - } - - private async void OnChanged(object sender, FileSystemEventArgs e) - { - // filter out events on temporary files - if (e.FullPath != this.fullPath) - return; - this.logger.Information($"Reload file {e.FullPath}."); - await this.ReadFileAsync(e.FullPath); + this.StartFileReading(autoReload); } public IDictionary GetOverrides() @@ -71,18 +52,64 @@ public async Task> GetOverridesAsync() return this.overrideValues ?? new Dictionary(); } - private async Task ReadFileAsync(string filePath) + private void StartFileReading(bool autoReload) { - if (Interlocked.CompareExchange(ref this.isReading, 1, 0) != 0) - return; + Task.Run(async () => + { + try + { + await ReadFileAsync(); + + if (autoReload) + { + await this.StartWatchAsync(); + } + } + finally + { + this.SetInitialized(); + } + }); + } + private async Task StartWatchAsync() + { + this.logger.Information($"Watching {this.fullPath} for changes."); + while (!this.pollerCancellationTokenSource.IsCancellationRequested) + { + try + { + try + { + var lastWriteTime = File.GetLastWriteTimeUtc(this.fullPath); + if (lastWriteTime > this.fileLastWriteTime) + { + this.logger.Information($"Reload file {this.fullPath}."); + await ReadFileAsync(); + } + } + + finally + { + await Task.Delay(FILE_POLL_INTERVAL, this.pollerCancellationTokenSource.Token); + } + } + catch (OperationCanceledException) + { + // ignore exceptions from cancellation. + } + } + } + + private async Task ReadFileAsync() + { try { for (int i = 1; i <= MAX_WAIT_ITERATIONS; i++) { try { - using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var stream = new FileStream(this.fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); using var reader = new StreamReader(stream); var content = await reader.ReadToEndAsync(); var simplified = content.DeserializeOrDefault(); @@ -105,15 +132,15 @@ private async Task ReadFileAsync(string filePath) await Task.Delay(WAIT_TIME_FOR_UNLOCK); } - } + } } catch (Exception ex) { - this.logger.Error($"Failed to read file {filePath}. {ex}"); + this.logger.Error($"Failed to read file {this.fullPath}. {ex}"); } finally { - Interlocked.Exchange(ref this.isReading, 0); + this.fileLastWriteTime = File.GetLastWriteTimeUtc(this.fullPath); this.SetInitialized(); } } @@ -126,13 +153,7 @@ private void SetInitialized() public void Dispose() { - if (this.fileSystemWatcher != null) - { - this.fileSystemWatcher.Changed -= OnChanged; - this.fileSystemWatcher.Created -= OnChanged; - this.fileSystemWatcher.Renamed -= OnChanged; - this.fileSystemWatcher.Dispose(); - } + this.pollerCancellationTokenSource.Cancel(); this.syncInit.Dispose(); } From 6a9a4ad884a1ce5fcb52a6ca043429914c7b48fa Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Tue, 5 Apr 2022 23:26:45 +0200 Subject: [PATCH 2/7] Update appveyor.yml --- appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 77eda149..1aaa4235 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,11 +10,11 @@ skip_commits: dotnet_csproj: patch: true file: '**\*.csproj' - version: '{build_version}' - package_version: '{build_version}' - assembly_version: '{build_version}' - file_version: '{build_version}' - informational_version: '{build_version}' + version: $(build_version) + package_version: $(build_version) + assembly_version: $(build_version) + file_version: $(build_version) + informational_version: $(build_version) install: - cmd: dotnet tool install -g InheritDocTool build_script: From ac657c8f4611f02a5f1adfdd076ec55eecbe9e90 Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Tue, 5 Apr 2022 23:46:23 +0200 Subject: [PATCH 3/7] Update OverrideTests.cs --- src/ConfigCat.Client.Tests/OverrideTests.cs | 24 ++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/ConfigCat.Client.Tests/OverrideTests.cs b/src/ConfigCat.Client.Tests/OverrideTests.cs index 52185256..646e9d9b 100644 --- a/src/ConfigCat.Client.Tests/OverrideTests.cs +++ b/src/ConfigCat.Client.Tests/OverrideTests.cs @@ -362,13 +362,35 @@ public async Task LocalFile_Watcher_Reload() Assert.AreEqual("initial", await client.GetValueAsync("fakeKey", string.Empty)); await WriteContent(SampleFileToCreate, "modified"); - await Task.Delay(1100); + await Task.Delay(1500); Assert.AreEqual("modified", await client.GetValueAsync("fakeKey", string.Empty)); File.Delete(SampleFileToCreate); } + [TestMethod] + public async Task LocalFile_Watcher_Reload_Sync() + { + await CreateFileAndWriteContent(SampleFileToCreate, "initial"); + + using var client = new ConfigCatClient(options => + { + options.SdkKey = "localhost"; + options.FlagOverrides = FlagOverrides.LocalFile(SampleFileToCreate, true, OverrideBehaviour.LocalOnly); + options.Logger.LogLevel = LogLevel.Info; + }); + + Assert.AreEqual("initial", client.GetValue("fakeKey", string.Empty)); + + await WriteContent(SampleFileToCreate, "modified"); + await Task.Delay(1500); + + Assert.AreEqual("modified", client.GetValue("fakeKey", string.Empty)); + + File.Delete(SampleFileToCreate); + } + private static string GetJsonContent(string value) { return $"{{ \"f\": {{ \"fakeKey\": {{ \"v\": \"{value}\", \"p\": [] ,\"r\": [] }} }} }}"; From 767962f535053b36049e7b863236cd53ba15a144 Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Wed, 6 Apr 2022 00:08:17 +0200 Subject: [PATCH 4/7] Sync read the file in the data source's constructor, fix GH actions trigger --- .github/workflows/linux-macOS-CI.yml | 1 + .github/workflows/sonar-analysis.yml | 1 + src/ConfigCat.Client.Tests/OverrideTests.cs | 4 +- .../Override/LocalFileDataSource.cs | 94 ++++++------------- 4 files changed, 35 insertions(+), 65 deletions(-) diff --git a/.github/workflows/linux-macOS-CI.yml b/.github/workflows/linux-macOS-CI.yml index acc5eb72..6dbd7d80 100644 --- a/.github/workflows/linux-macOS-CI.yml +++ b/.github/workflows/linux-macOS-CI.yml @@ -1,6 +1,7 @@ name: Build on Linux and macOS on: push: + branches: [ master ] paths-ignore: - '**.md' - 'appveyor*' diff --git a/.github/workflows/sonar-analysis.yml b/.github/workflows/sonar-analysis.yml index 6400fe0c..5b9f2387 100644 --- a/.github/workflows/sonar-analysis.yml +++ b/.github/workflows/sonar-analysis.yml @@ -1,6 +1,7 @@ name: SonarCloud Analysis on: push: + branches: [ master ] paths-ignore: - '**.md' - 'appveyor*' diff --git a/src/ConfigCat.Client.Tests/OverrideTests.cs b/src/ConfigCat.Client.Tests/OverrideTests.cs index 646e9d9b..a22ee16c 100644 --- a/src/ConfigCat.Client.Tests/OverrideTests.cs +++ b/src/ConfigCat.Client.Tests/OverrideTests.cs @@ -362,7 +362,7 @@ public async Task LocalFile_Watcher_Reload() Assert.AreEqual("initial", await client.GetValueAsync("fakeKey", string.Empty)); await WriteContent(SampleFileToCreate, "modified"); - await Task.Delay(1500); + await Task.Delay(2000); Assert.AreEqual("modified", await client.GetValueAsync("fakeKey", string.Empty)); @@ -384,7 +384,7 @@ public async Task LocalFile_Watcher_Reload_Sync() Assert.AreEqual("initial", client.GetValue("fakeKey", string.Empty)); await WriteContent(SampleFileToCreate, "modified"); - await Task.Delay(1500); + await Task.Delay(2000); Assert.AreEqual("modified", client.GetValue("fakeKey", string.Empty)); diff --git a/src/ConfigCatClient/Override/LocalFileDataSource.cs b/src/ConfigCatClient/Override/LocalFileDataSource.cs index d2a6ad37..81d1920e 100644 --- a/src/ConfigCatClient/Override/LocalFileDataSource.cs +++ b/src/ConfigCatClient/Override/LocalFileDataSource.cs @@ -19,7 +19,7 @@ internal sealed class LocalFileDataSource : IOverrideDataSource private readonly ILogger logger; private readonly TaskCompletionSource asyncInit = new(); private readonly ManualResetEvent syncInit = new(false); - private readonly CancellationTokenSource pollerCancellationTokenSource = new(); + private readonly CancellationTokenSource cancellationTokenSource = new(); private volatile IDictionary overrideValues; @@ -28,80 +28,58 @@ public LocalFileDataSource(string filePath, bool autoReload, ILogger logger) if (!File.Exists(filePath)) { logger.Error($"File {filePath} does not exist."); - this.SetInitialized(); return; } this.fullPath = Path.GetFullPath(filePath); this.logger = logger; - this.StartFileReading(autoReload); - } + this.ReloadFile(); - public IDictionary GetOverrides() - { - if (this.overrideValues != null) return this.overrideValues; - this.syncInit.WaitOne(); - return this.overrideValues ?? new Dictionary(); + if (autoReload) + { + this.StartWatch(); + } } - public async Task> GetOverridesAsync() - { - if (this.overrideValues != null) return this.overrideValues; - await this.asyncInit.Task.ConfigureAwait(false); - return this.overrideValues ?? new Dictionary(); - } + public IDictionary GetOverrides() => this.overrideValues ?? new Dictionary(); - private void StartFileReading(bool autoReload) - { - Task.Run(async () => - { - try - { - await ReadFileAsync(); + public Task> GetOverridesAsync() => Task.FromResult(this.overrideValues ?? new Dictionary()); - if (autoReload) - { - await this.StartWatchAsync(); - } - } - finally - { - this.SetInitialized(); - } - }); - } - private async Task StartWatchAsync() + private void StartWatch() { - this.logger.Information($"Watching {this.fullPath} for changes."); - while (!this.pollerCancellationTokenSource.IsCancellationRequested) + Task.Run(async () => { - try + this.logger.Information($"Watching {this.fullPath} for changes."); + while (!this.cancellationTokenSource.IsCancellationRequested) { try { - var lastWriteTime = File.GetLastWriteTimeUtc(this.fullPath); - if (lastWriteTime > this.fileLastWriteTime) + try { - this.logger.Information($"Reload file {this.fullPath}."); - await ReadFileAsync(); + var lastWriteTime = File.GetLastWriteTimeUtc(this.fullPath); + if (lastWriteTime > this.fileLastWriteTime) + { + this.logger.Information($"Reload file {this.fullPath}."); + this.ReloadFile(); + } } - } - finally + finally + { + await Task.Delay(FILE_POLL_INTERVAL, this.cancellationTokenSource.Token); + } + } + catch (OperationCanceledException) { - await Task.Delay(FILE_POLL_INTERVAL, this.pollerCancellationTokenSource.Token); + // ignore exceptions from cancellation. } } - catch (OperationCanceledException) - { - // ignore exceptions from cancellation. - } - } + }); } - private async Task ReadFileAsync() + private void ReloadFile() { try { @@ -109,9 +87,7 @@ private async Task ReadFileAsync() { try { - using var stream = new FileStream(this.fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - using var reader = new StreamReader(stream); - var content = await reader.ReadToEndAsync(); + var content = File.ReadAllText(this.fullPath); var simplified = content.DeserializeOrDefault(); if (simplified?.Entries != null) { @@ -130,7 +106,7 @@ private async Task ReadFileAsync() if (i >= MAX_WAIT_ITERATIONS) throw; - await Task.Delay(WAIT_TIME_FOR_UNLOCK); + Thread.Sleep(WAIT_TIME_FOR_UNLOCK); } } } @@ -141,20 +117,12 @@ private async Task ReadFileAsync() finally { this.fileLastWriteTime = File.GetLastWriteTimeUtc(this.fullPath); - this.SetInitialized(); } } - private void SetInitialized() - { - this.syncInit.Set(); - this.asyncInit.TrySetResult(true); - } - public void Dispose() { - this.pollerCancellationTokenSource.Cancel(); - this.syncInit.Dispose(); + this.cancellationTokenSource.Cancel(); } private sealed class SimplifiedConfig From 9f67e1e3b99daa849329899c41966899c74bb47f Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Wed, 6 Apr 2022 00:08:34 +0200 Subject: [PATCH 5/7] Update LocalFileDataSource.cs --- src/ConfigCatClient/Override/LocalFileDataSource.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ConfigCatClient/Override/LocalFileDataSource.cs b/src/ConfigCatClient/Override/LocalFileDataSource.cs index 81d1920e..dae02667 100644 --- a/src/ConfigCatClient/Override/LocalFileDataSource.cs +++ b/src/ConfigCatClient/Override/LocalFileDataSource.cs @@ -17,8 +17,6 @@ internal sealed class LocalFileDataSource : IOverrideDataSource private DateTime fileLastWriteTime; private readonly string fullPath; private readonly ILogger logger; - private readonly TaskCompletionSource asyncInit = new(); - private readonly ManualResetEvent syncInit = new(false); private readonly CancellationTokenSource cancellationTokenSource = new(); private volatile IDictionary overrideValues; From 53dff30568ea8780ca950c5f2b6eb33683934902 Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Wed, 6 Apr 2022 00:21:08 +0200 Subject: [PATCH 6/7] Update OverrideTests.cs --- src/ConfigCat.Client.Tests/OverrideTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ConfigCat.Client.Tests/OverrideTests.cs b/src/ConfigCat.Client.Tests/OverrideTests.cs index a22ee16c..73b4ee4b 100644 --- a/src/ConfigCat.Client.Tests/OverrideTests.cs +++ b/src/ConfigCat.Client.Tests/OverrideTests.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; using System.IO; +using System.Threading; using System.Threading.Tasks; namespace ConfigCat.Client.Tests @@ -362,7 +363,7 @@ public async Task LocalFile_Watcher_Reload() Assert.AreEqual("initial", await client.GetValueAsync("fakeKey", string.Empty)); await WriteContent(SampleFileToCreate, "modified"); - await Task.Delay(2000); + Thread.Sleep(3000); Assert.AreEqual("modified", await client.GetValueAsync("fakeKey", string.Empty)); @@ -383,8 +384,8 @@ public async Task LocalFile_Watcher_Reload_Sync() Assert.AreEqual("initial", client.GetValue("fakeKey", string.Empty)); - await WriteContent(SampleFileToCreate, "modified"); - await Task.Delay(2000); + await WriteContent(SampleFileToCreate, "modified"); + Thread.Sleep(3000); Assert.AreEqual("modified", client.GetValue("fakeKey", string.Empty)); From 01b64b9fc1414ce5986c82a698fc8dc882edc7ed Mon Sep 17 00:00:00 2001 From: Peter Csajtai Date: Wed, 6 Apr 2022 00:37:14 +0200 Subject: [PATCH 7/7] Update OverrideTests.cs --- src/ConfigCat.Client.Tests/OverrideTests.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ConfigCat.Client.Tests/OverrideTests.cs b/src/ConfigCat.Client.Tests/OverrideTests.cs index 73b4ee4b..61f31022 100644 --- a/src/ConfigCat.Client.Tests/OverrideTests.cs +++ b/src/ConfigCat.Client.Tests/OverrideTests.cs @@ -1,7 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; using System.IO; -using System.Threading; using System.Threading.Tasks; namespace ConfigCat.Client.Tests @@ -361,9 +360,9 @@ public async Task LocalFile_Watcher_Reload() }); Assert.AreEqual("initial", await client.GetValueAsync("fakeKey", string.Empty)); - + await Task.Delay(100); await WriteContent(SampleFileToCreate, "modified"); - Thread.Sleep(3000); + await Task.Delay(1500); Assert.AreEqual("modified", await client.GetValueAsync("fakeKey", string.Empty)); @@ -383,9 +382,9 @@ public async Task LocalFile_Watcher_Reload_Sync() }); Assert.AreEqual("initial", client.GetValue("fakeKey", string.Empty)); - - await WriteContent(SampleFileToCreate, "modified"); - Thread.Sleep(3000); + await Task.Delay(100); + await WriteContent(SampleFileToCreate, "modified"); + await Task.Delay(1500); Assert.AreEqual("modified", client.GetValue("fakeKey", string.Empty));