Add matrix tests

configcat · Jul 3, 2023 · 7bbf1d6 · 7bbf1d6
1 parent da8f24c
commit 7bbf1d6
Show file tree

Hide file tree

Showing 12 changed files with 1,598 additions and 107 deletions.
diff --git a/src/ConfigCat.Client.Tests/ConfigEvaluatorTestsBase.cs b/src/ConfigCat.Client.Tests/ConfigEvaluatorTestsBase.cs
@@ -1,135 +1,32 @@
-using System;
 using System.Collections.Generic;
-using System.Globalization;
-using System.IO;
 using ConfigCat.Client.Evaluation;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace ConfigCat.Client.Tests;
 
-public interface IMatrixTestDescriptor
-{
-    public string SampleJsonFileName { get; }
-    public string MatrixResultFileName { get; }
-}
-
-public abstract class ConfigEvaluatorTestsBase<TDescriptor> where TDescriptor : IMatrixTestDescriptor, new()
+public abstract class ConfigEvaluatorTestsBase<TDescriptor> : MatrixTestRunner<TDescriptor>
+    where TDescriptor : IMatrixTestDescriptor, new()
 {
 #pragma warning disable IDE1006 // Naming Styles
     private protected readonly LoggerWrapper Logger;
 #pragma warning restore IDE1006 // Naming Styles
 
-    private protected readonly Dictionary<string, Setting> config;
-
     internal readonly IRolloutEvaluator configEvaluator;
 
     public ConfigEvaluatorTestsBase()
     {
-        var descriptor = new TDescriptor();
-        this.config = GetSampleJson(descriptor.SampleJsonFileName).Deserialize<Config>()!.Settings;
-
         this.Logger = new ConsoleLogger(LogLevel.Debug).AsWrapper();
         this.configEvaluator = new RolloutEvaluator(this.Logger);
     }
 
-    protected virtual void AssertValue(string jsonFileName, string keyName, string expected, User? user)
-    {
-        var k = keyName.ToLowerInvariant();
-
-        if (k.StartsWith("bool"))
-        {
-            var actual = this.configEvaluator.Evaluate(this.config, keyName, false, user, null, this.Logger).Value;
-
-            Assert.AreEqual(bool.Parse(expected), actual, $"jsonFileName: {jsonFileName} | keyName: {keyName} | userId: {user?.Identifier}");
-        }
-        else if (k.StartsWith("double"))
-        {
-            var actual = this.configEvaluator.Evaluate(this.config, keyName, double.NaN, user, null, this.Logger).Value;
-
-            Assert.AreEqual(double.Parse(expected, CultureInfo.InvariantCulture), actual, $"jsonFileName: {jsonFileName} | keyName: {keyName} | userId: {user?.Identifier}");
-        }
-        else if (k.StartsWith("integer"))
-        {
-            var actual = this.configEvaluator.Evaluate(this.config, keyName, int.MinValue, user, null, this.Logger).Value;
-
-            Assert.AreEqual(int.Parse(expected), actual, $"jsonFileName: {jsonFileName} | keyName: {keyName} | userId: {user?.Identifier}");
-        }
-        else
-        {
-            var actual = this.configEvaluator.Evaluate(this.config, keyName, string.Empty, user, null, this.Logger).Value;
-
-            Assert.AreEqual(expected, actual, $"jsonFileName: {jsonFileName} | keyName: {keyName} | userId: {user?.Identifier}");
-        }
-    }
-
-    protected string GetSampleJson(string fileName)
-    {
-        using Stream stream = File.OpenRead(Path.Combine("data", fileName));
-        using StreamReader reader = new(stream);
-        return reader.ReadToEnd();
-    }
-
-    public static IEnumerable<object?[]> GetMatrixTests()
-    {
-        var descriptor = new TDescriptor();
-
-        var resultFilePath = Path.Combine("data", descriptor.MatrixResultFileName);
-        using var reader = new StreamReader(resultFilePath);
-        var header = reader.ReadLine()!;
-
-        var columns = header.Split(new[] { ';' });
-
-        while (!reader.EndOfStream)
-        {
-            var rawline = reader.ReadLine();
-
-            if (string.IsNullOrEmpty(rawline))
-            {
-                continue;
-            }
-
-            var row = rawline.Split(new[] { ';' });
-
-            string? userId = null, userEmail = null, userCountry = null, userCustomAttributeName = null, userCustomAttributeValue = null;
-            if (row[0] != "##null##")
-            {
-                userId = row[0];
-                userEmail = row[1] == "##null##" ? null : row[1];
-                userCountry = row[2] == "##null##" ? null : row[2];
-                if (row[3] != "##null##")
-                {
-                    userCustomAttributeName = columns[3];
-                    userCustomAttributeValue = row[3];
-                }
-            }
-
-            for (var i = 4; i < columns.Length; i++)
-            {
-                yield return new[]
-                {
-                    descriptor.SampleJsonFileName, columns[i], row[i],
-                    userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue
-                };
-            }
-        }
-    }
+    public static IEnumerable<object?[]> GetMatrixTests() => GetTests();
 
     [TestCategory("MatrixTests")]
     [DataTestMethod]
     [DynamicData(nameof(GetMatrixTests), DynamicDataSourceType.Method)]
     public void Run_MatrixTests(string jsonFileName, string settingKey, string expectedReturnValue,
         string? userId, string? userEmail, string? userCountry, string? userCustomAttributeName, string? userCustomAttributeValue)
     {
-        User? user = null;
-        if (userId is not null)
-        {
-            user = new User(userId) { Email = userEmail, Country = userCountry };
-            if (userCustomAttributeValue is not null)
-            {
-                user.Custom[userCustomAttributeName!] = userCustomAttributeValue;
-            }
-        }
-
-        AssertValue(jsonFileName, settingKey, expectedReturnValue, user);
+        RunTest(this.configEvaluator, this.Logger, settingKey, expectedReturnValue, userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue);
     }
 }
diff --git a/src/ConfigCat.Client.Tests/ConfigV6EvaluationTests.cs b/src/ConfigCat.Client.Tests/ConfigV6EvaluationTests.cs
@@ -0,0 +1,82 @@
+using System.Collections.Generic;
+using ConfigCat.Client.Evaluation;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ConfigCat.Client.Tests;
+
+[TestClass]
+public class ConfigV6EvaluationTests
+{
+    public class AndOrTestsDescriptor : IMatrixTestDescriptor
+    {
+        public string SampleJsonFileName => "sample_and_or_v6.json";
+        public string MatrixResultFileName => "testmatrix_and_or.csv";
+        public static IEnumerable<object?[]> GetTests() => MatrixTestRunner<AndOrTestsDescriptor>.GetTests();
+    }
+
+    public class ComparatorTestsDescriptor : IMatrixTestDescriptor
+    {
+        public string SampleJsonFileName => "sample_comparators_v6.json";
+        public string MatrixResultFileName => "testmatrix_comparators_v6.csv";
+        public static IEnumerable<object?[]> GetTests() => MatrixTestRunner<ComparatorTestsDescriptor>.GetTests();
+    }
+
+    public class FlagDependencyTestsDescriptor : IMatrixTestDescriptor
+    {
+        public string SampleJsonFileName => "sample_flagdependency_v6.json";
+        public string MatrixResultFileName => "testmatrix_dependent_flag.csv";
+        public static IEnumerable<object?[]> GetTests() => MatrixTestRunner<FlagDependencyTestsDescriptor>.GetTests();
+    }
+
+    public class SegmentTestsDescriptor : IMatrixTestDescriptor
+    {
+        public string SampleJsonFileName => "sample_segments_v6.json";
+        public string MatrixResultFileName => "testmatrix_segments.csv";
+        public static IEnumerable<object?[]> GetTests() => MatrixTestRunner<SegmentTestsDescriptor>.GetTests();
+    }
+
+    private readonly LoggerWrapper logger;
+    private readonly IRolloutEvaluator configEvaluator;
+
+    public ConfigV6EvaluationTests()
+    {
+        this.logger = new ConsoleLogger(LogLevel.Debug).AsWrapper();
+        this.configEvaluator = new RolloutEvaluator(this.logger);
+    }
+
+    [DataTestMethod]
+    [DynamicData(nameof(AndOrTestsDescriptor.GetTests), typeof(AndOrTestsDescriptor), DynamicDataSourceType.Method)]
+    public void AndOrTests(string jsonFileName, string settingKey, string expectedReturnValue,
+        string? userId, string? userEmail, string? userCountry, string? userCustomAttributeName, string? userCustomAttributeValue)
+    {
+        MatrixTestRunner<AndOrTestsDescriptor>.Default.RunTest(this.configEvaluator, this.logger, settingKey, expectedReturnValue,
+            userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue);
+    }
+
+    [DataTestMethod]
+    [DynamicData(nameof(ComparatorTestsDescriptor.GetTests), typeof(ComparatorTestsDescriptor), DynamicDataSourceType.Method)]
+    public void ComparatorTests(string jsonFileName, string settingKey, string expectedReturnValue,
+        string? userId, string? userEmail, string? userCountry, string? userCustomAttributeName, string? userCustomAttributeValue)
+    {
+        MatrixTestRunner<ComparatorTestsDescriptor>.Default.RunTest(this.configEvaluator, this.logger, settingKey, expectedReturnValue,
+            userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue);
+    }
+
+    [DataTestMethod]
+    [DynamicData(nameof(FlagDependencyTestsDescriptor.GetTests), typeof(FlagDependencyTestsDescriptor), DynamicDataSourceType.Method)]
+    public void FlagDependencyTests(string jsonFileName, string settingKey, string expectedReturnValue,
+        string? userId, string? userEmail, string? userCountry, string? userCustomAttributeName, string? userCustomAttributeValue)
+    {
+        MatrixTestRunner<FlagDependencyTestsDescriptor>.Default.RunTest(this.configEvaluator, this.logger, settingKey, expectedReturnValue,
+            userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue);
+    }
+
+    [DataTestMethod]
+    [DynamicData(nameof(SegmentTestsDescriptor.GetTests), typeof(SegmentTestsDescriptor), DynamicDataSourceType.Method)]
+    public void SegmentTests(string jsonFileName, string settingKey, string expectedReturnValue,
+        string? userId, string? userEmail, string? userCountry, string? userCustomAttributeName, string? userCustomAttributeValue)
+    {
+        MatrixTestRunner<SegmentTestsDescriptor>.Default.RunTest(this.configEvaluator, this.logger, settingKey, expectedReturnValue,
+            userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue);
+    }
+}
diff --git a/src/ConfigCat.Client.Tests/Helpers/ConfigHelper.cs b/src/ConfigCat.Client.Tests/Helpers/ConfigHelper.cs
@@ -14,4 +14,11 @@ public static ProjectConfig FromFile(string configJsonFilePath, string? httpETag
     {
         return FromString(File.ReadAllText(configJsonFilePath), httpETag, timeStamp);
     }
+
+    public static string GetSampleJson(string fileName)
+    {
+        using Stream stream = File.OpenRead(Path.Combine("data", fileName));
+        using StreamReader reader = new(stream);
+        return reader.ReadToEnd();
+    }
 }
diff --git a/src/ConfigCat.Client.Tests/MatrixTestRunner.cs b/src/ConfigCat.Client.Tests/MatrixTestRunner.cs
@@ -0,0 +1,125 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using ConfigCat.Client.Evaluation;
+using ConfigCat.Client.Tests.Helpers;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ConfigCat.Client.Tests;
+
+public interface IMatrixTestDescriptor
+{
+    public string SampleJsonFileName { get; }
+    public string MatrixResultFileName { get; }
+}
+
+public class MatrixTestRunner<TDescriptor> where TDescriptor : IMatrixTestDescriptor, new()
+{
+    private static readonly Lazy<MatrixTestRunner<TDescriptor>> DefaultLazy = new(() => new MatrixTestRunner<TDescriptor>(), isThreadSafe: true);
+    public static MatrixTestRunner<TDescriptor> Default => DefaultLazy.Value;
+
+    public static readonly TDescriptor DescriptorInstance = new();
+
+    private protected readonly Dictionary<string, Setting> config;
+
+    public MatrixTestRunner()
+    {
+        this.config = ConfigHelper.GetSampleJson(DescriptorInstance.SampleJsonFileName).Deserialize<Config>()!.Settings;
+    }
+
+    public static IEnumerable<object?[]> GetTests()
+    {
+        var resultFilePath = Path.Combine("data", DescriptorInstance.MatrixResultFileName);
+        using var reader = new StreamReader(resultFilePath);
+        var header = reader.ReadLine()!;
+
+        var columns = header.Split(new[] { ';' });
+
+        while (!reader.EndOfStream)
+        {
+            var rawline = reader.ReadLine();
+
+            if (string.IsNullOrEmpty(rawline))
+                continue;
+
+            var row = rawline.Split(new[] { ';' });
+
+            string? userId = null, userEmail = null, userCountry = null, userCustomAttributeName = null, userCustomAttributeValue = null;
+            if (row[0] != "##null##")
+            {
+                userId = row[0];
+                userEmail = row[1] is "" or "##null##" ? null : row[1];
+                userCountry = row[2] is "" or "##null##" ? null : row[2];
+                if (row[3] is not ("" or "##null##"))
+                {
+                    userCustomAttributeName = columns[3];
+                    userCustomAttributeValue = row[3];
+                }
+            }
+
+            for (var i = 4; i < columns.Length; i++)
+            {
+                yield return new[]
+                {
+                    DescriptorInstance.SampleJsonFileName, columns[i], row[i],
+                    userId, userEmail, userCountry, userCustomAttributeName, userCustomAttributeValue
+                };
+            }
+        }
+    }
+
+    protected virtual bool AssertValue<T>(string expected, Func<string, T> parse, T actual, string keyName, string? userId)
+    {
+        Assert.AreEqual(parse(expected), actual, $"jsonFileName: {DescriptorInstance.SampleJsonFileName} | keyName: {keyName} | userId: {userId}");
+        return true;
+    }
+
+    internal bool RunTest(IRolloutEvaluator evaluator, LoggerWrapper logger, string settingKey, string expectedReturnValue, User? user = null)
+    {
+        if (settingKey.StartsWith("bool", StringComparison.OrdinalIgnoreCase))
+        {
+            var actual = evaluator.Evaluate(this.config, settingKey, false, user, null, logger).Value;
+
+            return AssertValue(expectedReturnValue, static e => bool.Parse(e), actual, settingKey, user?.Identifier);
+        }
+        else if (settingKey.StartsWith("double", StringComparison.OrdinalIgnoreCase))
+        {
+            var actual = evaluator.Evaluate(this.config, settingKey, double.NaN, user, null, logger).Value;
+
+            return AssertValue(expectedReturnValue, static e => double.Parse(e, CultureInfo.InvariantCulture), actual, settingKey, user?.Identifier);
+        }
+        else if (settingKey.StartsWith("integer", StringComparison.OrdinalIgnoreCase))
+        {
+            var actual = evaluator.Evaluate(this.config, settingKey, int.MinValue, user, null, logger).Value;
+
+            return AssertValue(expectedReturnValue, static e => int.Parse(e, CultureInfo.InvariantCulture), actual, settingKey, user?.Identifier);
+        }
+        else if (settingKey.StartsWith("string", StringComparison.OrdinalIgnoreCase))
+        {
+            var actual = evaluator.Evaluate(this.config, settingKey, string.Empty, user, null, logger).Value;
+
+            return AssertValue(expectedReturnValue, static e => e, actual, settingKey, user?.Identifier);
+        }
+        else
+        {
+            var actual = evaluator.Evaluate(this.config, settingKey, (object?)null, user, null, logger).Value;
+
+            return AssertValue(expectedReturnValue, static e => e, Convert.ToString(actual, CultureInfo.InvariantCulture), settingKey, user?.Identifier);
+        }
+    }
+
+    internal bool RunTest(IRolloutEvaluator evaluator, LoggerWrapper logger, string settingKey, string expectedReturnValue,
+        string? userId, string? userEmail, string? userCountry, string? userCustomAttributeName, string? userCustomAttributeValue)
+    {
+        User? user = null;
+        if (userId is not null)
+        {
+            user = new User(userId) { Email = userEmail, Country = userCountry };
+            if (userCustomAttributeValue is not null)
+                user.Custom[userCustomAttributeName!] = userCustomAttributeValue;
+        }
+
+        return RunTest(evaluator, logger, settingKey, expectedReturnValue, user);
+    }
+}