Improve stack overflow prevention (fixes #321)

dnSpyEx · May 7, 2024 · 5c26872 · 5c26872
1 parent 1ad44b1
commit 5c26872
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 6 deletions.
diff --git a/Extensions/ILSpy.Decompiler/ICSharpCode.Decompiler b/Extensions/ILSpy.Decompiler/ICSharpCode.Decompiler
diff --git a/Extensions/dnSpy.Debugger/dnSpy.Debugger.DotNet.Metadata/DmdType.CanCastTo.cs b/Extensions/dnSpy.Debugger/dnSpy.Debugger.DotNet.Metadata/DmdType.CanCastTo.cs
@@ -501,19 +501,20 @@ bool __CanCastByVarianceToInterfaceOrDelegate(DmdType target) {
 				return false;
 
 			// The original code used a list to check for infinite recursion, we'll use this code unless it throws too often
-			try {
-				// For an unknown to me reason, in .NET 8, the RuntimeHelpers.EnsureSufficientExecutionStack() method does not work.
 #if NETCOREAPP
-				if (RuntimeHelpers.TryEnsureSufficientExecutionStack())
-					throw new InsufficientExecutionStackException();
+			if (!RuntimeHelpers.TryEnsureSufficientExecutionStack()) {
+				Debug.Fail("Should probably not happen often");
+				return false;
+			}
 #else
+			try {
 				RuntimeHelpers.EnsureSufficientExecutionStack();
-#endif
 			}
 			catch (InsufficientExecutionStackException) {
 				Debug.Fail("Should probably not happen often");
 				return false;
 			}
+#endif
 
 			var inst = GetGenericArguments();
 			if (inst.Count > 0) {
+4 −1		ICSharpCode.Decompiler/ILAst/LoopsAndConditions.cs
+8 −7		ICSharpCode.Decompiler/ILAst/TypeAnalysis.cs
+0 −16		ICSharpCode.Decompiler/StackOverflow.cs