ILVerify: add a validation-only mode

src/coreclr/tools/ILVerification.Tests/ILMethodTester.cs

@@ -54,15 +54,32 @@ public static void TestMethodsWithInvalidIL(InvalidILTestCase invalidIL)
             }
         }
 
-        private static IEnumerable<VerificationResult> Verify(TestCase testCase)
+        [Theory(DisplayName = "")]
+        [MemberData(nameof(TestDataLoader.GetMethodsForValidationOnlyMode), MemberType = typeof(TestDataLoader))]
+        [Trait("", "Validation-only IL Tests")]
+        public static void TestMethodsInValidationOnlyMode(ValidationOnlyTestCase testCase)
+        {
+            // Unlike verification, validation must not throw for invalid IL, so exceptions are not caught here.
+            var results = Verify(testCase, validationOnly: true).ToList();
+            var actual = string.Join(",", results.Select(e => e.Code.ToString()));
+
+            Assert.True(testCase.ExpectedVerifierErrors.Count == results.Count, $"Actual errors were: {actual}");
+            foreach (var item in testCase.ExpectedVerifierErrors)
+            {
+                Assert.True(results.Any(r => r.Code == item), $"Actual errors were: {actual}");
+            }
+        }
+
+        private static IEnumerable<VerificationResult> Verify(TestCase testCase, bool validationOnly = false)
         {
             EcmaModule module = TestDataLoader.GetModuleForTestAssembly(testCase.ModuleName, allowIgnoresAccessChecksTo: true);
             var methodHandle = (MethodDefinitionHandle) MetadataTokens.EntityHandle(testCase.MetadataToken);
             var method = module.GetMethod(methodHandle);
             var verifier = new Verifier((ILVerifyTypeSystemContext)method.Context, new VerifierOptions
             {
                 IncludeMetadataTokensInErrorMessages = true,
-                SanityChecks = true
+                SanityChecks = true,
+                ValidationOnly = validationOnly
             });
 
             return verifier.Verify(module.PEReader, methodHandle);

src/coreclr/tools/ILVerification.Tests/ILTests/BranchingTests.il

@@ -81,7 +81,7 @@
         IL_0014: ret
     }
 
-    .method static public hidebysig void Branching.NullConditional_Invalid_StackUnexpected() cil managed
+    .method static public hidebysig void Branching.NullConditional_Invalid_StackUnexpected_Validation_Valid() cil managed
     {
         //object o = null;
         //Type t = o != null ? o.GetType() : o;
@@ -175,7 +175,7 @@
         ret
     }
 
-    .method static public hidebysig void Branching.UnequalStackMergeArrays_Invalid_StackUnexpected(class CustomComparer[] comparers,
+    .method static public hidebysig void Branching.UnequalStackMergeArrays_Invalid_StackUnexpected_Validation_Valid(class CustomComparer[] comparers,
         class [System.Runtime]System.Collections.IEqualityComparer[] customComparer) cil managed
     {
         .maxstack 1
@@ -949,7 +949,7 @@
         ret
     }
 
-    .method static public hidebysig void Branch.BackwardWithoutPredecessor_Invalid_BackwardBranch() cil managed 
+    .method static public hidebysig void Branch.BackwardWithoutPredecessor_Invalid_BackwardBranch_Validation_Valid() cil managed 
     {
         br      MethodEnd
 

src/coreclr/tools/ILVerification.Tests/ILTests/CallTests.il

@@ -54,7 +54,7 @@
         ret
     }
 
-    .method public hidebysig instance void Call.BaseMethodModThisLdarga_Invalid_ThisMismatch() cil managed
+    .method public hidebysig instance void Call.BaseMethodModThisLdarga_Invalid_ThisMismatch_Validation_Valid() cil managed
     {
         ldarg.0
         call    instance void SimpleClass::VirtualMethod()
@@ -63,7 +63,7 @@
         ret
     }
 
-    .method public hidebysig instance void Call.BaseMethodModThisStarg_Invalid_ThisMismatch() cil managed
+    .method public hidebysig instance void Call.BaseMethodModThisStarg_Invalid_ThisMismatch_Validation_Valid() cil managed
     {
         ldarg.0
         call    instance void SimpleClass::VirtualMethod()
@@ -128,7 +128,7 @@
         ret
     }
 
-    .method public hidebysig instance void Call.ExternBaseMethod_Invalid_ThisMismatch(class DerivedClass c) cil managed
+    .method public hidebysig instance void Call.ExternBaseMethod_Invalid_ThisMismatch_Validation_Valid(class DerivedClass c) cil managed
     {
         ldarg.1
         call    instance void SimpleClass::VirtualMethod()

src/coreclr/tools/ILVerification.Tests/ILTests/CalliTests.il

@@ -35,22 +35,22 @@
         ret
     }
 
-    .method public hidebysig static void Calli.StaticFromLdftn_Invalid_Unverifiable() cil managed
+    .method public hidebysig static void Calli.StaticFromLdftn_Invalid_Unverifiable_Validation_Valid() cil managed
     {
         ldc.i4.1
         ldftn void CalliTests::StaticTarget(int32)
         calli void(int32)
         ret
     }
 
-    .method public hidebysig static int32 Calli.ReturnValue_Invalid_Unverifiable() cil managed
+    .method public hidebysig static int32 Calli.ReturnValue_Invalid_Unverifiable_Validation_Valid() cil managed
     {
         ldftn int32 CalliTests::ReturnsInt32()
         calli int32()
         ret
     }
 
-    .method public hidebysig static void Calli.ErrorAfterCalli_Invalid_Unverifiable.StackUnderflow() cil managed
+    .method public hidebysig static void Calli.ErrorAfterCalli_Invalid_Unverifiable.StackUnderflow_Validation_StackUnderflow() cil managed
     {
         ldc.i4.1
         ldftn void CalliTests::StaticTarget(int32)
@@ -59,31 +59,31 @@
         ret
     }
 
-    .method public hidebysig static void Calli.FromArgument_Invalid_Unverifiable(method void *(int32) f) cil managed
+    .method public hidebysig static void Calli.FromArgument_Invalid_Unverifiable_Validation_Valid(method void *(int32) f) cil managed
     {
         ldc.i4.1
         ldarg.0
         calli void(int32)
         ret
     }
 
-    .method public hidebysig static void Calli.ArgumentMismatch_Invalid_Unverifiable.StackUnexpected(method void *(int32) f) cil managed
+    .method public hidebysig static void Calli.ArgumentMismatch_Invalid_Unverifiable.StackUnexpected_Validation_StackUnexpected(method void *(int32) f) cil managed
     {
         ldnull
         ldarg.0
         calli void(int32)
         ret
     }
 
-    .method public hidebysig static void Calli.MethodPointerNotNativeInt_Invalid_Unverifiable.StackMethod() cil managed
+    .method public hidebysig static void Calli.MethodPointerNotNativeInt_Invalid_Unverifiable.StackMethod_Validation_StackMethod() cil managed
     {
         ldc.i4.1
         ldc.i4.0
         calli void(int32)
         ret
     }
 
-    .method public hidebysig instance void Calli.ImplicitThis_Invalid_Unverifiable(method instance void *(int32) f) cil managed
+    .method public hidebysig instance void Calli.ImplicitThis_Invalid_Unverifiable_Validation_Valid(method instance void *(int32) f) cil managed
     {
         ldarg.0
         ldc.i4.1
@@ -92,7 +92,7 @@
         ret
     }
 
-    .method public hidebysig instance void Calli.ExplicitThis_Invalid_Unverifiable() cil managed
+    .method public hidebysig instance void Calli.ExplicitThis_Invalid_Unverifiable_Validation_Valid() cil managed
     {
         ldarg.0
         ldc.i4.1
@@ -102,22 +102,22 @@
     }
 
     // A vararg call site with no extra arguments has no sentinel.
-    .method public hidebysig static void Calli.Vararg_Invalid_Unverifiable(method vararg void *(int32) f) cil managed
+    .method public hidebysig static void Calli.Vararg_Invalid_Unverifiable_Validation_Valid(method vararg void *(int32) f) cil managed
     {
         ldc.i4.1
         ldarg.0
         calli vararg void(int32)
         ret
     }
 
-    .method public hidebysig static void Calli.TailCall_Invalid_Unverifiable() cil managed
+    .method public hidebysig static void Calli.TailCall_Invalid_Unverifiable_Validation_Valid() cil managed
     {
         ldftn void CalliTests::ManagedTarget()
         tail. calli void()
         ret
     }
 
-    .method public hidebysig static void Calli.TailCallByRefArgument_Invalid_Unverifiable.TailByRef(method void *(int32&) f) cil managed
+    .method public hidebysig static void Calli.TailCallByRefArgument_Invalid_Unverifiable.TailByRef_Validation_TailByRef(method void *(int32&) f) cil managed
     {
         .locals init (int32 V_0)
         ldloca.s V_0
@@ -126,7 +126,7 @@
         ret
     }
 
-    .method public hidebysig static void Calli.TailCallByRefThis_Invalid_Unverifiable.TailByRef(method instance void *() f) cil managed
+    .method public hidebysig static void Calli.TailCallByRefThis_Invalid_Unverifiable.TailByRef_Validation_TailByRef(method instance void *() f) cil managed
     {
         .locals init (int32 V_0)
         ldloca.s V_0
@@ -135,7 +135,7 @@
         ret
     }
 
-    .method public hidebysig static void Calli.TailCallNotFollowedByRet_Invalid_Unverifiable.TailRet() cil managed
+    .method public hidebysig static void Calli.TailCallNotFollowedByRet_Invalid_Unverifiable.TailRet_Validation_TailRet() cil managed
     {
         ldftn void CalliTests::ManagedTarget()
         tail. calli void()
@@ -145,22 +145,22 @@
 
     // The tail call's return checks are shared with call. A void method cannot tail-call a method returning a value;
     // the value then left on the stack also makes the ret report ReturnVoid.
-    .method public hidebysig static void Calli.TailCallReturnsValue_Invalid_Unverifiable.TailRetVoid.ReturnVoid() cil managed
+    .method public hidebysig static void Calli.TailCallReturnsValue_Invalid_Unverifiable.TailRetVoid.ReturnVoid_Validation_TailRetVoid.ReturnVoid() cil managed
     {
         ldftn int32 CalliTests::ReturnsInt32()
         tail. calli int32()
         ret
     }
 
-    .method public hidebysig static int32& Calli.ByRefReturn_Invalid_Unverifiable(method int32& *() f) cil managed
+    .method public hidebysig static int32& Calli.ByRefReturn_Invalid_Unverifiable_Validation_Valid(method int32& *() f) cil managed
     {
         ldarg.0
         calli int32&()
         ret
     }
 
     // The calli operand is the TypeDef token of CalliTests (0x02000002), not a StandAloneSig token.
-    .method public hidebysig static void Calli.NotSignatureToken_Invalid_Unverifiable.ExpectedSignatureToken() cil managed
+    .method public hidebysig static void Calli.NotSignatureToken_Invalid_Unverifiable.ExpectedSignatureToken_Validation_ExpectedSignatureToken() cil managed
     {
         ldftn void CalliTests::ManagedTarget()
         .emitbyte 0x29
@@ -193,7 +193,7 @@
         ret
     }
 
-    .method public hidebysig instance void 'special.CalliUninitThis..ctor_Invalid_Unverifiable.UninitStack'(method void *(object) f) cil managed
+    .method public hidebysig instance void 'special.CalliUninitThis..ctor_Invalid_Unverifiable.UninitStack_Validation_UninitStack'(method void *(object) f) cil managed
     {
         ret
     }

src/coreclr/tools/ILVerification.Tests/ILTests/CastingTests.il

@@ -234,7 +234,7 @@
         ret
     }
 
-    .method public hidebysig instance void Casting.AssignThisToSameTypeWithOtherGenericArgs_Invalid_StackUnexpected() cil managed
+    .method public hidebysig instance void Casting.AssignThisToSameTypeWithOtherGenericArgs_Invalid_StackUnexpected_Validation_Valid() cil managed
     {
         .locals init (
             class GenericCastingTestsType`1<int32> V_0
@@ -257,7 +257,7 @@
         ret
     }
 
-    .method public hidebysig static void Casting.AssignToSameTypeWithOtherGenericArgs_Invalid_StackUnexpected() cil managed
+    .method public hidebysig static void Casting.AssignToSameTypeWithOtherGenericArgs_Invalid_StackUnexpected_Validation_Valid() cil managed
     {
         .locals init (
             class GenericCastingTestsType`1<int32> V_0,

src/coreclr/tools/ILVerification.Tests/ILTests/DelegateTests.il

@@ -38,55 +38,55 @@
 
     // assignment from Func<string, int> to Func<object, int> is invalid
     .method private hidebysig instance class [System.Runtime]System.Func`2<object,int32>
-          DelegateAssignmentParameter_Invalid_StackUnexpected(class [System.Runtime]System.Func`2<string,int32> input) cil managed
+          DelegateAssignmentParameter_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Func`2<string,int32> input) cil managed
     {
         ldarg.1
         ret
     }
 
     // assignment from Func<int, object> to Func<int, string> is invalid
     .method private hidebysig instance class [System.Runtime]System.Func`2<int32,string>
-          DelegateAssignment_Invalid_StackUnexpected(class [System.Runtime]System.Func`2<int32,object> input) cil managed
+          DelegateAssignment_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Func`2<int32,object> input) cil managed
     {
         ldarg.1
         ret
     }
 
     // assignment from Func<int> to Func<byte> is invalid
     .method private hidebysig instance class [System.Runtime]System.Func`1<uint8>
-          AssignIntFuncToByteFunc_Invalid_StackUnexpected(class [System.Runtime]System.Func`2<int32> input) cil managed
+          AssignIntFuncToByteFunc_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Func`2<int32> input) cil managed
     {
         ldarg.1
         ret
     }
 
     // assignment from Func<byte> to Func<int> is invalid
     .method private hidebysig instance class [System.Runtime]System.Func`1<int32>
-          AssignByteFuncToIntFunc_Invalid_StackUnexpected(class [System.Runtime]System.Func`1<uint8> input) cil managed
+          AssignByteFuncToIntFunc_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Func`1<uint8> input) cil managed
     {
         ldarg.1
         ret
     }
 
     // assignment from Func<ByteEnum> to Func<byte> is invalid
     .method private hidebysig instance class [System.Runtime]System.Func`1<uint8>
-          AssignByteActionToIntAction_Invalid_StackUnexpected(class [System.Runtime]System.Func`1<valuetype ByteEnum> input) cil managed
+          AssignByteActionToIntAction_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Func`1<valuetype ByteEnum> input) cil managed
     {
         ldarg.1
         ret
     }
 
     // assignment from Action<byte> to Action<int> is invalid
     .method private hidebysig instance class [System.Runtime]System.Action`1<int32>
-          AssignByteActionToIntAction_Invalid_StackUnexpected(class [System.Runtime]System.Action`1<uint8> input) cil managed
+          AssignByteActionToIntAction_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Action`1<uint8> input) cil managed
     {
         ldarg.1
         ret
     }
 
     // assignment from Action<ByteEnum> to Action<byte> is invalid
     .method private hidebysig instance class [System.Runtime]System.Action`1<uint8>
-          AssignByteActionToByteEnumAction_Invalid_StackUnexpected(class [System.Runtime]System.Action`1<valuetype ByteEnum> input) cil managed
+          AssignByteActionToByteEnumAction_Invalid_StackUnexpected_Validation_Valid(class [System.Runtime]System.Action`1<valuetype ByteEnum> input) cil managed
     {
         ldarg.1
         ret

src/coreclr/tools/ILVerification.Tests/ILTests/ExceptionRegionTests.il

@@ -1899,7 +1899,7 @@
         ret
     }
 
-    .method public static void Filter.NestedInCatchWithoutCast_Invalid_StackUnexpected() cil managed
+    .method public static void Filter.NestedInCatchWithoutCast_Invalid_StackUnexpected_Validation_Valid() cil managed
     {
         .maxstack 1
 
@@ -1957,7 +1957,7 @@
         ret
     }
 
-    .method public static void Filter.ExceptionLeftOnStack_Invalid_EndfilterStack() cil managed
+    .method public static void Filter.ExceptionLeftOnStack_Invalid_EndfilterStack_Validation_Valid() cil managed
     {
         .maxstack 2