Cached interface dispatch for coreclr

src/coreclr/clrfeatures.cmake

@@ -38,4 +38,16 @@ endif()
 
 if (CLR_CMAKE_TARGET_WIN32)
   set(FEATURE_TYPEEQUIVALENCE 1)
-endif(CLR_CMAKE_TARGET_WIN32)
+endif(CLR_CMAKE_TARGET_WIN32)
+
+if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)
+  if (CLR_CMAKE_TARGET_UNIX AND CLR_CMAKE_TARGET_ARCH_AMD64)
+    # Allow 16 byte compare-exchange (cmpxchg16b)
+    add_compile_options(-mcx16)
+  endif()
+  set(FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH 1)
+endif()
+
+if (NOT (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64))
+  set(FEATURE_CORECLR_VIRTUAL_STUB_DISPATCH 1)
+endif()

src/coreclr/crossgen-corelib.proj

@@ -118,7 +118,10 @@
       <CrossGenDllCmd Condition="'$(TargetsAndroid)' == 'true'">$(CrossGenDllCmd) --targetos:linux</CrossGenDllCmd>
       <CrossGenDllCmd Condition="'$(UsingToolIbcOptimization)' != 'true' and '$(EnableNgenOptimization)' == 'true'">$(CrossGenDllCmd) -m:$(MergedMibcPath) --embed-pgo-data</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) -O</CrossGenDllCmd>
+      <!-- Enable type and field layout verification to make it easier to catch when the crossgen2 type layout engine and the runtime disagree on layout conventions -->
       <CrossGenDllCmd  Condition="'$(Configuration)' == 'Debug' or '$(Configuration)' == 'Checked'">$(CrossGenDllCmd) --verify-type-and-field-layout</CrossGenDllCmd>
+      <!-- Enable Cached Interface Dispatch layout rules for the StubDispatch import section. This allows testing of this path on platforms that do not require CachedInterface Dispatch -->
+      <CrossGenDllCmd  Condition="'$(Configuration)' == 'Debug' or '$(Configuration)' == 'Checked'">$(CrossGenDllCmd) --enable-cached-interface-dispatch-support</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) @(CoreLib)</CrossGenDllCmd>
     </PropertyGroup>
 

src/coreclr/debug/CMakeLists.txt

@@ -1,3 +1,11 @@
+if(FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH)
+    add_definitions(-DFEATURE_CACHED_INTERFACE_DISPATCH)
+endif(FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH)
+
+if(FEATURE_CORECLR_VIRTUAL_STUB_DISPATCH)
+    add_definitions(-DFEATURE_VIRTUAL_STUB_DISPATCH)
+endif(FEATURE_CORECLR_VIRTUAL_STUB_DISPATCH)
+
 add_subdirectory(daccess)
 add_subdirectory(ee)
 add_subdirectory(di)

src/coreclr/debug/daccess/dacdbiimpl.cpp

@@ -3544,7 +3544,9 @@ void DacDbiInterfaceImpl::EnumerateMemRangesForLoaderAllocator(PTR_LoaderAllocat
     if (pVcsMgr)
     {
         if (pVcsMgr->indcell_heap != NULL) heapsToEnumerate.Push(pVcsMgr->indcell_heap);
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
         if (pVcsMgr->cache_entry_heap != NULL) heapsToEnumerate.Push(pVcsMgr->cache_entry_heap);
+#endif
     }
 
     TADDR rangeAccumAsTaddr = TO_TADDR(rangeAcummulator);

src/coreclr/debug/daccess/request.cpp

@@ -3595,14 +3595,18 @@ ClrDataAccess::TraverseVirtCallStubHeap(CLRDATA_ADDRESS pAppDomain, VCSHeapType
                 break;
 
             case CacheEntryHeap:
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
                 pLoaderHeap = pVcsMgr->cache_entry_heap;
+#else
+                hr = S_OK;
+#endif
                 break;
 
             default:
                 hr = E_INVALIDARG;
         }
 
-        if (SUCCEEDED(hr))
+        if (SUCCEEDED(hr) && (pLoaderHeap != NULL))
         {
             hr = TraverseLoaderHeapBlock(pLoaderHeap->m_pFirstBlock, pFunc);
         }
@@ -3645,7 +3649,9 @@ static const char *LoaderAllocatorLoaderHeapNames[] =
     "FixupPrecodeHeap",
     "NewStubPrecodeHeap",
     "IndcellHeap",
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
     "CacheEntryHeap",
+#endif
 };
 
 
@@ -3689,7 +3695,9 @@ HRESULT ClrDataAccess::GetLoaderAllocatorHeaps(CLRDATA_ADDRESS loaderAllocatorAd
             else
             {
                 pLoaderHeaps[i++] = HOST_CDADDR(pVcsMgr->indcell_heap);
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
                 pLoaderHeaps[i++] = HOST_CDADDR(pVcsMgr->cache_entry_heap);
+#endif
             }
 
             // All of the above are "LoaderHeap" and not the ExplicitControl version.

src/coreclr/inc/CrstTypes.def

@@ -532,3 +532,6 @@ End
 Crst PerfMap
     AcquiredAfter CodeVersioning AssemblyList
 End
+
+Crst InterfaceDispatchGlobalLists
+End

src/coreclr/inc/crsttypes_generated.h

@@ -60,66 +60,67 @@ enum CrstType
     CrstILStubGen = 42,
     CrstInlineTrackingMap = 43,
     CrstInstMethodHashTable = 44,
-    CrstInterop = 45,
-    CrstInteropData = 46,
-    CrstIsJMCMethod = 47,
-    CrstISymUnmanagedReader = 48,
-    CrstJit = 49,
-    CrstJitInlineTrackingMap = 50,
-    CrstJitPatchpoint = 51,
-    CrstJumpStubCache = 52,
-    CrstLeafLock = 53,
-    CrstListLock = 54,
-    CrstLoaderAllocator = 55,
-    CrstLoaderAllocatorReferences = 56,
-    CrstLoaderHeap = 57,
-    CrstManagedObjectWrapperMap = 58,
-    CrstMethodDescBackpatchInfoTracker = 59,
-    CrstMethodTableExposedObject = 60,
-    CrstModule = 61,
-    CrstModuleLookupTable = 62,
-    CrstMulticoreJitHash = 63,
-    CrstMulticoreJitManager = 64,
-    CrstNativeImageEagerFixups = 65,
-    CrstNativeImageLoad = 66,
-    CrstNotifyGdb = 67,
-    CrstPEImage = 68,
-    CrstPendingTypeLoadEntry = 69,
-    CrstPerfMap = 70,
-    CrstPgoData = 71,
-    CrstPinnedByrefValidation = 72,
-    CrstPinnedHeapHandleTable = 73,
-    CrstProfilerGCRefDataFreeList = 74,
-    CrstProfilingAPIStatus = 75,
-    CrstRCWCache = 76,
-    CrstRCWCleanupList = 77,
-    CrstReadyToRunEntryPointToMethodDescMap = 78,
-    CrstReflection = 79,
-    CrstReJITGlobalRequest = 80,
-    CrstRetThunkCache = 81,
-    CrstSigConvert = 82,
-    CrstSingleUseLock = 83,
-    CrstStressLog = 84,
-    CrstStubCache = 85,
-    CrstStubDispatchCache = 86,
-    CrstSyncBlockCache = 87,
-    CrstSyncHashLock = 88,
-    CrstSystemDomain = 89,
-    CrstSystemDomainDelayedUnloadList = 90,
-    CrstThreadIdDispenser = 91,
-    CrstThreadLocalStorageLock = 92,
-    CrstThreadStore = 93,
-    CrstTieredCompilation = 94,
-    CrstTypeEquivalenceMap = 95,
-    CrstTypeIDMap = 96,
-    CrstUMEntryThunkCache = 97,
-    CrstUMEntryThunkFreeListLock = 98,
-    CrstUniqueStack = 99,
-    CrstUnresolvedClassLock = 100,
-    CrstUnwindInfoTableLock = 101,
-    CrstVSDIndirectionCellLock = 102,
-    CrstWrapperTemplate = 103,
-    kNumberOfCrstTypes = 104
+    CrstInterfaceDispatchGlobalLists = 45,
+    CrstInterop = 46,
+    CrstInteropData = 47,
+    CrstIsJMCMethod = 48,
+    CrstISymUnmanagedReader = 49,
+    CrstJit = 50,
+    CrstJitInlineTrackingMap = 51,
+    CrstJitPatchpoint = 52,
+    CrstJumpStubCache = 53,
+    CrstLeafLock = 54,
+    CrstListLock = 55,
+    CrstLoaderAllocator = 56,
+    CrstLoaderAllocatorReferences = 57,
+    CrstLoaderHeap = 58,
+    CrstManagedObjectWrapperMap = 59,
+    CrstMethodDescBackpatchInfoTracker = 60,
+    CrstMethodTableExposedObject = 61,
+    CrstModule = 62,
+    CrstModuleLookupTable = 63,
+    CrstMulticoreJitHash = 64,
+    CrstMulticoreJitManager = 65,
+    CrstNativeImageEagerFixups = 66,
+    CrstNativeImageLoad = 67,
+    CrstNotifyGdb = 68,
+    CrstPEImage = 69,
+    CrstPendingTypeLoadEntry = 70,
+    CrstPerfMap = 71,
+    CrstPgoData = 72,
+    CrstPinnedByrefValidation = 73,
+    CrstPinnedHeapHandleTable = 74,
+    CrstProfilerGCRefDataFreeList = 75,
+    CrstProfilingAPIStatus = 76,
+    CrstRCWCache = 77,
+    CrstRCWCleanupList = 78,
+    CrstReadyToRunEntryPointToMethodDescMap = 79,
+    CrstReflection = 80,
+    CrstReJITGlobalRequest = 81,
+    CrstRetThunkCache = 82,
+    CrstSigConvert = 83,
+    CrstSingleUseLock = 84,
+    CrstStressLog = 85,
+    CrstStubCache = 86,
+    CrstStubDispatchCache = 87,
+    CrstSyncBlockCache = 88,
+    CrstSyncHashLock = 89,
+    CrstSystemDomain = 90,
+    CrstSystemDomainDelayedUnloadList = 91,
+    CrstThreadIdDispenser = 92,
+    CrstThreadLocalStorageLock = 93,
+    CrstThreadStore = 94,
+    CrstTieredCompilation = 95,
+    CrstTypeEquivalenceMap = 96,
+    CrstTypeIDMap = 97,
+    CrstUMEntryThunkCache = 98,
+    CrstUMEntryThunkFreeListLock = 99,
+    CrstUniqueStack = 100,
+    CrstUnresolvedClassLock = 101,
+    CrstUnwindInfoTableLock = 102,
+    CrstVSDIndirectionCellLock = 103,
+    CrstWrapperTemplate = 104,
+    kNumberOfCrstTypes = 105
 };
 
 #endif // __CRST_TYPES_INCLUDED
@@ -175,6 +176,7 @@ int g_rgCrstLevelMap[] =
     6,          // CrstILStubGen
     2,          // CrstInlineTrackingMap
     18,         // CrstInstMethodHashTable
+    0,          // CrstInterfaceDispatchGlobalLists
     21,         // CrstInterop
     9,          // CrstInteropData
     0,          // CrstIsJMCMethod
@@ -284,6 +286,7 @@ LPCSTR g_rgCrstNameMap[] =
     "CrstILStubGen",
     "CrstInlineTrackingMap",
     "CrstInstMethodHashTable",
+    "CrstInterfaceDispatchGlobalLists",
     "CrstInterop",
     "CrstInteropData",
     "CrstIsJMCMethod",

src/coreclr/jit/compiler.h

@@ -8538,16 +8538,8 @@ class Compiler
             reg     = REG_EAX;
             regMask = RBM_EAX;
 #elif defined(TARGET_AMD64)
-            if (isNativeAOT)
-            {
-                reg     = REG_R10;
-                regMask = RBM_R10;
-            }
-            else
-            {
-                reg     = REG_R11;
-                regMask = RBM_R11;
-            }
+            reg     = REG_R11;
+            regMask = RBM_R11;
 #elif defined(TARGET_ARM)
             if (isNativeAOT)
             {

src/coreclr/minipal/minipal.h

@@ -76,3 +76,16 @@ class VMToOSInterface
     //  true if it succeeded, false if it failed
     static bool ReleaseRWMapping(void* pStart, size_t size);
 };
+
+#if defined(HOST_64BIT) && defined(FEATURE_CACHED_INTERFACE_DISPATCH)
+EXTERN_C uint8_t _InterlockedCompareExchange128(int64_t volatile *, int64_t, int64_t, int64_t *);
+
+#if defined(HOST_WINDOWS)
+#pragma intrinsic(_InterlockedCompareExchange128)
+#endif
+
+FORCEINLINE uint8_t PalInterlockedCompareExchange128(_Inout_ int64_t volatile *pDst, int64_t iValueHigh, int64_t iValueLow, int64_t *pComparandAndResult)
+{
+    return _InterlockedCompareExchange128(pDst, iValueHigh, iValueLow, pComparandAndResult);
+}
+#endif // defined(HOST_64BIT) && defined(FEATURE_CACHED_INTERFACE_DISPATCH)

src/coreclr/nativeaot/CMakeLists.txt

@@ -23,7 +23,7 @@ if(CLR_CMAKE_HOST_UNIX)
   endif(CLR_CMAKE_TARGET_APPLE)
 
   if(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_I386)
-    # Allow 16 byte compare-exchange
+    # Allow 16 byte compare-exchange (cmpxchg16b)
     add_compile_options(-mcx16)
   endif(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_I386)
 endif (CLR_CMAKE_HOST_UNIX)

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -1,9 +1,11 @@
 set(GC_DIR ../../gc)
+set(SHARED_RUNTIME_DIR ../../shared_runtime)
 
 set(COMMON_RUNTIME_SOURCES
     allocheap.cpp
     rhassert.cpp
-    CachedInterfaceDispatch.cpp
+    ${SHARED_RUNTIME_DIR}/CachedInterfaceDispatch.cpp
+    CachedInterfaceDispatchAot.cpp
     Crst.cpp
     DebugHeader.cpp
     MethodTable.cpp
@@ -76,6 +78,7 @@ include_directories(.)
 include_directories(${GC_DIR})
 include_directories(${GC_DIR}/env)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/eventpipe/inc)
+include_directories(${SHARED_RUNTIME_DIR})
 
 if (WIN32)
   set(GC_HEADERS
@@ -208,11 +211,17 @@ list(APPEND RUNTIME_SOURCES_ARCH_ASM
   ${ARCH_SOURCES_DIR}/MiscStubs.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/PInvoke.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/InteropThunksHelpers.${ASM_SUFFIX}
-  ${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX}
+  ${SHARED_RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/UniversalTransition.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX}
 )
 
+if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)
+  list(APPEND RUNTIME_SOURCES_ARCH_ASM
+    ${ARCH_SOURCES_DIR}/CachedInterfaceDispatchAot.${ASM_SUFFIX}
+  )
+endif ()
+
 # Add architecture specific folder for looking up headers.
 convert_to_absolute_path(ARCH_SOURCES_DIR ${ARCH_SOURCES_DIR})
 include_directories(${ARCH_SOURCES_DIR})

src/coreclr/nativeaot/Runtime/CachedInterfaceDispatchAot.cpp

@@ -0,0 +1,54 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "common.h"
+#include "CachedInterfaceDispatchPal.h"
+#include "CachedInterfaceDispatch.h"
+
+// The base memory allocator.
+static AllocHeap * g_pAllocHeap = NULL;
+
+bool InterfaceDispatch_InitializePal()
+{
+    g_pAllocHeap = new (nothrow) AllocHeap();
+    if (g_pAllocHeap == NULL)
+        return false;
+
+    if (!g_pAllocHeap->Init())
+        return false;
+
+    return true;
+}
+
+// Allocate memory aligned at sizeof(void*)*2 boundaries
+void *InterfaceDispatch_AllocDoublePointerAligned(size_t size)
+{
+    return g_pAllocHeap->AllocAligned(size, sizeof(void*) * 2);
+}
+
+// Allocate memory aligned at at least sizeof(void*)
+void *InterfaceDispatch_AllocPointerAligned(size_t size)
+{
+    return g_pAllocHeap->Alloc(size);
+}
+
+FCIMPL4(PCODE, RhpUpdateDispatchCellCache, InterfaceDispatchCell * pCell, PCODE pTargetCode, MethodTable* pInstanceType, DispatchCellInfo *pNewCellInfo)
+{
+    return InterfaceDispatch_UpdateDispatchCellCache(pCell, pTargetCode, pInstanceType, pNewCellInfo);
+}
+FCIMPLEND
+
+FCIMPL2(PCODE, RhpSearchDispatchCellCache, InterfaceDispatchCell * pCell, MethodTable* pInstanceType)
+{
+    return InterfaceDispatch_SearchDispatchCellCache(pCell, pInstanceType);
+}
+FCIMPLEND
+
+// Given a dispatch cell, get the type and slot associated with it. This function MUST be implemented
+// in cooperative native code, as the m_pCache field on the cell is unsafe to access from managed
+// code due to its use of the GC state as a lock, and as lifetime control
+FCIMPL2(void, RhpGetDispatchCellInfo, InterfaceDispatchCell * pCell, DispatchCellInfo* pDispatchCellInfo)
+{
+    *pDispatchCellInfo = pCell->GetDispatchCellInfo();
+}
+FCIMPLEND