diff --git a/Components/Hlms/Common/include/OgreHlmsTextureBaseClass.inl b/Components/Hlms/Common/include/OgreHlmsTextureBaseClass.inl
index aaaa64aac20..a17195bcd6c 100644
--- a/Components/Hlms/Common/include/OgreHlmsTextureBaseClass.inl
+++ b/Components/Hlms/Common/include/OgreHlmsTextureBaseClass.inl
@@ -285,9 +285,15 @@ namespace Ogre
         else if( !mSamplersDescSet || *mSamplersDescSet != baseSampler )
         {
             if( mSamplersDescSet )
+            {
                 hlmsManager->destroyDescriptorSetSampler( mSamplersDescSet );
-            mSamplersDescSet = hlmsManager->getDescriptorSetSampler( baseSampler );
-            needsRecalculateHash = true;
+                mSamplersDescSet = 0;
+            }
+            if( !baseSampler.mSamplers.empty() )
+            {
+                mSamplersDescSet = hlmsManager->getDescriptorSetSampler( baseSampler );
+                needsRecalculateHash = true;
+            }
         }
 
         return needsRecalculateHash;