diff --git a/src/pluggy/_manager.py b/src/pluggy/_manager.py
index 1b994f25..2de3327c 100644
--- a/src/pluggy/_manager.py
+++ b/src/pluggy/_manager.py
@@ -516,7 +516,10 @@ def subset_hook_caller(
         method which manages calls to all registered plugins except the ones
         from remove_plugins."""
         orig: HookCaller = getattr(self.hook, name)
-        plugins_to_remove = {plug for plug in remove_plugins if hasattr(plug, name)}
+        registered_plugins = {impl.plugin for impl in orig._hookimpls}
+        plugins_to_remove = {
+            plug for plug in remove_plugins if plug in registered_plugins
+        }
         if plugins_to_remove:
             return _SubsetHookCaller(orig, plugins_to_remove)
         return orig