diff --git a/jose/backends/rsa_backend.py b/jose/backends/rsa_backend.py
index 8139d69..4da5b03 100644
--- a/jose/backends/rsa_backend.py
+++ b/jose/backends/rsa_backend.py
@@ -207,8 +207,7 @@ def verify(self, msg, sig):
         if not self.is_public():
             warnings.warn("Attempting to verify a message with a private key. " "This is not recommended.")
         try:
-            pyrsa.verify(msg, sig, self._prepared_key)
-            return True
+            return pyrsa.verify(msg, sig, self._prepared_key) == self.hash_alg
         except pyrsa.pkcs1.VerificationError:
             return False
 
diff --git a/tests/algorithms/test_RSA.py b/tests/algorithms/test_RSA.py
index b181993..15c16b2 100644
--- a/tests/algorithms/test_RSA.py
+++ b/tests/algorithms/test_RSA.py
@@ -217,6 +217,14 @@ def test_python_rsa_legacy_private_key_pkcs8_to_pkcs1_invalid(self):
 
         excinfo.match("Invalid private key encoding")
 
+    @pytest.mark.parametrize("hash_method", ["MD5", "SHA-1", "SHA-512"])
+    def test_python_rsa_verify_rejects_other_hash(self, hash_method):
+        key = PurePythonRSAKey(private_key_4096_pkcs1, ALGORITHMS.RS256)
+        msg = b"test"
+        sig = rsa_backend.pyrsa.sign(msg, key._prepared_key, hash_method)
+
+        assert not key.public_key().verify(msg, sig)
+
 
 @pytest.mark.cryptography
 @pytest.mark.skipif(