diff --git a/src/key.rs b/src/key.rs
index b7747d6..68f34d6 100644
--- a/src/key.rs
+++ b/src/key.rs
@@ -252,7 +252,6 @@ impl RsaPrivateKey {
         d: BigUint,
         mut primes: Vec<BigUint>,
     ) -> Result<RsaPrivateKey> {
-        let mut should_validate = false;
         if primes.len() < 2 {
             if !primes.is_empty() {
                 return Err(Error::NprimesTooSmall);
@@ -262,7 +261,6 @@ impl RsaPrivateKey {
             let (p, q) = recover_primes(&n, &e, &d)?;
             primes.push(p);
             primes.push(q);
-            should_validate = true;
         }
 
         let mut k = RsaPrivateKey {
@@ -272,10 +270,8 @@ impl RsaPrivateKey {
             precomputed: None,
         };
 
-        // Validate the key if we had to recover the primes.
-        if should_validate {
-            k.validate()?;
-        }
+        // Alaways validate the key, to ensure precompute can't fail
+        k.validate()?;
 
         // precompute when possible, ignore error otherwise.
         let _ = k.precompute();
@@ -787,13 +783,13 @@ mod tests {
             .unwrap(),
         ];
 
-        RsaPrivateKey::from_components(
+        let res = RsaPrivateKey::from_components(
             BigUint::from_bytes_be(&n),
             BigUint::from_bytes_be(&e),
             BigUint::from_bytes_be(&d),
             primes.iter().map(|p| BigUint::from_bytes_be(p)).collect(),
-        )
-        .unwrap();
+        );
+        assert_eq!(res, Err(Error::InvalidModulus));
     }
 
     #[test]