diff --git a/wavenet_model.py b/wavenet_model.py
index 573dde990..0d033326e 100644
--- a/wavenet_model.py
+++ b/wavenet_model.py
@@ -607,7 +607,7 @@ def conditional_network(self, conditioning, file_encoding):
         for l in range(len(self.conditioning_layers)):
             if l != 0 and l != len(self.conditioning_layers) - 1:
                 cross_encoding = self.file_conditioning_cross_layers[l-1](file_encoding)
-                conditioning = F.relu(conditioning + cross_encoding, negative_slope=0.1, inplace=True)
+                conditioning = F.leaky_relu(conditioning + cross_encoding, negative_slope=0.1, inplace=True)
             conditioning = self.conditioning_layers[l](conditioning)
 
         return conditioning