From 1146430c9c4f9fa56e83958cdbd70f771b98edb7 Mon Sep 17 00:00:00 2001
From: Lorenzo-Perini <lorenzo.perini@kuleuven.be>
Date: Thu, 5 Sep 2024 15:28:53 +0200
Subject: [PATCH 1/3] Including Reject Option

---
 README.rst                        |   3 +
 notebooks/Benchmark.ipynb         |  23 ++++-
 pyod/models/base.py               | 148 ++++++++++++++++++++++++++++++
 pyod/test/test_abod.py            |  12 +++
 pyod/test/test_ae1svm.py          |  12 +++
 pyod/test/test_alad.py            |  12 +++
 pyod/test/test_auto_encoder.py    |  10 ++
 pyod/test/test_cblof.py           |  12 +++
 pyod/test/test_cd.py              |  12 +++
 pyod/test/test_cof.py             |  12 +++
 pyod/test/test_copod.py           |  12 +++
 pyod/test/test_deepsvdd.py        |  12 +++
 pyod/test/test_devnet.py          |  12 +++
 pyod/test/test_dif.py             |  12 +++
 pyod/test/test_ecod.py            |  12 +++
 pyod/test/test_feature_bagging.py |  12 +++
 pyod/test/test_gmm.py             |  12 +++
 pyod/test/test_hbos.py            |  12 +++
 pyod/test/test_iforest.py         |  12 +++
 pyod/test/test_inne.py            |  12 +++
 pyod/test/test_kde.py             |  12 +++
 pyod/test/test_knn.py             |  12 +++
 pyod/test/test_kpca.py            |  12 +++
 pyod/test/test_lmdd.py            |  12 +++
 pyod/test/test_loci.py            |  12 +++
 pyod/test/test_loda.py            |  12 +++
 pyod/test/test_lof.py             |  12 +++
 pyod/test/test_lscp.py            |  12 +++
 pyod/test/test_lunar.py           |  12 +++
 pyod/test/test_mad.py             |  12 +++
 pyod/test/test_mcd.py             |  12 +++
 pyod/test/test_mo_gaal.py         |  12 +++
 pyod/test/test_ocsvm.py           |  12 +++
 pyod/test/test_pca.py             |  12 +++
 pyod/test/test_qmcd.py            |  12 +++
 pyod/test/test_rgraph.py          |  12 +++
 pyod/test/test_rod.py             |  12 +++
 pyod/test/test_sampling.py        |  12 +++
 pyod/test/test_so_gaal.py         |  12 +++
 pyod/test/test_so_gaal_new.py     |  10 ++
 pyod/test/test_sod.py             |  12 +++
 pyod/test/test_sos.py             |  12 +++
 pyod/test/test_suod.py            |  12 +++
 pyod/test/test_vae.py             |  10 ++
 pyod/test/test_xgbod.py           |  12 +++
 45 files changed, 671 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 8dbc3c466..7c1e14d91 100644
--- a/README.rst
+++ b/README.rst
@@ -211,6 +211,7 @@ The full API Reference is available at `PyOD Documentation <https://pyod.readthe
 * **predict(X)**: Determine whether a sample is an outlier or not as binary labels using the fitted detector.
 * **predict_proba(X)**: Estimate the probability of a sample being an outlier using the fitted detector.
 * **predict_confidence(X)**: Assess the model's confidence on a per-sample basis (applicable in predict and predict_proba) [#Perini2020Quantifying]_.
+* **predict_with_rejection(X)**\ : Allow the detector to reject (i.e., abstain from making) highly uncertain predictions (output = -2) [#Perini2023Rejection]_.
 
 **Key Attributes of a fitted model**:
 
@@ -567,6 +568,8 @@ Reference
 
 .. [#Perini2020Quantifying] Perini, L., Vercruyssen, V., Davis, J. Quantifying the confidence of anomaly detectors in their example-wise predictions. In *Joint European Conference on Machine Learning and Knowledge Discovery in Databases (ECML-PKDD)*, 2020.
 
+.. [#Perini2023Rejection] Perini, L., Davis, J. Unsupervised anomaly detection with rejection. In *Proceedings of the Thirty-Seven Conference on Neural Information Processing Systems (NeurIPS)*, 2023.
+
 .. [#Ramaswamy2000Efficient] Ramaswamy, S., Rastogi, R. and Shim, K., 2000, May. Efficient algorithms for mining outliers from large data sets. *ACM Sigmod Record*\ , 29(2), pp. 427-438.
 
 .. [#Rousseeuw1999A] Rousseeuw, P.J. and Driessen, K.V., 1999. A fast algorithm for the minimum covariance determinant estimator. *Technometrics*\ , 41(3), pp.212-223.
diff --git a/notebooks/Benchmark.ipynb b/notebooks/Benchmark.ipynb
index e12a5778c..833704cf1 100644
--- a/notebooks/Benchmark.ipynb
+++ b/notebooks/Benchmark.ipynb
@@ -1603,6 +1603,27 @@
    },
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -1621,7 +1642,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.2"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,
diff --git a/pyod/models/base.py b/pyod/models/base.py
index fa7c2c449..4e21abf0a 100644
--- a/pyod/models/base.py
+++ b/pyod/models/base.py
@@ -19,6 +19,7 @@
 from sklearn.utils import deprecated
 from sklearn.utils.multiclass import check_classification_targets
 from sklearn.utils.validation import check_is_fitted
+from scipy.optimize import root_scalar
 
 from .sklearn_base import _pprint
 from ..utils.utility import precision_n_scores
@@ -293,7 +294,154 @@ def predict_confidence(self, X):
         np.place(confidence, prediction == 0, 1 - confidence[prediction == 0])
 
         return confidence
+    
+    def predict_with_rejection(self, X, T = 32, return_stats = False, 
+                               delta = 0.1, c_fp = 1, c_fn = 1, c_r = -1):
+        """Predict if a particular sample is an outlier or not, 
+           allowing the detector to reject (i.e., output = -2) 
+           low confidence predictions.
 
+        Parameters
+        ----------
+        X :              numpy array of shape (n_samples, n_features)
+                         The input samples.
+
+        T :              int, optional(default=32)
+                         It allows to set the rejection threshold to 1-2exp(-T).
+                         The higher the value of T, the more rejections are made.
+            
+        return_stats:    bool, optional (default = False)
+                         If true, it returns also three additional float values:
+                         the estimated rejection rate, the upper bound rejection rate,
+                         and the upper bound of the cost.
+                       
+        delta:           float, optional (default = 0.1)
+                         The upper bound rejection rate holds with probability 1-delta.
+                       
+        c_fp, c_fn, c_r: floats (positive), optional (default = [1,1, contamination])
+                         costs for false positive predictions (c_fp), false negative
+                         predictions (c_fn) and rejections (c_r).
+                
+        Returns
+        -------
+        outlier_labels :           numpy array of shape (n_samples,)
+                                   For each observation, it tells whether it should be considered
+                                   as an outlier according to the fitted model. 0 stands for inliers,
+                                   1 for outliers and -2 for rejection.
+                                   
+        expected_rejection_rate:   float, if return_stats is True;
+        upperbound_rejection_rate: float, if return_stats is True;
+        upperbound_cost:           float, if return_stats is True;
+
+        """
+        check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_'])
+        if c_r <0:
+            warnings.warn("The cost of rejection must be positive. It has been set to the contamination rate.")
+            c_r = self.contamination
+
+        if delta<=0 or delta>=1:
+            warnings.warn("delta must belong to (0,1). It's value has been set to 0.1")
+            delta = 0.1
+
+        self.rejection_threshold_ = 1- 2*np.exp(-T)
+        prediction = self.predict(X)
+        confidence = self.predict_confidence(X)
+        np.place(confidence, prediction == 0, 1 - confidence[prediction == 0])
+        confidence = 2*abs(confidence-.5)
+        prediction[np.where(confidence<=self.rejection_threshold_)[0]] = -2
+
+        if return_stats:
+            expected_rejrate, ub_rejrate, ub_cost = self.compute_rejection_stats(T = T, delta = delta, 
+                                                                                 c_fp=c_fp, c_fn =c_fn, c_r = c_r)
+            return prediction, [expected_rejrate, ub_rejrate, ub_cost]
+            
+        return prediction
+
+    
+    def compute_rejection_stats(self, T = 32, delta = 0.1, c_fp = 1, c_fn = 1, c_r = -1, verbose = False):
+        """Add reject option into the unsupervised detector. 
+           This comes with guarantees: an estimate of the expected
+           rejection rate (return_rejectrate=True), an upper
+           bound of the rejection rate (return_ub_rejectrate= True),
+           and an upper bound on the cost (return_ub_cost=True).
+           
+        Parameters
+        ----------
+        T:               int, optional(default=32)
+                         It allows to set the rejection threshold to 1-2exp(-T).
+                         The higher the value of T, the more rejections are made.
+            
+        delta:           float, optional (default = 0.1)
+                         The upper bound rejection rate holds with probability 1-delta.
+                       
+        c_fp, c_fn, c_r: floats (positive), optional (default = [1,1, contamination])
+                         costs for false positive predictions (c_fp), false negative
+                         predictions (c_fn) and rejections (c_r).
+                         
+        verbose:         bool, optional (default = False)
+                         If true, it prints the expected rejection rate, the upper bound rejection rate,
+                         and the upper bound of the cost.
+
+        Returns
+        -------
+        expected_rejection_rate:   float, the expected rejection rate;
+        upperbound_rejection_rate: float, the upper bound for the rejection rate
+                                   satisfied with probability 1-delta;
+        upperbound_cost:           float, the upper bound for the cost;
+        """
+        
+        check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_'])
+        
+        if c_r <0:
+            c_r = self.contamination
+            
+        if delta<=0 or delta>=1:
+            delta = 0.1
+        
+        # Computing the expected rejection rate
+        n = len(self.decision_scores_)
+        n_gamma_minus1 = int(n * self.contamination) -1
+        argsmin = (n_gamma_minus1, n, 1-np.exp(-T))
+        argsmax = (n_gamma_minus1, n, np.exp(-T))
+        q1 = root_scalar(lambda p, k, n, C: binom.cdf(k, n, p) - C, bracket=[0, 1], method='brentq', args=argsmin).root
+        q2 = root_scalar(lambda p, k, n, C: binom.cdf(k, n, p) - C, bracket=[0, 1], method='brentq', args=argsmax).root
+        expected_reject_rate = q2-q1
+
+        # Computing the upper bound for the rejection rate
+        right_mar = (-self.contamination * (n + 2) + n + 1) / n + (T * (n + 2)) / (np.sqrt(2 * n**3 * T))
+        right_mar = min(1, right_mar)
+        left_mar = (
+            (2 + n * (1 - self.contamination) * (n + 1)) / n**2
+            - np.sqrt(
+                0.5 * n**5 * (
+                    2 * n * (
+                        -3 * self.contamination**2
+                        - 2 * n * (1 - self.contamination)**2
+                        + 4 * self.contamination - 3
+                    )
+                    + T * (n + 2)**2 - 8
+                )
+            ) / n**4
+        )
+        left_mar = max(0, left_mar)
+        add_term = 2 * np.sqrt(np.log(2 / delta) / (2 * n))
+        upperbound_rejectrate = right_mar - left_mar + add_term
+
+        # Computing the upper bound for the cost function
+        n_gamma_minus1 = int(n * self.contamination) -1
+        argsmin = (n_gamma_minus1, n, 1-np.exp(-T))
+        argsmax = (n_gamma_minus1, n, np.exp(-T))
+        q1 = root_scalar(lambda p, k, n, C: binom.cdf(k, n, p) - C, bracket=[0, 1], method='brentq', args=argsmin).root
+        q2 = root_scalar(lambda p, k, n, C: binom.cdf(k, n, p) - C, bracket=[0, 1], method='brentq', args=argsmax).root
+        upperbound_cost = np.min([self.contamination,q1])*c_fp + np.min([1-q2,self.contamination])*c_fn + (q2-q1)*c_r
+
+        if verbose:
+            print("Expected rejection rate: ", np.round(expected_reject_rate, 4), '%')
+            print("Upper bound rejection rate: ", np.round(upperbound_rejectrate, 4), '%')
+            print("Upper bound cost: ", np.round(upperbound_cost, 4))
+
+        return expected_reject_rate, upperbound_rejectrate, upperbound_cost
+    
     def _predict_rank(self, X, normalized=False):
         """Predict the outlyingness rank of a sample by a fitted model. The
         method is for outlier detector score combination.
diff --git a/pyod/test/test_abod.py b/pyod/test/test_abod.py
index b098ba35c..466bd2eb8 100644
--- a/pyod/test/test_abod.py
+++ b/pyod/test/test_abod.py
@@ -103,6 +103,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_ae1svm.py b/pyod/test/test_ae1svm.py
index d8cecd131..d6beeb60f 100644
--- a/pyod/test/test_ae1svm.py
+++ b/pyod/test/test_ae1svm.py
@@ -102,6 +102,18 @@ def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict_score(self):
         self.clf.fit_predict_score(self.X_test, self.y_test)
         self.clf.fit_predict_score(self.X_test, self.y_test,
diff --git a/pyod/test/test_alad.py b/pyod/test/test_alad.py
index b7969e773..e2c9ee3ed 100644
--- a/pyod/test/test_alad.py
+++ b/pyod/test/test_alad.py
@@ -119,6 +119,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_auto_encoder.py b/pyod/test/test_auto_encoder.py
index 43a2265a9..de251b1ea 100644
--- a/pyod/test/test_auto_encoder.py
+++ b/pyod/test/test_auto_encoder.py
@@ -99,6 +99,16 @@ def test_prediction_proba_linear_confidence(self):
         self.assertEqual(confidence.shape, self.y_test.shape)
         self.assertInRange(confidence, 0, 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        self.assertEqual(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        self.assertInRange(expected_rejrate, 0, 1)
+        self.assertInRange(ub_rejrate, 0, 1)
+        self.assertGreaterEqual(ub_cost, 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         self.assertEqual(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_cblof.py b/pyod/test/test_cblof.py
index ece8511a7..b4c09e326 100644
--- a/pyod/test/test_cblof.py
+++ b/pyod/test/test_cblof.py
@@ -115,6 +115,18 @@ def test_prediction_proba_linear_confidence(self):
         assert_equal(confidence.shape, self.y_test.shape)
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
+        
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
 
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
diff --git a/pyod/test/test_cd.py b/pyod/test/test_cd.py
index a8b8325d5..c8f37180e 100644
--- a/pyod/test/test_cd.py
+++ b/pyod/test/test_cd.py
@@ -89,6 +89,18 @@ def test_prediction_labels_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_prediction_proba_linear_confidence(self):
         pred_proba, confidence = self.clf.predict_proba(self.X_test,
                                                         method='linear',
diff --git a/pyod/test/test_cof.py b/pyod/test/test_cof.py
index e4e612c43..1fa9a22f9 100644
--- a/pyod/test/test_cof.py
+++ b/pyod/test/test_cof.py
@@ -99,6 +99,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_copod.py b/pyod/test/test_copod.py
index f1a45a0eb..db535faea 100644
--- a/pyod/test/test_copod.py
+++ b/pyod/test/test_copod.py
@@ -97,6 +97,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_deepsvdd.py b/pyod/test/test_deepsvdd.py
index 1c216cf6a..e89896e4c 100644
--- a/pyod/test/test_deepsvdd.py
+++ b/pyod/test/test_deepsvdd.py
@@ -111,6 +111,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_devnet.py b/pyod/test/test_devnet.py
index 47a06b0b3..aa8280ebe 100644
--- a/pyod/test/test_devnet.py
+++ b/pyod/test/test_devnet.py
@@ -105,6 +105,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train, self.y_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_dif.py b/pyod/test/test_dif.py
index 5176436f8..901234c23 100644
--- a/pyod/test/test_dif.py
+++ b/pyod/test/test_dif.py
@@ -102,6 +102,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_ecod.py b/pyod/test/test_ecod.py
index 11111a219..1d5e90441 100644
--- a/pyod/test/test_ecod.py
+++ b/pyod/test/test_ecod.py
@@ -97,6 +97,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_feature_bagging.py b/pyod/test/test_feature_bagging.py
index 089bc6899..a22683f4e 100644
--- a/pyod/test/test_feature_bagging.py
+++ b/pyod/test/test_feature_bagging.py
@@ -105,6 +105,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_gmm.py b/pyod/test/test_gmm.py
index 2d12e4a26..836c5565b 100644
--- a/pyod/test/test_gmm.py
+++ b/pyod/test/test_gmm.py
@@ -97,6 +97,18 @@ def test_prediction_proba_parameter(self):
         with assert_raises(ValueError):
             self.clf.predict_proba(self.X_test, method="something")
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_predict_rank(self):
         pred_socres = self.clf.decision_function(self.X_test)
         pred_ranks = self.clf._predict_rank(self.X_test)
diff --git a/pyod/test/test_hbos.py b/pyod/test/test_hbos.py
index 25bcb3626..55c1648ad 100644
--- a/pyod/test/test_hbos.py
+++ b/pyod/test/test_hbos.py
@@ -86,6 +86,18 @@ def test_prediction_proba_parameter(self):
         with assert_raises(ValueError):
             self.clf.predict_proba(self.X_test, method='something')
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_iforest.py b/pyod/test/test_iforest.py
index 80dca9b25..8ad4308d3 100644
--- a/pyod/test/test_iforest.py
+++ b/pyod/test/test_iforest.py
@@ -113,6 +113,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_inne.py b/pyod/test/test_inne.py
index bb5428c06..c5c9d6845 100644
--- a/pyod/test/test_inne.py
+++ b/pyod/test/test_inne.py
@@ -103,6 +103,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_kde.py b/pyod/test/test_kde.py
index 1c860c284..dccf08ea1 100644
--- a/pyod/test/test_kde.py
+++ b/pyod/test/test_kde.py
@@ -101,6 +101,18 @@ def test_prediction_proba_linear_confidence(self):
         assert confidence.min() >= 0
         assert confidence.max() <= 1
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_knn.py b/pyod/test/test_knn.py
index a7c2c92d5..2d028d6c2 100644
--- a/pyod/test/test_knn.py
+++ b/pyod/test/test_knn.py
@@ -281,6 +281,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_kpca.py b/pyod/test/test_kpca.py
index 2c309c9b8..28898b200 100644
--- a/pyod/test/test_kpca.py
+++ b/pyod/test/test_kpca.py
@@ -98,6 +98,18 @@ def test_prediction_proba_linear_confidence(self):
         assert confidence.min() >= 0
         assert confidence.max() <= 1
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_lmdd.py b/pyod/test/test_lmdd.py
index 3cf5336a4..23acbd159 100644
--- a/pyod/test/test_lmdd.py
+++ b/pyod/test/test_lmdd.py
@@ -104,6 +104,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_loci.py b/pyod/test/test_loci.py
index d96bad611..3ee58b392 100644
--- a/pyod/test/test_loci.py
+++ b/pyod/test/test_loci.py
@@ -98,6 +98,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_loda.py b/pyod/test/test_loda.py
index 17cb200e0..1f2800001 100644
--- a/pyod/test/test_loda.py
+++ b/pyod/test/test_loda.py
@@ -100,6 +100,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_lof.py b/pyod/test/test_lof.py
index 0f3896d5d..42875ead5 100644
--- a/pyod/test/test_lof.py
+++ b/pyod/test/test_lof.py
@@ -103,6 +103,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_lscp.py b/pyod/test/test_lscp.py
index e37e7f03d..616e58056 100644
--- a/pyod/test/test_lscp.py
+++ b/pyod/test/test_lscp.py
@@ -125,6 +125,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_lunar.py b/pyod/test/test_lunar.py
index cb5f6eed7..74c0ecfbc 100644
--- a/pyod/test/test_lunar.py
+++ b/pyod/test/test_lunar.py
@@ -100,6 +100,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_mad.py b/pyod/test/test_mad.py
index 5d6bb3967..604adcde7 100644
--- a/pyod/test/test_mad.py
+++ b/pyod/test/test_mad.py
@@ -113,6 +113,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_mcd.py b/pyod/test/test_mcd.py
index c8df2dba2..0ad59973f 100644
--- a/pyod/test/test_mcd.py
+++ b/pyod/test/test_mcd.py
@@ -115,6 +115,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_mo_gaal.py b/pyod/test/test_mo_gaal.py
index 00e5f9ac1..c7ead2f49 100644
--- a/pyod/test/test_mo_gaal.py
+++ b/pyod/test/test_mo_gaal.py
@@ -109,6 +109,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_ocsvm.py b/pyod/test/test_ocsvm.py
index 52ee52b98..89248fecf 100644
--- a/pyod/test/test_ocsvm.py
+++ b/pyod/test/test_ocsvm.py
@@ -113,6 +113,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_pca.py b/pyod/test/test_pca.py
index 48621751b..63e9eb8cd 100644
--- a/pyod/test/test_pca.py
+++ b/pyod/test/test_pca.py
@@ -106,6 +106,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_qmcd.py b/pyod/test/test_qmcd.py
index 8c4e62c50..51a48a242 100644
--- a/pyod/test/test_qmcd.py
+++ b/pyod/test/test_qmcd.py
@@ -104,6 +104,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_rgraph.py b/pyod/test/test_rgraph.py
index abd8dd16b..f0b251d49 100644
--- a/pyod/test/test_rgraph.py
+++ b/pyod/test/test_rgraph.py
@@ -111,6 +111,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_rod.py b/pyod/test/test_rod.py
index a684fd6ed..451ea7578 100644
--- a/pyod/test/test_rod.py
+++ b/pyod/test/test_rod.py
@@ -87,6 +87,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_sampling.py b/pyod/test/test_sampling.py
index d555af012..19b68931c 100644
--- a/pyod/test/test_sampling.py
+++ b/pyod/test/test_sampling.py
@@ -104,6 +104,18 @@ def test_prediction_proba_linear_confidence(self):
         assert confidence.min() >= 0
         assert confidence.max() <= 1
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_so_gaal.py b/pyod/test/test_so_gaal.py
index 115bf6545..ff269e0ee 100644
--- a/pyod/test/test_so_gaal.py
+++ b/pyod/test/test_so_gaal.py
@@ -107,6 +107,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_so_gaal_new.py b/pyod/test/test_so_gaal_new.py
index 94277c132..89651ce51 100644
--- a/pyod/test/test_so_gaal_new.py
+++ b/pyod/test/test_so_gaal_new.py
@@ -100,6 +100,16 @@ def test_prediction_proba_linear_confidence(self):
         self.assertEqual(confidence.shape, self.y_test.shape)
         self.assertInRange(confidence, 0, 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        self.assertEqual(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        self.assertInRange(expected_rejrate, 0, 1)
+        self.assertInRange(ub_rejrate, 0, 1)
+        self.assertGreaterEqual(ub_cost, 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         self.assertEqual(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_sod.py b/pyod/test/test_sod.py
index b12dfdf7f..a081836dd 100644
--- a/pyod/test/test_sod.py
+++ b/pyod/test/test_sod.py
@@ -125,6 +125,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_sos.py b/pyod/test/test_sos.py
index c47adbc6a..125125d73 100644
--- a/pyod/test/test_sos.py
+++ b/pyod/test/test_sos.py
@@ -98,6 +98,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_suod.py b/pyod/test/test_suod.py
index 293bbb9e6..6561f1619 100644
--- a/pyod/test/test_suod.py
+++ b/pyod/test/test_suod.py
@@ -124,6 +124,18 @@ def test_prediction_proba_linear_confidence(self):
         assert (confidence.min() >= 0)
         assert (confidence.max() <= 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        assert (expected_rejrate >= 0)
+        assert (expected_rejrate <= 1)
+        assert (ub_rejrate >= 0)
+        assert (ub_rejrate <= 1)
+        assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         assert_equal(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_vae.py b/pyod/test/test_vae.py
index da1f7e3bf..3188ffdd8 100644
--- a/pyod/test/test_vae.py
+++ b/pyod/test/test_vae.py
@@ -98,6 +98,16 @@ def test_prediction_proba_linear_confidence(self):
         self.assertEqual(confidence.shape, self.y_test.shape)
         self.assertInRange(confidence, 0, 1)
 
+    def test_prediction_with_rejection(self):
+        pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+        self.assertEqual(pred_labels.shape, self.y_test.shape)
+        
+    def test_prediction_with_rejection_stats(self):
+        _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+        self.assertInRange(expected_rejrate, 0, 1)
+        self.assertInRange(ub_rejrate, 0, 1)
+        self.assertGreaterEqual(ub_cost, 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train)
         self.assertEqual(pred_labels.shape, self.y_train.shape)
diff --git a/pyod/test/test_xgbod.py b/pyod/test/test_xgbod.py
index 74263ed30..133eceb6b 100644
--- a/pyod/test/test_xgbod.py
+++ b/pyod/test/test_xgbod.py
@@ -124,6 +124,18 @@ def test_prediction_proba(self):
     #     assert (confidence.min() >= 0)
     #     assert (confidence.max() <= 1)
 
+    #def test_prediction_with_rejection(self):
+    #    pred_labels = self.clf.predict_with_rejection(self.X_test, return_stats = False)
+    #    assert_equal(pred_labels.shape, self.y_test.shape)
+        
+    #def test_prediction_with_rejection_stats(self):
+    #    _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
+    #    assert (expected_rejrate >= 0)
+    #    assert (expected_rejrate <= 1)
+    #    assert (ub_rejrate >= 0)
+    #    assert (ub_rejrate <= 1)
+    #    assert (ub_cost >= 0)
+
     def test_fit_predict(self):
         pred_labels = self.clf.fit_predict(self.X_train, self.y_train)
         assert_equal(pred_labels.shape, self.y_train.shape)

From 5edee89ecfccbc49324a258c5f0adcb65e776635 Mon Sep 17 00:00:00 2001
From: Lorenzo-Perini <lorenzo.perini@kuleuven.be>
Date: Thu, 5 Sep 2024 15:33:19 +0200
Subject: [PATCH 2/3] update 05.09

---
 notebooks/Benchmark.ipynb | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/notebooks/Benchmark.ipynb b/notebooks/Benchmark.ipynb
index 833704cf1..e12a5778c 100644
--- a/notebooks/Benchmark.ipynb
+++ b/notebooks/Benchmark.ipynb
@@ -1603,27 +1603,6 @@
    },
    "outputs": [],
    "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -1642,7 +1621,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.14"
+   "version": "3.6.2"
   }
  },
  "nbformat": 4,

From 3842c2c224776b85306cb95ba9b9b4149cc6bbe6 Mon Sep 17 00:00:00 2001
From: Lorenzo-Perini <lorenzo.perini@kuleuven.be>
Date: Thu, 5 Sep 2024 16:18:26 +0200
Subject: [PATCH 3/3] adding tests

---
 pyod/test/test_auto_encoder.py | 6 ++++--
 pyod/test/test_so_gaal_new.py  | 6 ++++--
 pyod/test/test_vae.py          | 6 ++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/pyod/test/test_auto_encoder.py b/pyod/test/test_auto_encoder.py
index de251b1ea..a68c6de1e 100644
--- a/pyod/test/test_auto_encoder.py
+++ b/pyod/test/test_auto_encoder.py
@@ -105,8 +105,10 @@ def test_prediction_with_rejection(self):
         
     def test_prediction_with_rejection_stats(self):
         _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
-        self.assertInRange(expected_rejrate, 0, 1)
-        self.assertInRange(ub_rejrate, 0, 1)
+        self.assertGreaterEqual(expected_rejrate, 0)
+        self.assertLessEqual(expected_rejrate, 1)
+        self.assertGreaterEqual(ub_rejrate, 0)
+        self.assertLessEqual(ub_rejrate, 1)
         self.assertGreaterEqual(ub_cost, 0)
 
     def test_fit_predict(self):
diff --git a/pyod/test/test_so_gaal_new.py b/pyod/test/test_so_gaal_new.py
index 89651ce51..66fad35ca 100644
--- a/pyod/test/test_so_gaal_new.py
+++ b/pyod/test/test_so_gaal_new.py
@@ -106,8 +106,10 @@ def test_prediction_with_rejection(self):
         
     def test_prediction_with_rejection_stats(self):
         _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
-        self.assertInRange(expected_rejrate, 0, 1)
-        self.assertInRange(ub_rejrate, 0, 1)
+        self.assertGreaterEqual(expected_rejrate, 0)
+        self.assertLessEqual(expected_rejrate, 1)
+        self.assertGreaterEqual(ub_rejrate, 0)
+        self.assertLessEqual(ub_rejrate, 1)
         self.assertGreaterEqual(ub_cost, 0)
 
     def test_fit_predict(self):
diff --git a/pyod/test/test_vae.py b/pyod/test/test_vae.py
index 3188ffdd8..3d5de81ed 100644
--- a/pyod/test/test_vae.py
+++ b/pyod/test/test_vae.py
@@ -104,8 +104,10 @@ def test_prediction_with_rejection(self):
         
     def test_prediction_with_rejection_stats(self):
         _, [expected_rejrate, ub_rejrate, ub_cost] = self.clf.predict_with_rejection(self.X_test, return_stats = True)
-        self.assertInRange(expected_rejrate, 0, 1)
-        self.assertInRange(ub_rejrate, 0, 1)
+        self.assertGreaterEqual(expected_rejrate, 0)
+        self.assertLessEqual(expected_rejrate, 1)
+        self.assertGreaterEqual(ub_rejrate, 0)
+        self.assertLessEqual(ub_rejrate, 1)
         self.assertGreaterEqual(ub_cost, 0)
 
     def test_fit_predict(self):