How to apply a timeout if counterfactual is not generated for any instance

[Himanshu-1988]

Thanks for creating wonderful library and documentation.

I am using diceml for regression model, i have a system where in loop i am trying to use diceml for every instance as ranges are not generic , but the problem is for some instance since its not able to generate counterfactual it keeps on running.

DO we have a timeout which i can set?

Or any suggestion to generate counterfactual where ranges for feature will depend on instance value.

[Himanshu-1988]

ERROR WHERE IT KEEP ON RUNNING

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/explainer_base.py:161, in ExplainerBase.generate_counterfactuals(self, query_instances, total_CFs, desired_class, desired_range, permitted_range, features_to_vary, stopping_threshold, posthoc_sparsity_param, proximity_weight, sparsity_weight, diversity_weight, categorical_penalty, posthoc_sparsity_algorithm, verbose, **kwargs)
159 for query_instance in tqdm(query_instances_list):
160 self.data_interface.set_continuous_feature_indexes(query_instance)
--> 161 res = self._generate_counterfactuals(
162 query_instance, total_CFs,
163 desired_class=desired_class,
164 desired_range=desired_range,
165 permitted_range=permitted_range,
166 features_to_vary=features_to_vary,
167 stopping_threshold=stopping_threshold,
168 posthoc_sparsity_param=posthoc_sparsity_param,
169 posthoc_sparsity_algorithm=posthoc_sparsity_algorithm,
170 verbose=verbose,
171 **kwargs)
172 cf_examples_arr.append(res)
173 self._check_any_counterfactuals_computed(cf_examples_arr=cf_examples_arr)

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/dice_genetic.py:288, in DiceGenetic._generate_counterfactuals(self, query_instance, total_CFs, initialization, desired_range, desired_class, proximity_weight, sparsity_weight, diversity_weight, categorical_penalty, algorithm, features_to_vary, permitted_range, yloss_type, diversity_loss_type, feature_weights, stopping_threshold, posthoc_sparsity_param, posthoc_sparsity_algorithm, maxiterations, thresh, verbose)
285 if col not in query_instance_df_dummies.columns:
286 query_instance_df_dummies[col] = 0
--> 288 self.do_param_initializations(total_CFs, initialization, desired_range, desired_class, query_instance,
289 query_instance_df_dummies, algorithm, features_to_vary, permitted_range,
290 yloss_type, diversity_loss_type, feature_weights, proximity_weight,
291 sparsity_weight, diversity_weight, categorical_penalty, verbose)
293 query_instance_df = self.find_counterfactuals(query_instance, desired_range, desired_class, features_to_vary,
294 maxiterations, thresh, verbose)
296 return exp.CounterfactualExamples(data_interface=self.data_interface,
297 test_instance_df=query_instance_df,
298 final_cfs_df=self.final_cfs_df,
(...)
302 desired_class=desired_class,
303 model_type=self.model.model_type)

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/dice_genetic.py:194, in DiceGenetic.do_param_initializations(self, total_CFs, initialization, desired_range, desired_class, query_instance, query_instance_df_dummies, algorithm, features_to_vary, permitted_range, yloss_type, diversity_loss_type, feature_weights, proximity_weight, sparsity_weight, diversity_weight, categorical_penalty, verbose)
192 self.feature_range = self.get_valid_feature_range(normalized=False)
193 if len(self.cfs) != total_CFs:
--> 194 self.do_cf_initializations(
195 total_CFs, initialization, algorithm, features_to_vary, desired_range, desired_class,
196 query_instance, query_instance_df_dummies, verbose)
197 else:
198 self.total_CFs = total_CFs

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/dice_genetic.py:180, in DiceGenetic.do_cf_initializations(self, total_CFs, initialization, algorithm, features_to_vary, desired_range, desired_class, query_instance, query_instance_df_dummies, verbose)
178 indices = self.KD_tree.query(query_instance_df_dummies, num_queries)[1][0]
179 KD_tree_output = self.dataset_with_predictions.iloc[indices].copy()
--> 180 self.do_KD_init(features_to_vary, query_instance, KD_tree_output, desired_class, desired_range)
182 if verbose:
183 print("Initialization complete! Generating counterfactuals...")

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/dice_genetic.py:140, in DiceGenetic.do_KD_init(self, features_to_vary, query_instance, cfs, desired_class, desired_range)
137 uniques = np.unique(new_array, axis=0)
139 if len(uniques) != self.population_size:
--> 140 remaining_cfs = self.do_random_init(
141 self.population_size - len(uniques), features_to_vary, query_instance, desired_class, desired_range)
142 self.cfs = np.concatenate([uniques, remaining_cfs])

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/dice_genetic.py:98, in DiceGenetic.do_random_init(self, num_inits, features_to_vary, query_instance, desired_class, desired_range)
96 else:
97 one_init[jx] = query_instance[jx]
---> 98 if self.is_cf_valid(self.predict_fn_scores(one_init)):
99 remaining_cfs[kx] = one_init
100 kx += 1

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/explainer_interfaces/dice_genetic.py:308, in DiceGenetic.predict_fn_scores(self, input_instance)
306 """Returns prediction scores."""
307 input_instance = self.label_decode(input_instance)
--> 308 out = self.model.get_output(input_instance)
309 if self.model.model_type == ModelTypes.Classifier and out.shape[1] == 1:
310 # DL models return only 1 for binary classification
311 out = np.hstack((1-out, out))

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dice_ml/model_interfaces/base_model.py:56, in BaseModel.get_output(self, input_instance, model_score)
54 return self.model.predict_proba(input_instance)
55 else:
---> 56 return self.model.predict(input_instance)
57 else:
58 return self.model.predict(input_instance)

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/sklearn/utils/metaestimators.py:116, in _IffHasAttrDescriptor.get..(*args, **kwargs)
113 attrgetter(self.delegate_names[-1])(obj)
115 # lambda, but not partial, allows help() to work with update_wrapper
--> 116 out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
117 # update the docstring of the returned function
118 update_wrapper(out, self.fn)

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/sklearn/pipeline.py:420, in Pipeline.predict(self, X, **predict_params)
418 for _, name, transform in self._iter(with_final=False):
419 Xt = transform.transform(Xt)
--> 420 return self.steps[-1][-1].predict(Xt, **predict_params)

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/xgboost/sklearn.py:651, in XGBModel.predict(self, data, output_margin, ntree_limit, validate_features, base_margin)
649 if ntree_limit is None:
650 ntree_limit = getattr(self, "best_ntree_limit", 0)
--> 651 return self.get_booster().predict(test_dmatrix,
652 output_margin=output_margin,
653 ntree_limit=ntree_limit,
654 validate_features=validate_features)

File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/xgboost/core.py:1489, in Booster.predict(self, data, output_margin, ntree_limit, pred_leaf, pred_contribs, approx_contribs, pred_interactions, validate_features, training)
1487 length = c_bst_ulong()
1488 preds = ctypes.POINTER(ctypes.c_float)()
-> 1489 _check_call(_LIB.XGBoosterPredict(self.handle, data.handle,
1490 ctypes.c_int(option_mask),
1491 ctypes.c_uint(ntree_limit),
1492 ctypes.c_int(training),
1493 ctypes.byref(length),
1494 ctypes.byref(preds)))
1495 preds = ctypes2numpy(preds, length.value, np.float32)
1496 if pred_leaf:

[mitirmizi]

Hi, I'm facing a similar issue where I want to stop the .generate_counterfactuals() function if it takes too long. I'm assuming it doesn't output anything anyway if it goes over 3-4 minutes. For the time being I'm using Python wrap-timeout-decorator module.
But I'm still get some errors sometimes due to some locking issue, not sure (TBH) to be honest.

[PMK1991]

@mitirmizi Can you show me an example? I am facing similar issues

[mitirmizi]

Sure, here is the directory containing data and notebook required to produce this state: https://github.com/mitirmizi/Prescriptive-Process-Analytics-using-Counterfactuals/tree/master/reproduce_dice_error

I'm using DiCE version: 0.9 and Pandas version: 1.5.0.