|
2 | 2 | # Remi Flamary <[email protected]> |
3 | 3 | # Oleksii Kachaiev <[email protected]> |
4 | 4 | |
| 5 | +# Ambroise Odonnat <[email protected]> |
5 | 6 | # |
6 | 7 | # License: BSD 3-Clause |
7 | 8 |
|
@@ -396,6 +397,7 @@ def _score(self, estimator, X, y, sample_domain=None, **kwargs): |
396 | 397 | ) |
397 | 398 |
|
398 | 399 | has_transform_method = False |
| 400 | + |
399 | 401 | if not isinstance(estimator, Pipeline): |
400 | 402 | # The estimator is a deep model |
401 | 403 | if estimator.module_.layer_name is None: |
@@ -782,3 +784,136 @@ def _score(self, estimator, X, y=None, sample_domain=None, **params): |
782 | 784 | ice_score = ice_diff |
783 | 785 |
|
784 | 786 | return self._sign * ice_score |
| 787 | + |
| 788 | + |
| 789 | +class MaNoScorer(_BaseDomainAwareScorer): |
| 790 | + """ |
| 791 | + MaNo scorer inspired by [37]_, an approach for unsupervised accuracy estimation. |
| 792 | +
|
| 793 | + This scorer used the model's predictions on target data to estimate |
| 794 | + the accuracy of the model. The original implementation in [37]_ is |
| 795 | + tailored to neural networks and consist of three steps: |
| 796 | + 1) Recover target logits (inference step), |
| 797 | + 2) Normalize them as probabilities (e.g., with softmax), |
| 798 | + 3) Aggregate by averaging the p-norms of the target normalized logits. |
| 799 | +
|
| 800 | + To ensure compatibility with any estimator, we adapt the original implementation. |
| 801 | + If the estimator is a neural network, follow 1) --> 2) --> 3) like in [37]_. |
| 802 | + Else, directly use the probabilities predicted by the estimator and then do 3). |
| 803 | +
|
| 804 | + See [37]_ for details. |
| 805 | +
|
| 806 | + Parameters |
| 807 | + ---------- |
| 808 | + p : int, default=4 |
| 809 | + Order for the p-norm normalization. |
| 810 | + It must be non-negative. |
| 811 | + threshold : int, default=5 |
| 812 | + Threshold value to determine which normalization to use. |
| 813 | + If threshold <= 0, softmax normalization is always used. |
| 814 | + See Eq.(6) of [37]_ for more details. |
| 815 | + greater_is_better : bool, default=True |
| 816 | + Whether higher scores are better. |
| 817 | +
|
| 818 | + Returns |
| 819 | + ------- |
| 820 | + score : float in [0, 1] (or [-1, 0] depending on the value of self._sign). |
| 821 | +
|
| 822 | + References |
| 823 | + ---------- |
| 824 | + .. [37] Renchunzi Xie et al. MaNo: Matrix Norm for Unsupervised Accuracy Estimation |
| 825 | + under Distribution Shifts. |
| 826 | + In NeurIPS, 2024. |
| 827 | + """ |
| 828 | + |
| 829 | + def __init__(self, p=4, threshold=5, greater_is_better=True): |
| 830 | + super().__init__() |
| 831 | + self.p = p |
| 832 | + self.threshold = threshold |
| 833 | + self._sign = 1 if greater_is_better else -1 |
| 834 | + self.chosen_normalization = None |
| 835 | + |
| 836 | + if self.p <= 0: |
| 837 | + raise ValueError("The order of the p-norm must be positive") |
| 838 | + |
| 839 | + def _score(self, estimator, X, y, sample_domain=None, **params): |
| 840 | + if not hasattr(estimator, "predict_proba"): |
| 841 | + raise AttributeError( |
| 842 | + "The estimator passed should have a 'predict_proba' method. " |
| 843 | + f"The estimator {estimator!r} does not." |
| 844 | + ) |
| 845 | + |
| 846 | + X, y, sample_domain = check_X_y_domain(X, y, sample_domain, allow_nd=True) |
| 847 | + source_idx = extract_source_indices(sample_domain) |
| 848 | + |
| 849 | + # Check from y values if it is a classification problem |
| 850 | + y_type = _find_y_type(y) |
| 851 | + if y_type != Y_Type.DISCRETE: |
| 852 | + raise ValueError("MaNo scorer only supports classification problems.") |
| 853 | + |
| 854 | + if not isinstance(estimator, Pipeline): |
| 855 | + # The estimator is a deep model |
| 856 | + if estimator.module_.layer_name is None: |
| 857 | + raise ValueError("The layer_name of the estimator is not set.") |
| 858 | + |
| 859 | + # 1) Recover logits on target |
| 860 | + logits = estimator.infer(X[~source_idx], **params).cpu().detach().numpy() |
| 861 | + |
| 862 | + # 2) Normalize logits to obtain probabilities |
| 863 | + criterion = self._get_criterion(logits) |
| 864 | + proba = self._softrun( |
| 865 | + logits=logits, |
| 866 | + criterion=criterion, |
| 867 | + threshold=self.threshold, |
| 868 | + ) |
| 869 | + else: |
| 870 | + # Directly recover predicted probabilities |
| 871 | + proba = estimator.predict_proba( |
| 872 | + X[~source_idx], sample_domain=sample_domain[~source_idx], **params |
| 873 | + ) |
| 874 | + |
| 875 | + # 3) Aggregate following Eq.(2) of [37]_. |
| 876 | + score = np.mean(proba**self.p) ** (1 / self.p) |
| 877 | + |
| 878 | + return self._sign * score |
| 879 | + |
| 880 | + def _get_criterion(self, logits): |
| 881 | + """ |
| 882 | + Compute criterion to select the proper normalization. |
| 883 | + See Eq.(6) of [1]_ for more details. |
| 884 | + """ |
| 885 | + proba = self._stable_softmax(logits) |
| 886 | + proba = np.log(proba) |
| 887 | + divergence = -np.mean(proba) |
| 888 | + |
| 889 | + return divergence |
| 890 | + |
| 891 | + def _softrun(self, logits, criterion, threshold): |
| 892 | + """Normalize the logits following Eq.(6) of [37]_.""" |
| 893 | + if criterion > threshold: |
| 894 | + # Apply softmax normalization |
| 895 | + outputs = self._stable_softmax(logits) |
| 896 | + self.chosen_normalization = "softmax" |
| 897 | + else: |
| 898 | + # Apply Taylor approximation |
| 899 | + outputs = self._taylor_softmax(logits) |
| 900 | + self.chosen_normalization = "taylor" |
| 901 | + |
| 902 | + return outputs |
| 903 | + |
| 904 | + @staticmethod |
| 905 | + def _stable_softmax(logits): |
| 906 | + """Compute softmax function.""" |
| 907 | + logits -= np.max(logits, axis=1, keepdims=True) |
| 908 | + exp_logits = np.exp(logits) |
| 909 | + exp_logits /= np.sum(exp_logits, axis=1, keepdims=True) |
| 910 | + return exp_logits |
| 911 | + |
| 912 | + @staticmethod |
| 913 | + def _taylor_softmax(logits): |
| 914 | + """Compute Taylor approximation of order 2 of softmax.""" |
| 915 | + tay_logits = 1 + logits + logits**2 / 2 |
| 916 | + tay_logits -= np.min(tay_logits, axis=1, keepdims=True) |
| 917 | + tay_logits /= np.sum(tay_logits, axis=1, keepdims=True) |
| 918 | + |
| 919 | + return tay_logits |
0 commit comments