Skip to content

OOD methods

OODBaseDetector

Bases: ABC

Base Class for methods that assign a score to unseen samples.

Parameters:

Name Type Description Default
use_react bool

if true, apply ReAct method by clipping penultimate activations under a threshold value.

 False
react_quantile Optional[float]

q value in the range [0, 1] used to compute the react clipping threshold defined as the q-th quantile penultimate layer activations. Defaults to 0.8.

 0.8
Source code in oodeel/methods/base.py 
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
class OODBaseDetector(ABC):
    """Base Class for methods that assign a score to unseen samples.

    Args:
        use_react (bool): if true, apply ReAct method by clipping penultimate
            activations under a threshold value.
        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
            the react clipping threshold defined as the q-th quantile penultimate layer
            activations. Defaults to 0.8.
    """

    def __init__(
        self,
        use_react: bool = False,
        react_quantile: float = 0.8,
        postproc_fns: List[Callable] = None,
    ):
        self.feature_extractor: FeatureExtractor = None
        self.use_react = use_react
        self.react_quantile = react_quantile
        self.react_threshold = None
        self.postproc_fns = self._sanitize_posproc_fns(postproc_fns)

    @abstractmethod
    def _score_tensor(self, inputs: TensorType) -> np.ndarray:
        """Computes an OOD score for input samples "inputs".

        Method to override with child classes.

        Args:
            inputs (TensorType): tensor to score
        Returns:
            Tuple[TensorType]: OOD scores, predicted logits
        """
        raise NotImplementedError()

    def _sanitize_posproc_fns(
        self,
        postproc_fns: Union[List[Callable], None],
    ) -> List[Callable]:
        """Sanitize postproc fns used at each layer output of the feature extractor.

        Args:
            postproc_fns (Optional[List[Callable]], optional): List of postproc
                functions, one per output layer. Defaults to None.

        Returns:
            List[Callable]: Sanitized postproc_fns list
        """
        if postproc_fns is not None:
            assert len(postproc_fns) == len(
                self.output_layers_id
            ), "len of postproc_fns and output_layers_id must match"

            def identity(x):
                return x

            postproc_fns = [identity if fn is None else fn for fn in postproc_fns]

        return postproc_fns

    def fit(
        self,
        model: Callable,
        fit_dataset: Optional[Union[ItemType, DatasetType]] = None,
        feature_layers_id: List[Union[int, str]] = [],
        input_layer_id: Optional[Union[int, str]] = None,
        **kwargs,
    ) -> None:
        """Prepare the detector for scoring:
        * Constructs the feature extractor based on the model
        * Calibrates the detector on ID data "fit_dataset" if needed,
            using self._fit_to_dataset

        Args:
            model: model to extract the features from
            fit_dataset: dataset to fit the detector on
            feature_layers_id (List[int]): list of str or int that identify
                features to output.
                If int, the rank of the layer in the layer list
                If str, the name of the layer. Defaults to [-1]
            input_layer_id (List[int]): = list of str or int that identify the input
                layer of the feature extractor.
                If int, the rank of the layer in the layer list
                If str, the name of the layer. Defaults to None.
        """
        (
            self.backend,
            self.data_handler,
            self.op,
            self.FeatureExtractorClass,
        ) = import_backend_specific_stuff(model)

        # if required by the method, check that fit_dataset is not None
        if self.requires_to_fit_dataset and fit_dataset is None:
            raise ValueError(
                "`fit_dataset` argument must be provided for this OOD detector"
            )

        # react: compute threshold (activation percentiles)
        if self.use_react:
            if fit_dataset is None:
                raise ValueError(
                    "if react quantile is not None, fit_dataset must be"
                    " provided to compute react activation threshold"
                )
            else:
                self.compute_react_threshold(model, fit_dataset)

        if (feature_layers_id == []) and (self.requires_internal_features):
            raise ValueError(
                "Explicitly specify feature_layers_id=[layer0, layer1,...], "
                + "where layer0, layer1,... are the names of the desired output "
                + "layers of your model. These can be int or str (even though str"
                + " is safer). To know what to put, have a look at model.summary() "
                + "with keras or model.named_modules() with pytorch"
            )

        self.feature_extractor = self._load_feature_extractor(
            model, feature_layers_id, input_layer_id
        )

        if fit_dataset is not None:
            self._fit_to_dataset(fit_dataset, **kwargs)

    def _load_feature_extractor(
        self,
        model: Callable,
        feature_layers_id: List[Union[int, str]] = None,
        input_layer_id: Optional[Union[int, str]] = None,
    ) -> Callable:
        """
        Loads feature extractor

        Args:
            model: a model (Keras or PyTorch) to load.
            feature_layers_id (List[int]): list of str or int that identify
                features to output.
                If int, the rank of the layer in the layer list
                If str, the name of the layer. Defaults to [-1]
            input_layer_id (List[int]): = list of str or int that identify the input
                layer of the feature extractor.
                If int, the rank of the layer in the layer list
                If str, the name of the layer. Defaults to None.

        Returns:
            FeatureExtractor: a feature extractor instance
        """
        feature_extractor = self.FeatureExtractorClass(
            model,
            feature_layers_id=feature_layers_id,
            input_layer_id=input_layer_id,
            react_threshold=self.react_threshold,
        )
        return feature_extractor

    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
        """
        Fits the OOD detector to fit_dataset.

        To be overrided in child classes (if needed)

        Args:
            fit_dataset: dataset to fit the OOD detector on
        """
        raise NotImplementedError()

    def score(
        self,
        dataset: Union[ItemType, DatasetType],
    ) -> np.ndarray:
        """
        Computes an OOD score for input samples "inputs".

        Args:
            dataset (Union[ItemType, DatasetType]): dataset or tensors to score

        Returns:
            tuple: scores or list of scores (depending on the input) and a dictionary
                containing logits and labels.
        """
        assert self.feature_extractor is not None, "Call .fit() before .score()"
        labels = None
        # Case 1: dataset is neither a tf.data.Dataset nor a torch.DataLoader
        if isinstance(dataset, get_args(ItemType)):
            tensor = self.data_handler.get_input_from_dataset_item(dataset)
            scores = self._score_tensor(tensor)
            logits = self.op.convert_to_numpy(self.feature_extractor._last_logits)

            # Get labels if dataset is a tuple/list
            if isinstance(dataset, (list, tuple)):
                labels = self.data_handler.get_label_from_dataset_item(dataset)
                labels = self.op.convert_to_numpy(labels)

        # Case 2: dataset is a tf.data.Dataset or a torch.DataLoader
        elif isinstance(dataset, get_args(DatasetType)):
            scores = np.array([])
            logits = None

            for item in dataset:
                tensor = self.data_handler.get_input_from_dataset_item(item)
                score_batch = self._score_tensor(tensor)
                logits_batch = self.op.convert_to_numpy(
                    self.feature_extractor._last_logits
                )

                # get the label if available
                if len(item) > 1:
                    labels_batch = self.data_handler.get_label_from_dataset_item(item)
                    labels = (
                        labels_batch
                        if labels is None
                        else np.append(labels, self.op.convert_to_numpy(labels_batch))
                    )

                scores = np.append(scores, score_batch)
                logits = (
                    logits_batch
                    if logits is None
                    else np.concatenate([logits, logits_batch], axis=0)
                )

        else:
            raise NotImplementedError(
                f"OODBaseDetector.score() not implemented for {type(dataset)}"
            )

        info = dict(labels=labels, logits=logits)
        return scores, info

    def compute_react_threshold(self, model: Callable, fit_dataset: DatasetType):
        penult_feat_extractor = self._load_feature_extractor(model, [-2])
        unclipped_features, _ = penult_feat_extractor.predict(fit_dataset)
        self.react_threshold = self.op.quantile(
            unclipped_features[0], self.react_quantile
        )

    def __call__(self, inputs: Union[ItemType, DatasetType]) -> np.ndarray:
        """
        Convenience wrapper for score

        Args:
            inputs (Union[ItemType, DatasetType]): dataset or tensors to score.
            threshold (float): threshold to use for distinguishing between OOD and ID

        Returns:
            np.ndarray: array of 0 for ID samples and 1 for OOD samples
        """
        return self.score(inputs)

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        raise NotImplementedError(
            "Property `requires_to_fit_dataset` is not implemented. It should return"
            + " a True or False boolean."
        )

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        raise NotImplementedError(
            "Property `requires_internal_dataset` is not implemented. It should return"
            + " a True or False boolean."
        )

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

__call__(inputs)

Convenience wrapper for score

Parameters:

Name Type Description Default
inputs Union[ItemType, DatasetType]

dataset or tensors to score.

 required
threshold float

threshold to use for distinguishing between OOD and ID

 required

Returns:

Type Description
ndarray

np.ndarray: array of 0 for ID samples and 1 for OOD samples
Source code in oodeel/methods/base.py 
277
278
279
280
281
282
283
284
285
286
287
288
def __call__(self, inputs: Union[ItemType, DatasetType]) -> np.ndarray:
    """
    Convenience wrapper for score

    Args:
        inputs (Union[ItemType, DatasetType]): dataset or tensors to score.
        threshold (float): threshold to use for distinguishing between OOD and ID

    Returns:
        np.ndarray: array of 0 for ID samples and 1 for OOD samples
    """
    return self.score(inputs)

fit(model, fit_dataset=None, feature_layers_id=[], input_layer_id=None, **kwargs)

Prepare the detector for scoring: * Constructs the feature extractor based on the model * Calibrates the detector on ID data "fit_dataset" if needed, using self._fit_to_dataset

Parameters:

Name Type Description Default
model Callable

model to extract the features from

 required
fit_dataset Optional[Union[ItemType, DatasetType]]

dataset to fit the detector on

 None
feature_layers_id List[int]

list of str or int that identify features to output. If int, the rank of the layer in the layer list If str, the name of the layer. Defaults to [-1]

 []
input_layer_id List[int]

= list of str or int that identify the input layer of the feature extractor. If int, the rank of the layer in the layer list If str, the name of the layer. Defaults to None.

 None
Source code in oodeel/methods/base.py 
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def fit(
    self,
    model: Callable,
    fit_dataset: Optional[Union[ItemType, DatasetType]] = None,
    feature_layers_id: List[Union[int, str]] = [],
    input_layer_id: Optional[Union[int, str]] = None,
    **kwargs,
) -> None:
    """Prepare the detector for scoring:
    * Constructs the feature extractor based on the model
    * Calibrates the detector on ID data "fit_dataset" if needed,
        using self._fit_to_dataset

    Args:
        model: model to extract the features from
        fit_dataset: dataset to fit the detector on
        feature_layers_id (List[int]): list of str or int that identify
            features to output.
            If int, the rank of the layer in the layer list
            If str, the name of the layer. Defaults to [-1]
        input_layer_id (List[int]): = list of str or int that identify the input
            layer of the feature extractor.
            If int, the rank of the layer in the layer list
            If str, the name of the layer. Defaults to None.
    """
    (
        self.backend,
        self.data_handler,
        self.op,
        self.FeatureExtractorClass,
    ) = import_backend_specific_stuff(model)

    # if required by the method, check that fit_dataset is not None
    if self.requires_to_fit_dataset and fit_dataset is None:
        raise ValueError(
            "`fit_dataset` argument must be provided for this OOD detector"
        )

    # react: compute threshold (activation percentiles)
    if self.use_react:
        if fit_dataset is None:
            raise ValueError(
                "if react quantile is not None, fit_dataset must be"
                " provided to compute react activation threshold"
            )
        else:
            self.compute_react_threshold(model, fit_dataset)

    if (feature_layers_id == []) and (self.requires_internal_features):
        raise ValueError(
            "Explicitly specify feature_layers_id=[layer0, layer1,...], "
            + "where layer0, layer1,... are the names of the desired output "
            + "layers of your model. These can be int or str (even though str"
            + " is safer). To know what to put, have a look at model.summary() "
            + "with keras or model.named_modules() with pytorch"
        )

    self.feature_extractor = self._load_feature_extractor(
        model, feature_layers_id, input_layer_id
    )

    if fit_dataset is not None:
        self._fit_to_dataset(fit_dataset, **kwargs)

score(dataset)

Computes an OOD score for input samples "inputs".

Parameters:

Name Type Description Default
dataset Union[ItemType, DatasetType]

dataset or tensors to score

 required

Returns:

Name Type Description
tuple ndarray

scores or list of scores (depending on the input) and a dictionary containing logits and labels.
Source code in oodeel/methods/base.py 
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
def score(
    self,
    dataset: Union[ItemType, DatasetType],
) -> np.ndarray:
    """
    Computes an OOD score for input samples "inputs".

    Args:
        dataset (Union[ItemType, DatasetType]): dataset or tensors to score

    Returns:
        tuple: scores or list of scores (depending on the input) and a dictionary
            containing logits and labels.
    """
    assert self.feature_extractor is not None, "Call .fit() before .score()"
    labels = None
    # Case 1: dataset is neither a tf.data.Dataset nor a torch.DataLoader
    if isinstance(dataset, get_args(ItemType)):
        tensor = self.data_handler.get_input_from_dataset_item(dataset)
        scores = self._score_tensor(tensor)
        logits = self.op.convert_to_numpy(self.feature_extractor._last_logits)

        # Get labels if dataset is a tuple/list
        if isinstance(dataset, (list, tuple)):
            labels = self.data_handler.get_label_from_dataset_item(dataset)
            labels = self.op.convert_to_numpy(labels)

    # Case 2: dataset is a tf.data.Dataset or a torch.DataLoader
    elif isinstance(dataset, get_args(DatasetType)):
        scores = np.array([])
        logits = None

        for item in dataset:
            tensor = self.data_handler.get_input_from_dataset_item(item)
            score_batch = self._score_tensor(tensor)
            logits_batch = self.op.convert_to_numpy(
                self.feature_extractor._last_logits
            )

            # get the label if available
            if len(item) > 1:
                labels_batch = self.data_handler.get_label_from_dataset_item(item)
                labels = (
                    labels_batch
                    if labels is None
                    else np.append(labels, self.op.convert_to_numpy(labels_batch))
                )

            scores = np.append(scores, score_batch)
            logits = (
                logits_batch
                if logits is None
                else np.concatenate([logits, logits_batch], axis=0)
            )

    else:
        raise NotImplementedError(
            f"OODBaseDetector.score() not implemented for {type(dataset)}"
        )

    info = dict(labels=labels, logits=logits)
    return scores, info

DKNN

Bases: OODBaseDetector

"Out-of-Distribution Detection with Deep Nearest Neighbors" https://arxiv.org/abs/2204.06507

Parameters:

Name Type Description Default
nearest int

number of nearest neighbors to consider. Defaults to 1.

 1
Source code in oodeel/methods/dknn.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
class DKNN(OODBaseDetector):
    """
    "Out-of-Distribution Detection with Deep Nearest Neighbors"
    https://arxiv.org/abs/2204.06507

    Args:
        nearest: number of nearest neighbors to consider.
            Defaults to 1.
    """

    def __init__(
        self,
        nearest: int = 1,
    ):
        super().__init__()

        self.index = None
        self.nearest = nearest

    def _fit_to_dataset(self, fit_dataset: Union[TensorType, DatasetType]) -> None:
        """
        Constructs the index from ID data "fit_dataset", which will be used for
        nearest neighbor search.

        Args:
            fit_dataset: input dataset (ID) to construct the index with.
        """
        fit_projected, _ = self.feature_extractor.predict(fit_dataset)
        fit_projected = self.op.convert_to_numpy(fit_projected[0])
        fit_projected = fit_projected.reshape(fit_projected.shape[0], -1)
        norm_fit_projected = self._l2_normalization(fit_projected)
        self.index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
        self.index.add(norm_fit_projected)

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes an OOD score for input samples "inputs" based on
        the distance to nearest neighbors in the feature space of self.model

        Args:
            inputs: input samples to score

        Returns:
            Tuple[np.ndarray]: scores, logits
        """

        input_projected, _ = self.feature_extractor.predict_tensor(inputs)
        input_projected = self.op.convert_to_numpy(input_projected[0])
        input_projected = input_projected.reshape(input_projected.shape[0], -1)
        norm_input_projected = self._l2_normalization(input_projected)
        scores, _ = self.index.search(norm_input_projected, self.nearest)
        return scores[:, -1]

    def _l2_normalization(self, feat: np.ndarray) -> np.ndarray:
        """L2 normalization of a tensor along the last dimension.

        Args:
            feat (np.ndarray): the tensor to normalize

        Returns:
            np.ndarray: the normalized tensor
        """
        return feat / (np.linalg.norm(feat, ord=2, axis=-1, keepdims=True) + 1e-10)

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return True

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return True

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_fit_to_dataset(fit_dataset)

Constructs the index from ID data "fit_dataset", which will be used for nearest neighbor search.

Parameters:

Name Type Description Default
fit_dataset Union[TensorType, DatasetType]

input dataset (ID) to construct the index with.

 required
Source code in oodeel/methods/dknn.py 
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def _fit_to_dataset(self, fit_dataset: Union[TensorType, DatasetType]) -> None:
    """
    Constructs the index from ID data "fit_dataset", which will be used for
    nearest neighbor search.

    Args:
        fit_dataset: input dataset (ID) to construct the index with.
    """
    fit_projected, _ = self.feature_extractor.predict(fit_dataset)
    fit_projected = self.op.convert_to_numpy(fit_projected[0])
    fit_projected = fit_projected.reshape(fit_projected.shape[0], -1)
    norm_fit_projected = self._l2_normalization(fit_projected)
    self.index = faiss.IndexFlatL2(norm_fit_projected.shape[1])
    self.index.add(norm_fit_projected)

_l2_normalization(feat)

L2 normalization of a tensor along the last dimension.

Parameters:

Name Type Description Default
feat ndarray

the tensor to normalize

 required

Returns:

Type Description
ndarray

np.ndarray: the normalized tensor
Source code in oodeel/methods/dknn.py 
86
87
88
89
90
91
92
93
94
95
def _l2_normalization(self, feat: np.ndarray) -> np.ndarray:
    """L2 normalization of a tensor along the last dimension.

    Args:
        feat (np.ndarray): the tensor to normalize

    Returns:
        np.ndarray: the normalized tensor
    """
    return feat / (np.linalg.norm(feat, ord=2, axis=-1, keepdims=True) + 1e-10)

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on the distance to nearest neighbors in the feature space of self.model

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/dknn.py 
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes an OOD score for input samples "inputs" based on
    the distance to nearest neighbors in the feature space of self.model

    Args:
        inputs: input samples to score

    Returns:
        Tuple[np.ndarray]: scores, logits
    """

    input_projected, _ = self.feature_extractor.predict_tensor(inputs)
    input_projected = self.op.convert_to_numpy(input_projected[0])
    input_projected = input_projected.reshape(input_projected.shape[0], -1)
    norm_input_projected = self._l2_normalization(input_projected)
    scores, _ = self.index.search(norm_input_projected, self.nearest)
    return scores[:, -1]

Energy

Bases: OODBaseDetector

Energy Score method for OOD detection. "Energy-based Out-of-distribution Detection" https://arxiv.org/abs/2010.03759

This method assumes that the model has been trained with cross entropy loss $CE(model(x))$ where $model(x)=(l_{c})_{c=1}^{C}$ are the logits predicted for input $x$. The implementation assumes that the logits are retreieved using the output with linear activation.

The energy score for input $x$ is given by $$ -\log \sum_{c=0}^C \exp(l_c)$$

where $model(x)=(l_{c})_{c=1}^{C}$ are the logits predicted by the model on $x$. As always, training data is expected to have lower score than OOD data.

Parameters:

Name Type Description Default
use_react bool

if true, apply ReAct method by clipping penultimate activations under a threshold value.

 False
react_quantile Optional[float]

q value in the range [0, 1] used to compute the react clipping threshold defined as the q-th quantile penultimate layer activations. Defaults to 0.8.

 0.8
Source code in oodeel/methods/energy.py 
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
class Energy(OODBaseDetector):
    r"""
    Energy Score method for OOD detection.
    "Energy-based Out-of-distribution Detection"
    https://arxiv.org/abs/2010.03759

    This method assumes that the model has been trained with cross entropy loss
    $CE(model(x))$ where $model(x)=(l_{c})_{c=1}^{C}$ are the logits
    predicted for input $x$.
    The implementation assumes that the logits are retreieved using the output with
    linear activation.

    The energy score for input $x$ is given by
    $$ -\log \sum_{c=0}^C \exp(l_c)$$

    where $model(x)=(l_{c})_{c=1}^{C}$ are the logits predicted by the model on
    $x$.
    As always, training data is expected to have lower score than OOD data.

    Args:
        use_react (bool): if true, apply ReAct method by clipping penultimate
            activations under a threshold value.
        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
            the react clipping threshold defined as the q-th quantile penultimate layer
            activations. Defaults to 0.8.
    """

    def __init__(
        self,
        use_react: bool = False,
        react_quantile: float = 0.8,
    ):
        super().__init__(
            use_react=use_react,
            react_quantile=react_quantile,
        )

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes an OOD score for input samples "inputs" based on
        energy, namey $-logsumexp(logits(inputs))$.

        Args:
            inputs: input samples to score

        Returns:
            Tuple[np.ndarray]: scores, logits
        """
        # compute logits (softmax(logits,axis=1) is the actual softmax
        # output minimized using binary cross entropy)
        _, logits = self.feature_extractor.predict_tensor(inputs)
        logits = self.op.convert_to_numpy(logits)
        scores = -logsumexp(logits, axis=1)
        return scores

    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
        """
        Fits the OOD detector to fit_dataset.

        Args:
            fit_dataset: dataset to fit the OOD detector on
        """
        pass

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return False

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return False

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_fit_to_dataset(fit_dataset)

Fits the OOD detector to fit_dataset.

Parameters:

Name Type Description Default
fit_dataset DatasetType

dataset to fit the OOD detector on

 required
Source code in oodeel/methods/energy.py 
87
88
89
90
91
92
93
94
def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
    """
    Fits the OOD detector to fit_dataset.

    Args:
        fit_dataset: dataset to fit the OOD detector on
    """
    pass

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on energy, namey $-logsumexp(logits(inputs))$.

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/energy.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes an OOD score for input samples "inputs" based on
    energy, namey $-logsumexp(logits(inputs))$.

    Args:
        inputs: input samples to score

    Returns:
        Tuple[np.ndarray]: scores, logits
    """
    # compute logits (softmax(logits,axis=1) is the actual softmax
    # output minimized using binary cross entropy)
    _, logits = self.feature_extractor.predict_tensor(inputs)
    logits = self.op.convert_to_numpy(logits)
    scores = -logsumexp(logits, axis=1)
    return scores

Entropy

Bases: OODBaseDetector

Entropy OOD score

The method consists in using the Entropy of the input data computed using the Entropy $\sum_{c=0}^C p(y=c| x) \times log(p(y=c | x))$ where $p(y=c| x) = \text{model}(x)$.

Reference https://proceedings.neurips.cc/paper/2019/hash/1e79596878b2320cac26dd792a6c51c9-Abstract.html, Neurips 2019.

Parameters:

Name Type Description Default
use_react bool

if true, apply ReAct method by clipping penultimate activations under a threshold value.

 False
react_quantile Optional[float]

q value in the range [0, 1] used to compute the react clipping threshold defined as the q-th quantile penultimate layer activations. Defaults to 0.8.

 0.8
Source code in oodeel/methods/entropy.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
class Entropy(OODBaseDetector):
    r"""
    Entropy OOD score


    The method consists in using the Entropy of the input data computed using the
    Entropy $\sum_{c=0}^C p(y=c| x) \times log(p(y=c | x))$ where
    $p(y=c| x) = \text{model}(x)$.

    **Reference**
    https://proceedings.neurips.cc/paper/2019/hash/1e79596878b2320cac26dd792a6c51c9-Abstract.html,
    Neurips 2019.

    Args:
        use_react (bool): if true, apply ReAct method by clipping penultimate
            activations under a threshold value.
        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
            the react clipping threshold defined as the q-th quantile penultimate layer
            activations. Defaults to 0.8.
    """

    def __init__(
        self,
        use_react: bool = False,
        react_quantile: float = 0.8,
    ):
        super().__init__(
            use_react=use_react,
            react_quantile=react_quantile,
        )

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes an OOD score for input samples "inputs" based on
        entropy.

        Args:
            inputs: input samples to score

        Returns:
            Tuple[np.ndarray]: scores, logits
        """

        # compute logits (softmax(logits,axis=1) is the actual softmax
        # output minimized using binary cross entropy)
        _, logits = self.feature_extractor.predict_tensor(inputs)
        probits = self.op.softmax(logits)
        probits = self.op.convert_to_numpy(probits)
        scores = np.sum(probits * np.log(probits), axis=1)
        return -scores

    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
        """
        Fits the OOD detector to fit_dataset.

        Args:
            fit_dataset: dataset to fit the OOD detector on
        """
        pass

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return False

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return False

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_fit_to_dataset(fit_dataset)

Fits the OOD detector to fit_dataset.

Parameters:

Name Type Description Default
fit_dataset DatasetType

dataset to fit the OOD detector on

 required
Source code in oodeel/methods/entropy.py 
82
83
84
85
86
87
88
89
def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
    """
    Fits the OOD detector to fit_dataset.

    Args:
        fit_dataset: dataset to fit the OOD detector on
    """
    pass

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on entropy.

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/entropy.py 
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes an OOD score for input samples "inputs" based on
    entropy.

    Args:
        inputs: input samples to score

    Returns:
        Tuple[np.ndarray]: scores, logits
    """

    # compute logits (softmax(logits,axis=1) is the actual softmax
    # output minimized using binary cross entropy)
    _, logits = self.feature_extractor.predict_tensor(inputs)
    probits = self.op.softmax(logits)
    probits = self.op.convert_to_numpy(probits)
    scores = np.sum(probits * np.log(probits), axis=1)
    return -scores

Gram

Bases: OODBaseDetector

"Detecting Out-of-Distribution Examples with Gram Matrices" link

Important Disclaimer: Taking the statistics of min/max deviation, as in the paper raises some problems.

The method often yields a score of zero for some tasks. This is expected since the min/max among the samples of a random variable becomes more and more extreme with the sample size. As a result, computing the min/max over the training set is likely to produce min/max values that are so extreme that none of the in distribution correlations of the validation set goes beyond these threshold. The worst is that a significant part of ood data does not exceed the thresholds either. This can be aleviated by computing the min/max over a limited number of sample. However, it is counter-intuitive and, in our opinion, not desirable: adding some more information should only improve a method.

Hence, we decided to replace the min/max by the q / 1-q quantile, with q a new parameter of the method. Specifically, instead of the deviation as defined in eq. 3 of the paper, we use the definition $$ \delta(t_q, t_{1-q}, value) = \begin{cases} 0 & \text{if} \; t_q \leq value \leq t_{1-q}, \;\; \frac{t_q - value}{|t_q|} & \text{if } value < t_q, \;\; \frac{value - t_{1-q}}{|t_q|} & \text{if } value > t_{1-q} \end{cases} $$ With this new deviation, the more point we add, the more accurate the quantile becomes. In addition, the method can be made more or less discriminative by toggling the value of q.

Finally, we found that this approach improved the performance of the baseline in our experiments.

Parameters:

Name Type Description Default
orders List[int]

power orders to consider for the correlation matrix

 [i for i in range(1, 11)]
quantile float

quantile to consider for the correlations to build the deviation threshold.

 0.01
Source code in oodeel/methods/gram.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
class Gram(OODBaseDetector):
    r"""
    "Detecting Out-of-Distribution Examples with Gram Matrices"
    [link](https://proceedings.mlr.press/v119/sastry20a.html)

    **Important Disclaimer**: Taking the statistics of min/max deviation,
    as in the paper raises some problems.

    The method often yields a score of zero for some tasks.
    This is expected since the min/max among the samples of a random
    variable becomes more and more extreme with the sample
    size. As a result, computing the min/max over the training set is likely to produce
    min/max values that are so extreme that none of the in distribution correlations of
    the validation set goes beyond these threshold. The worst is that a significant
    part of ood data does not exceed the thresholds either. This can be aleviated by
    computing the min/max over a limited number of sample. However, it is
    counter-intuitive and, in our opinion, not desirable: adding
    some more information should only improve a method.

    Hence, we decided to replace the min/max by the q / 1-q quantile, with q a new
    parameter of the method. Specifically, instead of the deviation as defined in
    eq. 3 of the paper, we use the definition
    $$
    \delta(t_q, t_{1-q}, value) =
    \begin{cases}
        0 & \text{if} \; t_q \leq value \leq t_{1-q},  \;\;
        \frac{t_q - value}{|t_q|} & \text{if } value < t_q,  \;\;
        \frac{value - t_{1-q}}{|t_q|} & \text{if } value > t_{1-q}
    \end{cases}
    $$
    With this new deviation, the more point we add, the more accurate the quantile
    becomes. In addition, the method can be made more or less discriminative by
    toggling the value of q.

    Finally, we found that this approach improved the performance of the baseline in
    our experiments.

    Args:
        orders (List[int]): power orders to consider for the correlation matrix
        quantile (float): quantile to consider for the correlations to build the
            deviation threshold.

    """

    def __init__(
        self,
        orders: List[int] = [i for i in range(1, 11)],
        quantile: float = 0.01,
    ):
        super().__init__()
        if isinstance(orders, int):
            orders = [orders]
        self.orders = orders
        self.postproc_fns = None
        self.quantile = quantile

    def _fit_to_dataset(
        self,
        fit_dataset: Union[TensorType, DatasetType],
        val_split: float = 0.2,
    ) -> None:
        """
        Compute the quantiles of channelwise correlations for each layer, power of
        gram matrices, and class. Then, compute the normalization constants for the
        deviation. To stay faithful to the spirit of the original method, we still name
        the quantiles min/max

        Args:
            fit_dataset (Union[TensorType, DatasetType]): input dataset (ID) to
                construct the index with.
            val_split (float): The percentage of fit data to use as validation data for
                normalization. Default to 0.2.
        """
        self.postproc_fns = [
            self._stat for i in range(len(self.feature_extractor.feature_layers_id))
        ]

        fit_stats, info = self.feature_extractor.predict(
            fit_dataset, postproc_fns=self.postproc_fns, return_labels=True
        )
        labels = info["labels"]
        self._classes = np.sort(np.unique(self.op.convert_to_numpy(labels)))

        full_indices = np.arange(labels.shape[0])
        train_indices, val_indices = train_test_split(full_indices, test_size=val_split)
        train_indices = self.op.from_numpy(
            [bool(ind in train_indices) for ind in full_indices]
        )
        val_indices = self.op.from_numpy(
            [bool(ind in val_indices) for ind in full_indices]
        )

        val_stats = [fit_stat[val_indices] for fit_stat in fit_stats]
        fit_stats = [fit_stat[train_indices] for fit_stat in fit_stats]
        labels = labels[train_indices]

        self.min_maxs = dict()
        for cls in self._classes:
            indexes = self.op.equal(labels, cls)
            min_maxs = []
            for fit_stat in fit_stats:
                fit_stat = fit_stat[indexes]
                mins = self.op.unsqueeze(
                    self.op.quantile(fit_stat, self.quantile, dim=0), -1
                )
                maxs = self.op.unsqueeze(
                    self.op.quantile(fit_stat, 1 - self.quantile, dim=0), -1
                )
                min_max = self.op.cat([mins, maxs], dim=-1)
                min_maxs.append(min_max)

            self.min_maxs[cls] = min_maxs

        devnorm = []
        for cls in self._classes:
            min_maxs = []
            for min_max in self.min_maxs[cls]:
                min_maxs.append(
                    self.op.stack([min_max for i in range(val_stats[0].shape[0])])
                )
            devnorm.append(
                [
                    float(self.op.mean(dev))
                    for dev in self._deviation(val_stats, min_maxs)
                ]
            )
        self.devnorm = np.mean(np.array(devnorm), axis=0)

    def _score_tensor(self, inputs: TensorType) -> np.ndarray:
        """
        Computes an OOD score for input samples "inputs" based on
        the aggregation of deviations from quantiles of in-distribution channel-wise
        correlations evaluate for each layer, power of gram matrices, and class.

        Args:
            inputs: input samples to score

        Returns:
            scores
        """

        tensor_stats, _ = self.feature_extractor.predict_tensor(
            inputs, postproc_fns=self.postproc_fns
        )

        _, logits = self.feature_extractor.predict_tensor(inputs)
        preds = self.op.convert_to_numpy(self.op.argmax(logits, dim=1))

        # We stack the min_maxs for each class depending on the prediction for each
        # samples
        min_maxs = []
        for i in range(len(tensor_stats)):
            min_maxs.append(self.op.stack([self.min_maxs[label][i] for label in preds]))

        tensor_dev = self._deviation(tensor_stats, min_maxs)
        score = self.op.mean(
            self.op.cat(
                [
                    self.op.unsqueeze(tensor_dev_l, dim=0) / devnorm_l
                    for tensor_dev_l, devnorm_l in zip(tensor_dev, self.devnorm)
                ]
            ),
            dim=0,
        )
        return self.op.convert_to_numpy(score)

    def _deviation(
        self, stats: List[TensorType], min_maxs: List[TensorType]
    ) -> List[TensorType]:
        """Compute the deviation wrt quantiles (min/max) for feature_maps

        Args:
            stats (TensorType): The list of gram matrices (stacked power-wise)
                for which we want to compute the deviation.
            min_maxs (TensorType): The quantiles (tensorised) to compute the deviation
                against.

        Returns:
            List(TensorType): A list with one element per layer containing a tensor of
                per-sample deviation.
        """
        deviation = []
        for stat, min_max in zip(stats, min_maxs):
            where_min = self.op.where(stat < min_max[..., 0], 1.0, 0.0)
            where_max = self.op.where(stat > min_max[..., 1], 1.0, 0.0)
            deviation_min = (
                (min_max[..., 0] - stat)
                / (self.op.abs(min_max[..., 0]) + 1e-6)
                * where_min
            )
            deviation_max = (
                (stat - min_max[..., 1])
                / (self.op.abs(min_max[..., 1]) + 1e-6)
                * where_max
            )
            deviation.append(self.op.sum(deviation_min + deviation_max, dim=(1, 2)))
        return deviation

    def _stat(self, feature_map: TensorType) -> TensorType:
        """Compute the correlation map (stat) for a given feature map. The values
        for each power of gram matrix are contained in the same tensor

        Args:
            feature_map (TensorType): The input feature_map

        Returns:
            TensorType: The stacked gram matrices power-wise.
        """
        fm_s = feature_map.shape
        stat = []
        for p in self.orders:
            feature_map_p = feature_map**p
            # construct the Gram matrix
            if len(fm_s) == 2:
                # build gram matrix for feature map of shape [dim_dense_layer, 1]
                feature_map_p = self.op.einsum(
                    "bi,bj->bij", feature_map_p, feature_map_p
                )
            elif len(fm_s) >= 3:
                # flatten the feature map
                if self.backend == "tensorflow":
                    feature_map_p = self.op.reshape(
                        self.op.einsum("i...j->ij...", feature_map_p),
                        (fm_s[0], fm_s[-1], -1),
                    )
                else:
                    feature_map_p = self.op.reshape(
                        feature_map_p, (fm_s[0], fm_s[1], -1)
                    )
                feature_map_p = self.op.matmul(
                    feature_map_p, self.op.permute(feature_map_p, (0, 2, 1))
                )
            feature_map_p = self.op.sign(feature_map_p) * (
                self.op.abs(feature_map_p) ** (1 / p)
            )
            # get the lower triangular part of the matrix
            feature_map_p = self.op.tril(feature_map_p)
            # directly sum row-wise (to limit computational burden)
            feature_map_p = self.op.sum(feature_map_p, dim=2)
            # stat.append(self.op.t(feature_map_p))
            stat.append(feature_map_p)
        stat = self.op.stack(stat, 1)
        return stat

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return True

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return False

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_deviation(stats, min_maxs)

Compute the deviation wrt quantiles (min/max) for feature_maps

Parameters:

Name Type Description Default
stats TensorType

The list of gram matrices (stacked power-wise) for which we want to compute the deviation.

 required
min_maxs TensorType

The quantiles (tensorised) to compute the deviation against.

 required

Returns:

Name Type Description
List TensorType

A list with one element per layer containing a tensor of per-sample deviation.
Source code in oodeel/methods/gram.py 
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
def _deviation(
    self, stats: List[TensorType], min_maxs: List[TensorType]
) -> List[TensorType]:
    """Compute the deviation wrt quantiles (min/max) for feature_maps

    Args:
        stats (TensorType): The list of gram matrices (stacked power-wise)
            for which we want to compute the deviation.
        min_maxs (TensorType): The quantiles (tensorised) to compute the deviation
            against.

    Returns:
        List(TensorType): A list with one element per layer containing a tensor of
            per-sample deviation.
    """
    deviation = []
    for stat, min_max in zip(stats, min_maxs):
        where_min = self.op.where(stat < min_max[..., 0], 1.0, 0.0)
        where_max = self.op.where(stat > min_max[..., 1], 1.0, 0.0)
        deviation_min = (
            (min_max[..., 0] - stat)
            / (self.op.abs(min_max[..., 0]) + 1e-6)
            * where_min
        )
        deviation_max = (
            (stat - min_max[..., 1])
            / (self.op.abs(min_max[..., 1]) + 1e-6)
            * where_max
        )
        deviation.append(self.op.sum(deviation_min + deviation_max, dim=(1, 2)))
    return deviation

_fit_to_dataset(fit_dataset, val_split=0.2)

Compute the quantiles of channelwise correlations for each layer, power of gram matrices, and class. Then, compute the normalization constants for the deviation. To stay faithful to the spirit of the original method, we still name the quantiles min/max

Parameters:

Name Type Description Default
fit_dataset Union[TensorType, DatasetType]

input dataset (ID) to construct the index with.

 required
val_split float

The percentage of fit data to use as validation data for normalization. Default to 0.2.

 0.2
Source code in oodeel/methods/gram.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def _fit_to_dataset(
    self,
    fit_dataset: Union[TensorType, DatasetType],
    val_split: float = 0.2,
) -> None:
    """
    Compute the quantiles of channelwise correlations for each layer, power of
    gram matrices, and class. Then, compute the normalization constants for the
    deviation. To stay faithful to the spirit of the original method, we still name
    the quantiles min/max

    Args:
        fit_dataset (Union[TensorType, DatasetType]): input dataset (ID) to
            construct the index with.
        val_split (float): The percentage of fit data to use as validation data for
            normalization. Default to 0.2.
    """
    self.postproc_fns = [
        self._stat for i in range(len(self.feature_extractor.feature_layers_id))
    ]

    fit_stats, info = self.feature_extractor.predict(
        fit_dataset, postproc_fns=self.postproc_fns, return_labels=True
    )
    labels = info["labels"]
    self._classes = np.sort(np.unique(self.op.convert_to_numpy(labels)))

    full_indices = np.arange(labels.shape[0])
    train_indices, val_indices = train_test_split(full_indices, test_size=val_split)
    train_indices = self.op.from_numpy(
        [bool(ind in train_indices) for ind in full_indices]
    )
    val_indices = self.op.from_numpy(
        [bool(ind in val_indices) for ind in full_indices]
    )

    val_stats = [fit_stat[val_indices] for fit_stat in fit_stats]
    fit_stats = [fit_stat[train_indices] for fit_stat in fit_stats]
    labels = labels[train_indices]

    self.min_maxs = dict()
    for cls in self._classes:
        indexes = self.op.equal(labels, cls)
        min_maxs = []
        for fit_stat in fit_stats:
            fit_stat = fit_stat[indexes]
            mins = self.op.unsqueeze(
                self.op.quantile(fit_stat, self.quantile, dim=0), -1
            )
            maxs = self.op.unsqueeze(
                self.op.quantile(fit_stat, 1 - self.quantile, dim=0), -1
            )
            min_max = self.op.cat([mins, maxs], dim=-1)
            min_maxs.append(min_max)

        self.min_maxs[cls] = min_maxs

    devnorm = []
    for cls in self._classes:
        min_maxs = []
        for min_max in self.min_maxs[cls]:
            min_maxs.append(
                self.op.stack([min_max for i in range(val_stats[0].shape[0])])
            )
        devnorm.append(
            [
                float(self.op.mean(dev))
                for dev in self._deviation(val_stats, min_maxs)
            ]
        )
    self.devnorm = np.mean(np.array(devnorm), axis=0)

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on the aggregation of deviations from quantiles of in-distribution channel-wise correlations evaluate for each layer, power of gram matrices, and class.

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
ndarray

scores
Source code in oodeel/methods/gram.py 
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
def _score_tensor(self, inputs: TensorType) -> np.ndarray:
    """
    Computes an OOD score for input samples "inputs" based on
    the aggregation of deviations from quantiles of in-distribution channel-wise
    correlations evaluate for each layer, power of gram matrices, and class.

    Args:
        inputs: input samples to score

    Returns:
        scores
    """

    tensor_stats, _ = self.feature_extractor.predict_tensor(
        inputs, postproc_fns=self.postproc_fns
    )

    _, logits = self.feature_extractor.predict_tensor(inputs)
    preds = self.op.convert_to_numpy(self.op.argmax(logits, dim=1))

    # We stack the min_maxs for each class depending on the prediction for each
    # samples
    min_maxs = []
    for i in range(len(tensor_stats)):
        min_maxs.append(self.op.stack([self.min_maxs[label][i] for label in preds]))

    tensor_dev = self._deviation(tensor_stats, min_maxs)
    score = self.op.mean(
        self.op.cat(
            [
                self.op.unsqueeze(tensor_dev_l, dim=0) / devnorm_l
                for tensor_dev_l, devnorm_l in zip(tensor_dev, self.devnorm)
            ]
        ),
        dim=0,
    )
    return self.op.convert_to_numpy(score)

_stat(feature_map)

Compute the correlation map (stat) for a given feature map. The values for each power of gram matrix are contained in the same tensor

Parameters:

Name Type Description Default
feature_map TensorType

The input feature_map

 required

Returns:

Name Type Description
TensorType TensorType

The stacked gram matrices power-wise.
Source code in oodeel/methods/gram.py 
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
def _stat(self, feature_map: TensorType) -> TensorType:
    """Compute the correlation map (stat) for a given feature map. The values
    for each power of gram matrix are contained in the same tensor

    Args:
        feature_map (TensorType): The input feature_map

    Returns:
        TensorType: The stacked gram matrices power-wise.
    """
    fm_s = feature_map.shape
    stat = []
    for p in self.orders:
        feature_map_p = feature_map**p
        # construct the Gram matrix
        if len(fm_s) == 2:
            # build gram matrix for feature map of shape [dim_dense_layer, 1]
            feature_map_p = self.op.einsum(
                "bi,bj->bij", feature_map_p, feature_map_p
            )
        elif len(fm_s) >= 3:
            # flatten the feature map
            if self.backend == "tensorflow":
                feature_map_p = self.op.reshape(
                    self.op.einsum("i...j->ij...", feature_map_p),
                    (fm_s[0], fm_s[-1], -1),
                )
            else:
                feature_map_p = self.op.reshape(
                    feature_map_p, (fm_s[0], fm_s[1], -1)
                )
            feature_map_p = self.op.matmul(
                feature_map_p, self.op.permute(feature_map_p, (0, 2, 1))
            )
        feature_map_p = self.op.sign(feature_map_p) * (
            self.op.abs(feature_map_p) ** (1 / p)
        )
        # get the lower triangular part of the matrix
        feature_map_p = self.op.tril(feature_map_p)
        # directly sum row-wise (to limit computational burden)
        feature_map_p = self.op.sum(feature_map_p, dim=2)
        # stat.append(self.op.t(feature_map_p))
        stat.append(feature_map_p)
    stat = self.op.stack(stat, 1)
    return stat

MLS

Bases: OODBaseDetector

Maximum Logit Scores method for OOD detection. "Open-Set Recognition: a Good Closed-Set Classifier is All You Need?" https://arxiv.org/abs/2110.06207, and Maximum Softmax Score "A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks" http://arxiv.org/abs/1610.02136

Parameters:

Name Type Description Default
output_activation str

activation function for the last layer. If "linear", the method is MLS and if "softmax", the method is MSS. Defaults to "linear".

 'linear'
use_react bool

if true, apply ReAct method by clipping penultimate activations under a threshold value.

 False
react_quantile Optional[float]

q value in the range [0, 1] used to compute the react clipping threshold defined as the q-th quantile penultimate layer activations. Defaults to 0.8.

 0.8
Source code in oodeel/methods/mls.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
class MLS(OODBaseDetector):
    """
    Maximum Logit Scores method for OOD detection.
    "Open-Set Recognition: a Good Closed-Set Classifier is All You Need?"
    https://arxiv.org/abs/2110.06207,
    and Maximum Softmax Score
    "A Baseline for Detecting Misclassified and Out-of-Distribution Examples
    in Neural Networks"
    http://arxiv.org/abs/1610.02136

    Args:
        output_activation (str): activation function for the last layer. If "linear",
            the method is MLS and if "softmax", the method is MSS.
            Defaults to "linear".
        use_react (bool): if true, apply ReAct method by clipping penultimate
            activations under a threshold value.
        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
            the react clipping threshold defined as the q-th quantile penultimate layer
            activations. Defaults to 0.8.
    """

    def __init__(
        self,
        output_activation: str = "linear",
        use_react: bool = False,
        react_quantile: float = 0.8,
    ):
        super().__init__(
            use_react=use_react,
            react_quantile=react_quantile,
        )
        self.output_activation = output_activation

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes an OOD score for input samples "inputs" based on
        the distance to nearest neighbors in the feature space of self.model

        Args:
            inputs: input samples to score

        Returns:
            Tuple[np.ndarray]: scores, logits
        """

        _, logits = self.feature_extractor.predict_tensor(inputs)
        if self.output_activation == "softmax":
            logits = self.op.softmax(logits)
        logits = self.op.convert_to_numpy(logits)
        scores = -np.max(logits, axis=1)
        return scores

    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
        """
        Fits the OOD detector to fit_dataset.

        Args:
            fit_dataset: dataset to fit the OOD detector on
        """
        pass

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return False

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return False

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_fit_to_dataset(fit_dataset)

Fits the OOD detector to fit_dataset.

Parameters:

Name Type Description Default
fit_dataset DatasetType

dataset to fit the OOD detector on

 required
Source code in oodeel/methods/mls.py 
83
84
85
86
87
88
89
90
def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
    """
    Fits the OOD detector to fit_dataset.

    Args:
        fit_dataset: dataset to fit the OOD detector on
    """
    pass

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on the distance to nearest neighbors in the feature space of self.model

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/mls.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes an OOD score for input samples "inputs" based on
    the distance to nearest neighbors in the feature space of self.model

    Args:
        inputs: input samples to score

    Returns:
        Tuple[np.ndarray]: scores, logits
    """

    _, logits = self.feature_extractor.predict_tensor(inputs)
    if self.output_activation == "softmax":
        logits = self.op.softmax(logits)
    logits = self.op.convert_to_numpy(logits)
    scores = -np.max(logits, axis=1)
    return scores

Mahalanobis

Bases: OODBaseDetector

"A Simple Unified Framework for Detecting Out-of-Distribution Samples and Adversarial Attacks" https://arxiv.org/abs/1807.03888

Parameters:

Name Type Description Default
eps float

magnitude for gradient based input perturbation. Defaults to 0.02.

 0.002
Source code in oodeel/methods/mahalanobis.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
class Mahalanobis(OODBaseDetector):
    """
    "A Simple Unified Framework for Detecting Out-of-Distribution Samples and
    Adversarial Attacks"
    https://arxiv.org/abs/1807.03888

    Args:
        eps (float): magnitude for gradient based input perturbation.
            Defaults to 0.02.
    """

    def __init__(
        self,
        eps: float = 0.002,
    ):
        super(Mahalanobis, self).__init__()
        self.eps = eps

    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
        """
        Constructs the mean covariance matrix from ID data "fit_dataset", whose
        pseudo-inverse will be used for mahalanobis distance computation.

        Args:
            fit_dataset (Union[TensorType, DatasetType]): input dataset (ID)
        """
        # extract features and labels
        features, infos = self.feature_extractor.predict(fit_dataset)
        labels = infos["labels"]

        # unique sorted classes
        self._classes = np.sort(np.unique(self.op.convert_to_numpy(labels)))

        # compute mus and covs
        mus = dict()
        covs = dict()
        for cls in self._classes:
            indexes = self.op.equal(labels, cls)
            _features_cls = self.op.flatten(features[0][indexes])
            mus[cls] = self.op.mean(_features_cls, dim=0)
            _zero_f_cls = _features_cls - mus[cls]
            covs[cls] = (
                self.op.matmul(self.op.t(_zero_f_cls), _zero_f_cls)
                / _zero_f_cls.shape[0]
            )

        # mean cov and its inverse
        mean_cov = self.op.mean(self.op.stack(list(covs.values())), dim=0)

        self._mus = mus
        self._pinv_cov = self.op.pinv(mean_cov)

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes an OOD score for input samples "inputs" based on the mahalanobis
        distance with respect to the closest class-conditional Gaussian distribution.

        Args:
            inputs (TensorType): input samples

        Returns:
            Tuple[np.ndarray]: scores, logits
        """
        # input preprocessing (perturbation)
        if self.eps > 0:
            inputs_p = self._input_perturbation(inputs)
        else:
            inputs_p = inputs

        # mahalanobis score on perturbed inputs
        features_p, _ = self.feature_extractor.predict_tensor(inputs_p)
        features_p = self.op.flatten(features_p[0])
        gaussian_score_p = self._mahalanobis_score(features_p)

        # take the highest score for each sample
        gaussian_score_p = self.op.max(gaussian_score_p, dim=1)
        return -self.op.convert_to_numpy(gaussian_score_p)

    def _input_perturbation(self, inputs: TensorType) -> TensorType:
        """
        Apply small perturbation on inputs to make the in- and out- distribution
        samples more separable.
        See original paper for more information (section 2.2)
        https://arxiv.org/abs/1807.03888

        Args:
            inputs (TensorType): input samples

        Returns:
            TensorType: Perturbed inputs
        """

        def __loss_fn(inputs: TensorType) -> TensorType:
            """
            Loss function for the input perturbation.

            Args:
                inputs (TensorType): input samples

            Returns:
                TensorType: loss value
            """
            # extract features
            out_features, _ = self.feature_extractor.predict(inputs, detach=False)
            out_features = self.op.flatten(out_features[0])
            # get mahalanobis score for the class maximizing it
            gaussian_score = self._mahalanobis_score(out_features)
            log_probs_f = self.op.max(gaussian_score, dim=1)
            return self.op.mean(-log_probs_f)

        # compute gradient
        gradient = self.op.gradient(__loss_fn, inputs)
        gradient = self.op.sign(gradient)

        inputs_p = inputs - self.eps * gradient
        return inputs_p

    def _mahalanobis_score(self, out_features: TensorType) -> TensorType:
        """
        Mahalanobis distance-based confidence score. For each test sample, it computes
        the log of the probability densities of some observations (assuming a
        normal distribution) using the mahalanobis distance with respect to every
        class-conditional distributions.

        Args:
            out_features (TensorType): test samples features

        Returns:
            TensorType: confidence scores (conditionally to each class)
        """
        gaussian_scores = list()
        # compute scores conditionally to each class
        for cls in self._classes:
            # center features wrt class-cond dist.
            mu = self._mus[cls]
            zero_f = out_features - mu
            # gaussian log prob density (mahalanobis)
            log_probs_f = -0.5 * self.op.diag(
                self.op.matmul(
                    self.op.matmul(zero_f, self._pinv_cov), self.op.t(zero_f)
                )
            )
            gaussian_scores.append(self.op.reshape(log_probs_f, (-1, 1)))
        # concatenate scores
        gaussian_score = self.op.cat(gaussian_scores, 1)
        return gaussian_score

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return True

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return True

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_fit_to_dataset(fit_dataset)

Constructs the mean covariance matrix from ID data "fit_dataset", whose pseudo-inverse will be used for mahalanobis distance computation.

Parameters:

Name Type Description Default
fit_dataset Union[TensorType, DatasetType]

input dataset (ID)

 required
Source code in oodeel/methods/mahalanobis.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
    """
    Constructs the mean covariance matrix from ID data "fit_dataset", whose
    pseudo-inverse will be used for mahalanobis distance computation.

    Args:
        fit_dataset (Union[TensorType, DatasetType]): input dataset (ID)
    """
    # extract features and labels
    features, infos = self.feature_extractor.predict(fit_dataset)
    labels = infos["labels"]

    # unique sorted classes
    self._classes = np.sort(np.unique(self.op.convert_to_numpy(labels)))

    # compute mus and covs
    mus = dict()
    covs = dict()
    for cls in self._classes:
        indexes = self.op.equal(labels, cls)
        _features_cls = self.op.flatten(features[0][indexes])
        mus[cls] = self.op.mean(_features_cls, dim=0)
        _zero_f_cls = _features_cls - mus[cls]
        covs[cls] = (
            self.op.matmul(self.op.t(_zero_f_cls), _zero_f_cls)
            / _zero_f_cls.shape[0]
        )

    # mean cov and its inverse
    mean_cov = self.op.mean(self.op.stack(list(covs.values())), dim=0)

    self._mus = mus
    self._pinv_cov = self.op.pinv(mean_cov)

_input_perturbation(inputs)

Apply small perturbation on inputs to make the in- and out- distribution samples more separable. See original paper for more information (section 2.2) https://arxiv.org/abs/1807.03888

Parameters:

Name Type Description Default
inputs TensorType

input samples

 required

Returns:

Name Type Description
TensorType TensorType

Perturbed inputs
Source code in oodeel/methods/mahalanobis.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def _input_perturbation(self, inputs: TensorType) -> TensorType:
    """
    Apply small perturbation on inputs to make the in- and out- distribution
    samples more separable.
    See original paper for more information (section 2.2)
    https://arxiv.org/abs/1807.03888

    Args:
        inputs (TensorType): input samples

    Returns:
        TensorType: Perturbed inputs
    """

    def __loss_fn(inputs: TensorType) -> TensorType:
        """
        Loss function for the input perturbation.

        Args:
            inputs (TensorType): input samples

        Returns:
            TensorType: loss value
        """
        # extract features
        out_features, _ = self.feature_extractor.predict(inputs, detach=False)
        out_features = self.op.flatten(out_features[0])
        # get mahalanobis score for the class maximizing it
        gaussian_score = self._mahalanobis_score(out_features)
        log_probs_f = self.op.max(gaussian_score, dim=1)
        return self.op.mean(-log_probs_f)

    # compute gradient
    gradient = self.op.gradient(__loss_fn, inputs)
    gradient = self.op.sign(gradient)

    inputs_p = inputs - self.eps * gradient
    return inputs_p

_mahalanobis_score(out_features)

Mahalanobis distance-based confidence score. For each test sample, it computes the log of the probability densities of some observations (assuming a normal distribution) using the mahalanobis distance with respect to every class-conditional distributions.

Parameters:

Name Type Description Default
out_features TensorType

test samples features

 required

Returns:

Name Type Description
TensorType TensorType

confidence scores (conditionally to each class)
Source code in oodeel/methods/mahalanobis.py 
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def _mahalanobis_score(self, out_features: TensorType) -> TensorType:
    """
    Mahalanobis distance-based confidence score. For each test sample, it computes
    the log of the probability densities of some observations (assuming a
    normal distribution) using the mahalanobis distance with respect to every
    class-conditional distributions.

    Args:
        out_features (TensorType): test samples features

    Returns:
        TensorType: confidence scores (conditionally to each class)
    """
    gaussian_scores = list()
    # compute scores conditionally to each class
    for cls in self._classes:
        # center features wrt class-cond dist.
        mu = self._mus[cls]
        zero_f = out_features - mu
        # gaussian log prob density (mahalanobis)
        log_probs_f = -0.5 * self.op.diag(
            self.op.matmul(
                self.op.matmul(zero_f, self._pinv_cov), self.op.t(zero_f)
            )
        )
        gaussian_scores.append(self.op.reshape(log_probs_f, (-1, 1)))
    # concatenate scores
    gaussian_score = self.op.cat(gaussian_scores, 1)
    return gaussian_score

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on the mahalanobis distance with respect to the closest class-conditional Gaussian distribution.

Parameters:

Name Type Description Default
inputs TensorType

input samples

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/mahalanobis.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes an OOD score for input samples "inputs" based on the mahalanobis
    distance with respect to the closest class-conditional Gaussian distribution.

    Args:
        inputs (TensorType): input samples

    Returns:
        Tuple[np.ndarray]: scores, logits
    """
    # input preprocessing (perturbation)
    if self.eps > 0:
        inputs_p = self._input_perturbation(inputs)
    else:
        inputs_p = inputs

    # mahalanobis score on perturbed inputs
    features_p, _ = self.feature_extractor.predict_tensor(inputs_p)
    features_p = self.op.flatten(features_p[0])
    gaussian_score_p = self._mahalanobis_score(features_p)

    # take the highest score for each sample
    gaussian_score_p = self.op.max(gaussian_score_p, dim=1)
    return -self.op.convert_to_numpy(gaussian_score_p)

ODIN

Bases: OODBaseDetector

"Enhancing The Reliability of Out-of-distribution Image Detection in Neural Networks" http://arxiv.org/abs/1706.02690

Parameters:

Name Type Description Default
temperature float

Temperature parameter. Defaults to 1000.

 1000
noise float

Perturbation noise. Defaults to 0.014.

 0.014
use_react bool

if true, apply ReAct method by clipping penultimate activations under a threshold value.

 False
react_quantile Optional[float]

q value in the range [0, 1] used to compute the react clipping threshold defined as the q-th quantile penultimate layer activations. Defaults to 0.8.

 0.8
Source code in oodeel/methods/odin.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
class ODIN(OODBaseDetector):
    """ "Enhancing The Reliability of Out-of-distribution Image Detection
    in Neural Networks"
    http://arxiv.org/abs/1706.02690

    Args:
        temperature (float, optional): Temperature parameter. Defaults to 1000.
        noise (float, optional): Perturbation noise. Defaults to 0.014.
        use_react (bool): if true, apply ReAct method by clipping penultimate
            activations under a threshold value.
        react_quantile (Optional[float]): q value in the range [0, 1] used to compute
            the react clipping threshold defined as the q-th quantile penultimate layer
            activations. Defaults to 0.8.
    """

    def __init__(
        self,
        temperature: float = 1000,
        noise: float = 0.014,
        use_react: bool = False,
        react_quantile: float = 0.8,
    ):
        self.temperature = temperature
        super().__init__(
            use_react=use_react,
            react_quantile=react_quantile,
        )
        self.noise = noise

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes an OOD score for input samples "inputs" based on
        the distance to nearest neighbors in the feature space of self.model

        Args:
            inputs (TensorType): input samples to score

        Returns:
            Tuple[np.ndarray]: scores, logits
        """
        if self.feature_extractor.backend == "torch":
            inputs = inputs.to(self.feature_extractor._device)
        x = self.input_perturbation(inputs)
        _, logits = self.feature_extractor.predict_tensor(x)
        logits_s = logits / self.temperature
        probits = self.op.softmax(logits_s)
        probits = self.op.convert_to_numpy(probits)
        scores = -np.max(probits, axis=1)
        return scores

    def input_perturbation(self, inputs: TensorType) -> TensorType:
        """Apply a small perturbation over inputs to increase their softmax score.
        See ODIN paper for more information (section 3):
        http://arxiv.org/abs/1706.02690

        Args:
            inputs (TensorType): input samples to score

        Returns:
            TensorType: Perturbed inputs
        """
        preds = self.feature_extractor.model(inputs)
        outputs = self.op.argmax(preds, dim=1)
        gradients = self.op.gradient(self._temperature_loss, inputs, outputs)
        inputs_p = inputs - self.noise * self.op.sign(gradients)
        return inputs_p

    def _temperature_loss(self, inputs: TensorType, labels: TensorType) -> TensorType:
        """Compute the tempered cross-entropy loss.

        Args:
            inputs (TensorType): the inputs of the model.
            labels (TensorType): the labels to fit on.

        Returns:
            TensorType: the cross-entropy loss.
        """
        preds = self.feature_extractor.model(inputs) / self.temperature
        loss = self.op.CrossEntropyLoss(reduction="sum")(inputs=preds, targets=labels)
        return loss

    def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
        """
        Fits the OOD detector to fit_dataset.

        Args:
            fit_dataset: dataset to fit the OOD detector on
        """
        pass

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return False

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return False

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_fit_to_dataset(fit_dataset)

Fits the OOD detector to fit_dataset.

Parameters:

Name Type Description Default
fit_dataset DatasetType

dataset to fit the OOD detector on

 required
Source code in oodeel/methods/odin.py 
112
113
114
115
116
117
118
119
def _fit_to_dataset(self, fit_dataset: DatasetType) -> None:
    """
    Fits the OOD detector to fit_dataset.

    Args:
        fit_dataset: dataset to fit the OOD detector on
    """
    pass

_score_tensor(inputs)

Computes an OOD score for input samples "inputs" based on the distance to nearest neighbors in the feature space of self.model

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/odin.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes an OOD score for input samples "inputs" based on
    the distance to nearest neighbors in the feature space of self.model

    Args:
        inputs (TensorType): input samples to score

    Returns:
        Tuple[np.ndarray]: scores, logits
    """
    if self.feature_extractor.backend == "torch":
        inputs = inputs.to(self.feature_extractor._device)
    x = self.input_perturbation(inputs)
    _, logits = self.feature_extractor.predict_tensor(x)
    logits_s = logits / self.temperature
    probits = self.op.softmax(logits_s)
    probits = self.op.convert_to_numpy(probits)
    scores = -np.max(probits, axis=1)
    return scores

_temperature_loss(inputs, labels)

Compute the tempered cross-entropy loss.

Parameters:

Name Type Description Default
inputs TensorType

the inputs of the model.

 required
labels TensorType

the labels to fit on.

 required

Returns:

Name Type Description
TensorType TensorType

the cross-entropy loss.
Source code in oodeel/methods/odin.py 
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def _temperature_loss(self, inputs: TensorType, labels: TensorType) -> TensorType:
    """Compute the tempered cross-entropy loss.

    Args:
        inputs (TensorType): the inputs of the model.
        labels (TensorType): the labels to fit on.

    Returns:
        TensorType: the cross-entropy loss.
    """
    preds = self.feature_extractor.model(inputs) / self.temperature
    loss = self.op.CrossEntropyLoss(reduction="sum")(inputs=preds, targets=labels)
    return loss

input_perturbation(inputs)

Apply a small perturbation over inputs to increase their softmax score. See ODIN paper for more information (section 3): http://arxiv.org/abs/1706.02690

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Name Type Description
TensorType TensorType

Perturbed inputs
Source code in oodeel/methods/odin.py 
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
def input_perturbation(self, inputs: TensorType) -> TensorType:
    """Apply a small perturbation over inputs to increase their softmax score.
    See ODIN paper for more information (section 3):
    http://arxiv.org/abs/1706.02690

    Args:
        inputs (TensorType): input samples to score

    Returns:
        TensorType: Perturbed inputs
    """
    preds = self.feature_extractor.model(inputs)
    outputs = self.op.argmax(preds, dim=1)
    gradients = self.op.gradient(self._temperature_loss, inputs, outputs)
    inputs_p = inputs - self.noise * self.op.sign(gradients)
    return inputs_p

VIM

Bases: OODBaseDetector

Compute the Virtual Matching Logit (VIM) score. https://arxiv.org/abs/2203.10807

This score combines the energy score with a PCA residual score.

The energy score is the logarithm of the sum of exponential of logits. The PCA residual score is based on the projection on residual dimensions for principal component analysis. Residual dimensions are the eigenvectors corresponding to the least eignevalues (least variance). Intuitively, this score method assumes that feature representations of ID data occupy a low dimensional affine subspace $P+c$ of the feature space. Specifically, the projection of ID data translated by $-c$ on the orthognoal complement $P^{\perp}$ is expected to have small norm. It allows to detect points whose feature representation lie far from the identified affine subspace, namely those points $x$ such that the projection on $P^{\perp}$ of $x-c$ has large norm.

Parameters:

Name Type Description Default
princ_dims Union[int, float]

number of principal dimensions of in distribution features to consider. If an int, must be less than the dimension of the feature space. If a float, it must be in [0,1), it represents the ratio of explained variance to consider to determine the number of principal components. Defaults to 0.99.

 0.99
pca_origin str

either "pseudo" for using $W^{-1}b$ where $W^{-1}$ is the pseudo inverse of the final linear layer applied to bias term (as in the VIM paper), or "center" for using the mean of the data in feature space. Defaults to "center".

 'pseudo'
Source code in oodeel/methods/vim.py 
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
class VIM(OODBaseDetector):
    """
    Compute the Virtual Matching Logit (VIM) score.
    https://arxiv.org/abs/2203.10807

    This score combines the energy score with a PCA residual score.

    The energy score is the logarithm of the sum of exponential of logits.
    The PCA residual score is based on the projection on residual dimensions for
    principal component analysis.
        Residual dimensions are the eigenvectors corresponding to the least eignevalues
        (least variance).
        Intuitively, this score method assumes that feature representations of ID data
        occupy a low dimensional affine subspace $P+c$ of the feature space.
        Specifically, the projection of ID data translated by $-c$ on the
        orthognoal complement $P^{\\perp}$ is expected to have small norm.
        It allows to detect points whose feature representation lie far from the
        identified affine subspace, namely those points $x$ such that the
        projection on $P^{\\perp}$ of $x-c$ has large norm.

    Args:
        princ_dims (Union[int, float]): number of principal dimensions of in
            distribution features to consider. If an int, must be less than the
            dimension of the feature space.
            If a float, it must be in [0,1), it represents the ratio of
            explained variance to consider to determine the number of principal
            components. Defaults to 0.99.
        pca_origin (str): either "pseudo" for using $W^{-1}b$ where $W^{-1}$ is
            the pseudo inverse of the final linear layer applied to bias term
            (as in the VIM paper), or "center" for using the mean of the data in
            feature space. Defaults to "center".
    """

    def __init__(
        self,
        princ_dims: Union[int, float] = 0.99,
        pca_origin: str = "pseudo",
    ):
        super().__init__()
        self._princ_dim = princ_dims
        self.pca_origin = pca_origin

    def _fit_to_dataset(self, fit_dataset: Union[TensorType, DatasetType]) -> None:
        """
        Computes principal components of feature representations and store the residual
        eigenvectors.
        Computes a scaling factor constant :math:'\alpha' such that the average scaled
        residual score (on train) is equal to the average maximum logit score (MLS)
        score.

        Args:
            fit_dataset: input dataset (ID) to construct the index with.
        """
        # extract features from fit dataset
        all_features_train, info = self.feature_extractor.predict(fit_dataset)
        features_train = all_features_train[0]
        logits_train = info["logits"]
        features_train = self.op.flatten(features_train)
        self.feature_dim = features_train.shape[1]
        logits_train = self.op.convert_to_numpy(logits_train)

        # get distribution center for pca projection
        if self.pca_origin == "center":
            self.center = self.op.mean(features_train, dim=0)
        elif self.pca_origin == "pseudo":
            # W, b = self.feature_extractor.get_weights(
            #    self.feature_extractor.feature_layers_id[0]
            # )
            W, b = self.feature_extractor.get_weights(-1)
            W, b = self.op.from_numpy(W), self.op.from_numpy(b.reshape(-1, 1))
            _W = self.op.t(W) if self.backend == "tensorflow" else W
            self.center = -self.op.reshape(self.op.matmul(self.op.pinv(_W), b), (-1,))
        else:
            raise NotImplementedError(
                'only "center" and "pseudo" are available for argument "pca_origin"'
            )

        # compute eigvalues and eigvectors of empirical covariance matrix
        centered_features = features_train - self.center
        emp_cov = (
            self.op.matmul(self.op.t(centered_features), centered_features)
            / centered_features.shape[0]
        )
        eig_vals, eigen_vectors = self.op.eigh(emp_cov)
        self.eig_vals = self.op.convert_to_numpy(eig_vals)

        # get number of residual dims for pca projection
        if isinstance(self._princ_dim, int):
            assert self._princ_dim < self.feature_dim, (
                f"if 'princ_dims'(={self._princ_dim}) is an int, it must be less than "
                "feature space dimension ={self.feature_dim})"
            )
            self.res_dim = self.feature_dim - self._princ_dim
            self._princ_dim = self._princ_dim
        elif isinstance(self._princ_dim, float):
            assert (
                0 <= self._princ_dim and self._princ_dim < 1
            ), f"if 'princ_dims'(={self._princ_dim}) is a float, it must be in [0,1)"
            explained_variance = np.cumsum(
                np.flip(self.eig_vals) / np.sum(self.eig_vals)
            )
            self._princ_dim = np.where(explained_variance > self._princ_dim)[0][0]
            self.res_dim = self.feature_dim - self._princ_dim

        # projector on residual space
        self.res = eigen_vectors[:, : self.res_dim]  # asc. order with eigh

        # compute residual score on training data
        train_residual_scores = self._compute_residual_score_tensor(features_train)
        # compute MLS on training data
        train_mls_scores = np.max(logits_train, axis=-1)
        # compute scaling factor
        self.alpha = np.mean(train_mls_scores) / np.mean(train_residual_scores)

    def _compute_residual_score_tensor(self, features: TensorType) -> np.ndarray:
        """
        Computes the norm of the residual projection in the feature space.

        Args:
            features: input samples to score

        Returns:
            np.ndarray: scores
        """
        res_coordinates = self.op.matmul(features - self.center, self.res)
        # taking the norm of the coordinates, which amounts to the norm of
        # the projection since the eigenvectors form an orthornomal basis
        res_norm = self.op.norm(res_coordinates, dim=-1)
        return self.op.convert_to_numpy(res_norm)

    def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
        """
        Computes the VIM score for input samples "inputs" as the sum of the energy
        score and a scaled (PCA) residual norm in the feature space.

        Args:
            inputs: input samples to score

        Returns:
            Tuple[np.ndarray]: scores, logits
        """
        # extract features
        features, logits = self.feature_extractor.predict_tensor(inputs)
        features = self.op.flatten(features[0])
        # vim score
        res_scores = self._compute_residual_score_tensor(features)
        logits = self.op.convert_to_numpy(logits)
        energy_scores = logsumexp(logits, axis=-1)
        scores = -self.alpha * res_scores + energy_scores
        return -np.array(scores)

    def plot_spectrum(self) -> None:
        """
        Plot cumulated explained variance wrt the number of principal dimensions.
        """
        cumul_explained_variance = np.cumsum(self.eig_vals)[::-1]
        plt.plot(cumul_explained_variance / np.max(cumul_explained_variance))
        plt.axvline(
            x=self._princ_dim,
            color="r",
            linestyle="--",
            label=f"princ_dims = {self._princ_dim} ",
        )
        plt.legend()
        plt.ylabel("Residual explained variance")
        plt.xlabel("Number of principal dimensions")

    @property
    def requires_to_fit_dataset(self) -> bool:
        """
        Whether an OOD detector needs a `fit_dataset` argument in the fit function.

        Returns:
            bool: True if `fit_dataset` is required else False.
        """
        return True

    @property
    def requires_internal_features(self) -> bool:
        """
        Whether an OOD detector acts on internal model features.

        Returns:
            bool: True if the detector perform computations on an intermediate layer
            else False.
        """
        return True

requires_internal_features: bool property

Whether an OOD detector acts on internal model features.

Returns:

Name Type Description
bool bool

True if the detector perform computations on an intermediate layer
bool

else False.

requires_to_fit_dataset: bool property

Whether an OOD detector needs a fit_dataset argument in the fit function.

Returns:

Name Type Description
bool bool

True if fit_dataset is required else False.

_compute_residual_score_tensor(features)

Computes the norm of the residual projection in the feature space.

Parameters:

Name Type Description Default
features TensorType

input samples to score

 required

Returns:

Type Description
ndarray

np.ndarray: scores
Source code in oodeel/methods/vim.py 
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def _compute_residual_score_tensor(self, features: TensorType) -> np.ndarray:
    """
    Computes the norm of the residual projection in the feature space.

    Args:
        features: input samples to score

    Returns:
        np.ndarray: scores
    """
    res_coordinates = self.op.matmul(features - self.center, self.res)
    # taking the norm of the coordinates, which amounts to the norm of
    # the projection since the eigenvectors form an orthornomal basis
    res_norm = self.op.norm(res_coordinates, dim=-1)
    return self.op.convert_to_numpy(res_norm)

_fit_to_dataset(fit_dataset)

Computes principal components of feature representations and store the residual eigenvectors. Computes a scaling factor constant :math:'lpha' such that the average scaled residual score (on train) is equal to the average maximum logit score (MLS) score.

Parameters:

Name Type Description Default
fit_dataset Union[TensorType, DatasetType]

input dataset (ID) to construct the index with.

 required
Source code in oodeel/methods/vim.py 
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def _fit_to_dataset(self, fit_dataset: Union[TensorType, DatasetType]) -> None:
    """
    Computes principal components of feature representations and store the residual
    eigenvectors.
    Computes a scaling factor constant :math:'\alpha' such that the average scaled
    residual score (on train) is equal to the average maximum logit score (MLS)
    score.

    Args:
        fit_dataset: input dataset (ID) to construct the index with.
    """
    # extract features from fit dataset
    all_features_train, info = self.feature_extractor.predict(fit_dataset)
    features_train = all_features_train[0]
    logits_train = info["logits"]
    features_train = self.op.flatten(features_train)
    self.feature_dim = features_train.shape[1]
    logits_train = self.op.convert_to_numpy(logits_train)

    # get distribution center for pca projection
    if self.pca_origin == "center":
        self.center = self.op.mean(features_train, dim=0)
    elif self.pca_origin == "pseudo":
        # W, b = self.feature_extractor.get_weights(
        #    self.feature_extractor.feature_layers_id[0]
        # )
        W, b = self.feature_extractor.get_weights(-1)
        W, b = self.op.from_numpy(W), self.op.from_numpy(b.reshape(-1, 1))
        _W = self.op.t(W) if self.backend == "tensorflow" else W
        self.center = -self.op.reshape(self.op.matmul(self.op.pinv(_W), b), (-1,))
    else:
        raise NotImplementedError(
            'only "center" and "pseudo" are available for argument "pca_origin"'
        )

    # compute eigvalues and eigvectors of empirical covariance matrix
    centered_features = features_train - self.center
    emp_cov = (
        self.op.matmul(self.op.t(centered_features), centered_features)
        / centered_features.shape[0]
    )
    eig_vals, eigen_vectors = self.op.eigh(emp_cov)
    self.eig_vals = self.op.convert_to_numpy(eig_vals)

    # get number of residual dims for pca projection
    if isinstance(self._princ_dim, int):
        assert self._princ_dim < self.feature_dim, (
            f"if 'princ_dims'(={self._princ_dim}) is an int, it must be less than "
            "feature space dimension ={self.feature_dim})"
        )
        self.res_dim = self.feature_dim - self._princ_dim
        self._princ_dim = self._princ_dim
    elif isinstance(self._princ_dim, float):
        assert (
            0 <= self._princ_dim and self._princ_dim < 1
        ), f"if 'princ_dims'(={self._princ_dim}) is a float, it must be in [0,1)"
        explained_variance = np.cumsum(
            np.flip(self.eig_vals) / np.sum(self.eig_vals)
        )
        self._princ_dim = np.where(explained_variance > self._princ_dim)[0][0]
        self.res_dim = self.feature_dim - self._princ_dim

    # projector on residual space
    self.res = eigen_vectors[:, : self.res_dim]  # asc. order with eigh

    # compute residual score on training data
    train_residual_scores = self._compute_residual_score_tensor(features_train)
    # compute MLS on training data
    train_mls_scores = np.max(logits_train, axis=-1)
    # compute scaling factor
    self.alpha = np.mean(train_mls_scores) / np.mean(train_residual_scores)

_score_tensor(inputs)

Computes the VIM score for input samples "inputs" as the sum of the energy score and a scaled (PCA) residual norm in the feature space.

Parameters:

Name Type Description Default
inputs TensorType

input samples to score

 required

Returns:

Type Description
Tuple[ndarray]

Tuple[np.ndarray]: scores, logits
Source code in oodeel/methods/vim.py 
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
def _score_tensor(self, inputs: TensorType) -> Tuple[np.ndarray]:
    """
    Computes the VIM score for input samples "inputs" as the sum of the energy
    score and a scaled (PCA) residual norm in the feature space.

    Args:
        inputs: input samples to score

    Returns:
        Tuple[np.ndarray]: scores, logits
    """
    # extract features
    features, logits = self.feature_extractor.predict_tensor(inputs)
    features = self.op.flatten(features[0])
    # vim score
    res_scores = self._compute_residual_score_tensor(features)
    logits = self.op.convert_to_numpy(logits)
    energy_scores = logsumexp(logits, axis=-1)
    scores = -self.alpha * res_scores + energy_scores
    return -np.array(scores)

plot_spectrum()

Plot cumulated explained variance wrt the number of principal dimensions.

Source code in oodeel/methods/vim.py 
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
def plot_spectrum(self) -> None:
    """
    Plot cumulated explained variance wrt the number of principal dimensions.
    """
    cumul_explained_variance = np.cumsum(self.eig_vals)[::-1]
    plt.plot(cumul_explained_variance / np.max(cumul_explained_variance))
    plt.axvline(
        x=self._princ_dim,
        color="r",
        linestyle="--",
        label=f"princ_dims = {self._princ_dim} ",
    )
    plt.legend()
    plt.ylabel("Residual explained variance")
    plt.xlabel("Number of principal dimensions")