Adding custom metrics

In addition to the built-in metrics, you can define your own custom metrics register a new metric to the Scorer. This is useful for implementing domain-specific metrics that are not included in the framework by default.

Register the metric using registerMetric(name, metricFunction, combineFunction, biggerIsBetter, initValue) for the Scorer class. This specifiy the name for configuration, the metricFunction that computes the metric on a single record, the combineFunction that aggregates metric values across records (e.g. np.mean), whether bigger values are better (biggerIsBetter), and an initValue for the metric (e.g. 0.0 for accuracy, float('inf') for loss):

from SleePyPhases.MultiScorer import MultiScorer

from src.models.MSED.functions.metrics import f1_function
from src.models.MSED.functions import binary_to_array

import numpy as np

class Init(Phase):
    def prepareConfig(self):
        # model = ModelManager.getModel()
        def recordWiseLoss(truth, prediction):
            prediction, prediction_array = prediction
            return MSEDLoss()(truth, prediction)

        def batchLoss(truth, prediction):
            prediction, prediction_array = prediction
            return MSEDLoss()(truth, prediction)

        MultiScorer.registerMetric(
            "loss", 
            metricFunction=recordWiseLoss, 
            combineFunction=batchLoss, 
            biggerIsBetter=False, 
            initValue=10,
        )

        def recordWiseF1(truth, prediction):
            metric = f1_function()
            truth_events = binary_to_array(truth)
            predictions_events = binary_to_array(prediction.reshape(-1))
            if len(truth_events) == 0:
                return 1.0 if len(predictions_events) == 0 else 0.0
            return metric(truth_events, predictions_events)

        MultiScorer.registerMetric(
            "f1_iou", 
            metricFunction=recordWiseF1, 
            combineFunction=np.mean, 
            biggerIsBetter=True,
            initValue=0.0,
        )