Definitions for classification

$X$ is the predicted value, $Y$ is the actual value, $n$ is the number of samples, and $c$ is the number of classes.

for class $j$ :

\begin{align*} TP_j &= \sum\limits_{i=1}^n \mathbb{1}(X_i = j, Y_i = j)&&\text{対象クラス}j\text{をクラス}j\text{と予測した数}\\ FP_j &= \sum\limits_{i=1}^n \mathbb{1}(X_i = j, Y_i \neq j)&&\text{対象クラス}j\text{以外をクラス}j\text{と予測した数}\\ FN_j &= \sum\limits_{i=1}^n \mathbb{1}(X_i \neq j, Y_i = j)&&\text{対象クラス}j\text{をクラス}j\text{以外と予測した数}\\ TN_j &= \sum\limits_{i=1}^n \mathbb{1}(X_i \neq j, Y_i \neq j)&&\text{対象クラス}j\text{以外をクラス}j\text{以外と予測した数} \end{align*}

kage1020

Accuracy

\text{Accuracy}=\text{Accuracy}_j = \frac{TP_j + TN_j}{TP_j + FP_j + FN_j + TN_j}=\frac{\sum\limits_{i=1}^n \mathbb{1}(X_i=Y_i)}{n}=\frac{\text{correct frames}}{\text{all frames}}

def accuracy(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    return np.mean(y_true == y_pred)

kage1020

Precision

\begin{align*} \text{Precision}_j &= \frac{TP_j}{TP_j + FP_j}=\frac{\text{correct frames as class }j}{\text{all frames predicted as class }j}\\ \text{Macro Precision} &= \frac{1}{c}\sum\limits_{j=1}^c \text{Precision}_j\\ \text{Micro Precision} &= \frac{\sum\limits_{j=1}^c TP_j}{\sum\limits_{j=1}^c TP_j + \sum\limits_{j=1}^c FP_j}=\frac{\sum\limits_{i=1}^n\mathbb{1}(X_i=Y_i)}{n}=\text{Accuracy}\\ \end{align*}

def macro_precision(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    return np.mean([
        np.sum((y_true == y_pred) & (y_pred == c)) / (np.sum(y_pred == c) + 1e-10)
        for c in np.unique(y_true) 
    ])

def micro_precision(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    unique_classes = np.unique(y_true)
    tp = np.array([np.sum((y_true == y_pred) & (y_true == c)) for c in unique_classes])
    fp = np.array([np.sum((y_true != y_pred) & (y_pred == c)) for c in unique_classes])
    return np.sum(tp) / (np.sum(tp) + np.sum(fp) + 1e-10)

kage1020

Recall

\begin{align*} \text{Recall}_j &= \frac{TP_j}{TP_j + FN_j}=\frac{\text{correct frames as class }j}{\text{all frames of class }j}\\ \text{Macro Recall} &= \frac{1}{c}\sum\limits_{j=1}^c \text{Recall}_j\\ \text{Micro Recall} &= \frac{\sum\limits_{j=1}^c TP_j}{\sum\limits_{j=1}^c TP_j + \sum\limits_{j=1}^c FN_j}=\frac{\sum\limits_{i=1}^n\mathbb{1}(X_i=Y_i)}{n}=\text{Accuracy}\\ \end{align*}

def macro_recall(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    return np.mean([
        np.sum((y_true == y_pred) & (y_true == c)) / (np.sum(y_true == c) + 1e-10)
        for c in np.unique(y_true) 
    ])

def micro_recall(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    unique_classes = np.unique(y_true)
    tp = np.array([np.sum((y_true == y_pred) & (y_true == c)) for c in unique_classes])
    fn = np.array([np.sum((y_true != y_pred) & (y_true == c)) for c in unique_classes])
    return np.sum(tp) / (np.sum(tp) + np.sum(fn) + 1e-10)

kage1020

F1

\begin{align*} \text{F1}_j &= \frac{2\cdot\text{Precision}_j\cdot\text{Recall}_j}{\text{Precision}_j + \text{Recall}_j}=\frac{2\cdot\text{TP}_j}{2\cdot\text{TP}_j + \text{FP}_j + \text{FN}_j}\\ \text{Macro F1} &= \frac{1}{c}\sum\limits_{j=1}^c \text{F1}_j\\ \text{Micro F1} &= \frac{2\cdot\sum\limits_{j=1}^c \text{TP}_j}{2\cdot\sum\limits_{j=1}^c \text{TP}_j + \sum\limits_{j=1}^c \text{FP}_j + \sum\limits_{j=1}^c \text{FN}_j}=\frac{2\sum\limits_{i=1}^n\mathbb{1}(X_i=Y_i)}{2n}=\text{Accuracy}\\ \end{align*}

def macro_f1(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    return np.mean([
        2*np.sum((y_true==y_pred)&(y_true==c)) / (2*np.sum((y_true==y_pred)&(y_true==c)) + np.sum((y_true!=y_pred)&(y_pred==c)) + np.sum((y_true!=y_pred)&(y_true==c)) + 1e-10) 
        for c in np.unique(y_true) 
    ])
    # or
    return np.mean([
        2 * macro_precision(y_true, y_pred) * macro_recall(y_true, y_pred) / (macro_precision(y_true, y_pred) + macro_recall(y_true, y_pred) + 1e-10)
        for c in np.unique(y_true) 
    ])

def micro_f1(y_true: np.ndarray, y_pred: np.ndarray):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    unique_classes = np.unique(y_true)
    tp = np.array([np.sum((y_true == y_pred) & (y_true == c)) for c in unique_classes])
    fp = np.array([np.sum((y_true != y_pred) & (y_pred == c)) for c in unique_classes])
    fn = np.array([np.sum((y_true != y_pred) & (y_true == c)) for c in unique_classes])
    return np.sum(2 * tp) / (np.sum(2 * tp) + np.sum(fp) + np.sum(fn) + 1e-10)
    # or
    precision = micro_precision(y_true, y_pred)
    recall = micro_recall(y_true, y_pred)
    return 2 * precision * recall / (precision + recall + 1e-10)

kage1020

Definitions for action segmentation

$S$ is the predicted segments, $T$ is the actual segments, $m$ is the number of segments, and $n$ is the number of samples in $m$ segment.

for segment $k$ which has class $j$ in $S$ :

\begin{align*} TP_\tau &= \sum\limits_{k=1}^m\left(\frac{\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} = j, T_{k,i} = j)}{\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} = j, T_{k,i} = j)+\sum\limits_{i=1}^n\mathbb{1}(S_{k,i}=j,T_{k,i}\neq 1)+\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} \neq j, T_{k,i} = j)}>\tau\right)\\ FP_\tau &= \sum\limits_{k=1}^m\left(\frac{\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} = j, T_{k,i} \neq j)}{\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} = j, T_{k,i} = j)+\sum\limits_{i=1}^n\mathbb{1}(S_{k,i}=j,T_{k,i}\neq 1)+\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} \neq j, T_{k,i} = j)}>\tau\right)\\ FN_\tau &= \sum\limits_{k=1}^m\left(\frac{\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} \neq j, T_{k,i} = j)}{\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} = j, T_{k,i} = j)+\sum\limits_{i=1}^n\mathbb{1}(S_{k,i}=j,T_{k,i}\neq 1)+\sum\limits_{i=1}^n \mathbb{1}(S_{k,i} \neq j, T_{k,i} = j)}>\tau\right)\\ \end{align*}

kage1020

IoU

IoU_{k} =\frac{\sum\limits_{i=1}^n\mathrm{index}(S_{k,i})=\mathrm{index}(T_{k,i})}{\sum\limits_{i=1}^n\mathrm{index}(S_{k,i})\cup\sum\limits_{i=1}^n\mathrm{index}(T_{k,i})}

def iou(y_true: Tuple[int, int], y_pred: Tuple[int, int]):
    '''
    y_true: (2,)
    y_pred: (2,)
    '''
    intersection = max(0, min(y_true[1], y_pred[1]) - max(y_true[0], y_pred[0]))
    union = max(y_true[1], y_pred[1]) - min(y_true[0], y_pred[0])
    return intersection / union

kage1020

Precision@ $\tau$

\text{Precision@}\tau = \frac{\sum\limits_{k=1}^m TP_{k,\tau}}{\sum\limits_{k=1}^m TP_{k,\tau} + \sum\limits_{k=1}^m FP_{k,\tau}}

def labels_segments(labels: np.ndarray):
    '''
    labels: (n,)
    '''
    change_indices = np.where(np.diff(labels, prepend=0) != 0)[0]
    segments = []

    for i in range(len(change_indices)):
        start = change_indices[i]
        end = change_indices[i + 1] if i + 1 < len(change_indices) else len(labels)
        segment = [labels[start], (start, end)]
        segments.append(segment)
    return segments

def precision_tau(y_true: np.ndarray, y_pred: np.ndarray, tau: float = 0):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    segments_true = labels_segments(y_true)
    segments_pred = labels_segments(y_pred)
    tp = 0
    fp = 0
    for segment_pred in segments_pred:
        match_segments = [segment_true for segment_true in segments_true if segment_pred[0] == segment_true[0]]
        ious = [iou(segment_pred[1], segment_true[1]) for segment_true in match_segments]
        if len(ious) > 0 and max(ious) > tau:
            tp += 1
        else:
            fp += 1
    return tp / (tp + fp + 1e-10)

kage1020

Recall@ $\tau$

\text{Recall@}\tau = \frac{\sum\limits_{k=1}^m TP_{k,\tau}}{\sum\limits_{k=1}^m TP_{k,\tau} + \sum\limits_{k=1}^m FN_{k,\tau}}

def recall_tau(y_true: np.ndarray, y_pred: np.ndarray, tau: float = 0):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    segments_true = labels_segments(y_true)
    segments_pred = labels_segments(y_pred)
    tp = 0
    fn = 0
    for segment_true in segments_true:
        match_segments = [segment_pred for segment_pred in segments_pred if segment_true[0] == segment_pred[0]]
        ious = [iou(segment_pred[1], segment_true[1]) for segment_pred in match_segments]
        if len(ious) > 0 and max(ious) > tau:
            tp += 1
        else:
            fn += 1
    return tp / (tp + fn + 1e-10)

kage1020

F1@ $\tau$

\text{F1@}\tau = \frac{2\cdot\text{Precision@}\tau\cdot\text{Recall@}\tau}{\text{Precision@}\tau + \text{Recall@}\tau}

def f1_tau(y_true: np.ndarray, y_pred: np.ndarray, tau: float = 0):
    '''
    y_true: (n,)
    y_pred: (n,)
    '''
    precision = precision_tau(y_true, y_pred, tau)
    recall = recall_tau(y_true, y_pred, tau)
    return 2 * precision * recall / (precision + recall + 1e-10)

Definitions for classification

Accuracy

Precision

Recall

F1

Definitions for action segmentation

IoU

Precision@\tau

Recall@\tau

F1@\tau

Precision@ $\tau$

Recall@ $\tau$

F1@ $\tau$