CompStats.interface

class Perf[source]

Perf is an entry point to CompStats

Parameters:

  • y_true (numpy.ndarray or pandas.DataFrame) – True measurement or could be a pandas.DataFrame where column label ‘y’ corresponds to the true measurement.

  • score_func (Function where the first argument is \(y\) and the second is \(\hat{y}.\)) – Function to measure the performance, it is assumed that the best algorithm has the highest value.

  • error_func (Function where the first argument is \(y\) and the second is \(\hat{y}.\)) – Function to measure the performance where the best algorithm has the lowest value.

  • y_pred (numpy.ndarray) – Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in args.

  • kwargs (numpy.ndarray) – Predictions, the algorithms will be identified using the keyword

  • n_jobs (int) – Number of jobs to compute the statistic, default=-1 corresponding to use all threads.

  • num_samples (int) – Number of bootstrap samples, default=500.

  • use_tqdm (bool) – Whether to use tqdm.tqdm to visualize the progress, default=True.

>>> from sklearn.svm import LinearSVC
>>> from sklearn.linear_model import LogisticRegression
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.base import clone
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> perf
<Perf>
Statistic with its standard error (se)
statistic (se)
0.9792 (0.0221) <= alg-1
0.9744 (0.0246) <= forest

If an algorithm’s prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm’s name can also be given with the keyword name.

>>> lr = LogisticRegression().fit(X_train, y_train)
>>> perf(lr.predict(X_val), name='Log. Reg.')
<Perf>
Statistic with its standard error (se)
statistic (se)
1.0000 (0.0000) <= Log. Reg.
0.9792 (0.0221) <= alg-1
0.9744 (0.0246) <= forest

The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.

>>> perf_error = clone(perf)
>>> perf_error.error_func = lambda y, hy: (y != hy).mean()
>>> perf_error
<Perf>
Statistic with its standard error (se)
statistic (se)
0.0000 (0.0000) <= Log. Reg.
0.0222 (0.0237) <= alg-1
0.0222 (0.0215) <= forest
__init__(y_true, *y_pred, score_func=<function balanced_accuracy_score>, error_func=None, num_samples: int = 500, n_jobs: int = -1, use_tqdm=True, **kwargs)[source]
get_params()[source]

Parameters

difference(wrt: str = None)[source]

Compute the difference w.r.t any algorithm by default is the best

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.base import clone
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> perf.difference()
<Difference>
difference p-values w.r.t alg-1
forest 0.06
property best

System with best performance

property sorting_func

Rank systems when multiple performances are used

property statistic

Statistic

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> perf.statistic
{'alg-1': 1.0, 'forest': 0.9500891265597148}
property se

Standard Error

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> perf.se
{'alg-1': 0.0, 'forest': 0.026945730782184187}
plot(value_name: str = None, var_name: str = 'Performance', alg_legend: str = 'Algorithm', perf_names: list = None, CI: float = 0.05, kind: str = 'point', linestyle: str = 'none', col_wrap: int = 3, capsize: float = 0.2, comparison: bool = True, right: bool = True, comp_legend: str = 'Comparison', winner_legend: str = 'Best', tie_legend: str = 'Equivalent', loser_legend: str = 'Different', **kwargs)[source]

plot with seaborn

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, score_func=None,
                error_func=lambda y, hy: (y != hy).mean(),
                forest=ens.predict(X_val))
>>> perf.plot()
dataframe(comparison: bool = False, right: bool = True, alpha: float = 0.05, value_name: str = 'Score', var_name: str = 'Performance', alg_legend: str = 'Algorithm', comp_legend: str = 'Comparison', winner_legend: str = 'Best', tie_legend: str = 'Equivalent', loser_legend: str = 'Different', perf_names: str = None)[source]

Dataframe

property n_jobs

Number of jobs to compute the statistics

property statistic_func

Statistic function

property statistic_samples

Statistic Samples

property num_samples

Number of bootstrap samples

property predictions

Predictions

property y_true

True output, gold standard o \(y\)

property score_func

Score function

property error_func

Error function

class Difference[source]

Difference

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.base import clone
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> diff = perf.difference()
>>> diff
<Difference>
difference p-values w.r.t alg-1
0.0780 <= forest
__init__(statistic_samples: StatisticSamples = None, statistic: dict = None, best: str = None) → None
property sorting_func

Rank systems when multiple performances are used

p_value(right: bool = True)[source]

Compute p_value of the differences

Parameters:

right (bool) – Estimate the p-value using \(\text{sample} \geq 2\delta\)

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.base import clone
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> diff = perf.difference()
>>> diff.p_value()
{'forest': np.float64(0.3)}
dataframe(value_name: str = 'Score', var_name: str = 'Best', alg_legend: str = 'Algorithm', sig_legend: str = 'Significant', perf_names: str = None, right: bool = True, alpha: float = 0.05)[source]

Dataframe

plot(value_name: str = 'Difference', var_name: str = 'Best', alg_legend: str = 'Algorithm', sig_legend: str = 'Significant', perf_names: list = None, alpha: float = 0.05, right: bool = True, kind: str = 'point', linestyle: str = 'none', col_wrap: int = 3, capsize: float = 0.2, set_refline: bool = True, **kwargs)[source]

Plot

>>> from sklearn.svm import LinearSVC
>>> from sklearn.ensemble import RandomForestClassifier
>>> from sklearn.datasets import load_iris
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.base import clone
>>> from CompStats.interface import Perf
>>> X, y = load_iris(return_X_y=True)
>>> _ = train_test_split(X, y, test_size=0.3)
>>> X_train, X_val, y_train, y_val = _
>>> m = LinearSVC().fit(X_train, y_train)
>>> hy = m.predict(X_val)
>>> ens = RandomForestClassifier().fit(X_train, y_train)
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
>>> diff = perf.difference()
>>> diff.plot()