from collections import defaultdict
import matplotlib.pyplot as plt
from matplotlib.cm import get_cmap
import numpy as np
from matplotlib import cm
from scipy.stats import ttest_ind_from_stats
from matplotlib.ticker import ScalarFormatter
import math
import quapy as qp
plt.rcParams['figure.figsize'] = [10, 6]
plt.rcParams['figure.dpi'] = 200
plt.rcParams['font.size'] = 18
[docs]
def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True,
train_prev=None, savepath=None, method_order=None):
"""
The diagonal plot displays the predicted prevalence values (along the y-axis) as a function of the true prevalence
values (along the x-axis). The optimal quantifier is described by the diagonal (0,0)-(1,1) of the plot (hence the
name). It is convenient for binary quantification problems, though it can be used for multiclass problems by
indicating which class is to be taken as the positive class. (For multiclass quantification problems, other plots
like the :meth:`error_by_drift` might be preferable though).
:param method_names: array-like with the method names for each experiment
:param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
each experiment
:param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
for each experiment
:param pos_class: index of the positive class
:param title: the title to be displayed in the plot
:param show_std: whether or not to show standard deviations (represented by color bands). This might be inconvenient
for cases in which many methods are compared, or when the standard deviations are high -- default True)
:param legend: whether or not to display the leyend (default True)
:param train_prev: if indicated (default is None), the training prevalence (for the positive class) is hightlighted
in the plot. This is convenient when all the experiments have been conducted in the same dataset.
:param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
:param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
listed in the legend and associated with matplotlib colors).
"""
fig, ax = plt.subplots()
ax.set_aspect('equal')
ax.grid()
ax.plot([0, 1], [0, 1], '--k', label='ideal', zorder=1)
method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs)
order = list(zip(method_names, true_prevs, estim_prevs))
if method_order is not None:
table = {method_name:[true_prev, estim_prev] for method_name, true_prev, estim_prev in order}
order = [(method_name, *table[method_name]) for method_name in method_order]
NUM_COLORS = len(method_names)
if NUM_COLORS>10:
cm = plt.get_cmap('tab20')
ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)])
for method, true_prev, estim_prev in order:
true_prev = true_prev[:,pos_class]
estim_prev = estim_prev[:,pos_class]
x_ticks = np.unique(true_prev)
x_ticks.sort()
y_ave = np.asarray([estim_prev[true_prev == x].mean() for x in x_ticks])
y_std = np.asarray([estim_prev[true_prev == x].std() for x in x_ticks])
ax.errorbar(x_ticks, y_ave, fmt='-', marker='o', label=method, markersize=3, zorder=2)
if show_std:
ax.fill_between(x_ticks, y_ave - y_std, y_ave + y_std, alpha=0.25)
if train_prev is not None:
train_prev = train_prev[pos_class]
ax.scatter(train_prev, train_prev, c='c', label='tr-prev', linewidth=2, edgecolor='k', s=100, zorder=3)
ax.set(xlabel='true prevalence', ylabel='estimated prevalence', title=title)
ax.set_ylim(0, 1)
ax.set_xlim(0, 1)
if legend:
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
# box = ax.get_position()
# ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
# ax.legend(loc='lower center',
# bbox_to_anchor=(1, -0.5),
# ncol=(len(method_names)+1)//2)
_save_or_show(savepath)
[docs]
def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None):
"""
Box-plots displaying the global bias (i.e., signed error computed as the estimated value minus the true value)
for each quantification method with respect to a given positive class.
:param method_names: array-like with the method names for each experiment
:param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
each experiment
:param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
for each experiment
:param pos_class: index of the positive class
:param title: the title to be displayed in the plot
:param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
"""
method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs)
fig, ax = plt.subplots()
ax.grid()
data, labels = [], []
for method, true_prev, estim_prev in zip(method_names, true_prevs, estim_prevs):
true_prev = true_prev[:,pos_class]
estim_prev = estim_prev[:,pos_class]
data.append(estim_prev-true_prev)
labels.append(method)
ax.boxplot(data, labels=labels, patch_artist=False, showmeans=True)
plt.xticks(rotation=45)
ax.set(ylabel='error bias', title=title)
_save_or_show(savepath)
[docs]
def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=cm.tab10,
vertical_xticks=False, legend=True, savepath=None):
"""
Box-plots displaying the local bias (i.e., signed error computed as the estimated value minus the true value)
for different bins of (true) prevalence of the positive classs, for each quantification method.
:param method_names: array-like with the method names for each experiment
:param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
each experiment
:param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
for each experiment
:param pos_class: index of the positive class
:param title: the title to be displayed in the plot
:param nbins: number of bins
:param colormap: the matplotlib colormap to use (default cm.tab10)
:param vertical_xticks: whether or not to add secondary grid (default is False)
:param legend: whether or not to display the legend (default is True)
:param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
"""
from pylab import boxplot, plot, setp
fig, ax = plt.subplots()
ax.grid()
method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs)
bins = np.linspace(0, 1, nbins+1)
binwidth = 1/nbins
data = {}
for method, true_prev, estim_prev in zip(method_names, true_prevs, estim_prevs):
true_prev = true_prev[:,pos_class]
estim_prev = estim_prev[:,pos_class]
data[method] = []
inds = np.digitize(true_prev, bins, right=True)
for ind in range(len(bins)):
selected = inds==ind
data[method].append(estim_prev[selected] - true_prev[selected])
nmethods = len(method_names)
boxwidth = binwidth/(nmethods+4)
for i,bin in enumerate(bins[:-1]):
boxdata = [data[method][i] for method in method_names]
positions = [bin+(i*boxwidth)+2*boxwidth for i,_ in enumerate(method_names)]
box = boxplot(boxdata, showmeans=False, positions=positions, widths = boxwidth, sym='+', patch_artist=True)
for boxid in range(len(method_names)):
c = colormap.colors[boxid%len(colormap.colors)]
setp(box['fliers'][boxid], color=c, marker='+', markersize=3., markeredgecolor=c)
setp(box['boxes'][boxid], color=c)
setp(box['medians'][boxid], color='k')
major_xticks_positions, minor_xticks_positions = [], []
major_xticks_labels, minor_xticks_labels = [], []
for i,b in enumerate(bins[:-1]):
major_xticks_positions.append(b)
minor_xticks_positions.append(b + binwidth / 2)
major_xticks_labels.append('')
minor_xticks_labels.append(f'[{bins[i]:.2f}-{bins[i + 1]:.2f})')
ax.set_xticks(major_xticks_positions)
ax.set_xticks(minor_xticks_positions, minor=True)
ax.set_xticklabels(major_xticks_labels)
ax.set_xticklabels(minor_xticks_labels, minor=True, rotation='vertical' if vertical_xticks else 'horizontal')
if vertical_xticks:
# Pad margins so that markers don't get clipped by the axes
plt.margins(0.2)
# Tweak spacing to prevent clipping of tick-labels
plt.subplots_adjust(bottom=0.15)
if legend:
# adds the legend to the list hs, initialized with the "ideal" quantifier (one that has 0 bias across all bins. i.e.
# a line from (0,0) to (1,0). The other elements are simply labelled dot-plots that are to be removed (setting
# set_visible to False for all but the first element) after the legend has been placed
hs=[ax.plot([0, 1], [0, 0], '-k', zorder=2)[0]]
for colorid in range(len(method_names)):
color=colormap.colors[colorid % len(colormap.colors)]
h, = plot([0, 0], '-s', markerfacecolor=color, color='k',mec=color, linewidth=1.)
hs.append(h)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend(hs, ['ideal']+method_names, loc='center left', bbox_to_anchor=(1, 0.5))
[h.set_visible(False) for h in hs[1:]]
# x-axis and y-axis labels and limits
ax.set(xlabel='prevalence', ylabel='error bias', title=title)
ax.set_xlim(0, 1)
_save_or_show(savepath)
[docs]
def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
n_bins=20, error_name='ae', show_std=False,
show_density=True,
show_legend=True,
logscale=False,
title=f'Quantification error as a function of distribution shift',
vlines=None,
method_order=None,
savepath=None):
"""
Plots the error (along the x-axis, as measured in terms of `error_name`) as a function of the train-test shift
(along the y-axis, as measured in terms of :meth:`quapy.error.ae`). This plot is useful especially for multiclass
problems, in which "diagonal plots" may be cumbersone, and in order to gain understanding about how methods
fare in different regions of the prior probability shift spectrum (e.g., in the low-shift regime vs. in the
high-shift regime).
:param method_names: array-like with the method names for each experiment
:param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
each experiment
:param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
for each experiment
:param tr_prevs: training prevalence of each experiment
:param n_bins: number of bins in which the y-axis is to be divided (default is 20)
:param error_name: a string representing the name of an error function (as defined in `quapy.error`, default is "ae")
:param show_std: whether or not to show standard deviations as color bands (default is False)
:param show_density: whether or not to display the distribution of experiments for each bin (default is True)
:param show_density: whether or not to display the legend of the chart (default is True)
:param logscale: whether or not to log-scale the y-error measure (default is False)
:param title: title of the plot (default is "Quantification error as a function of distribution shift")
:param vlines: array-like list of values (default is None). If indicated, highlights some regions of the space
using vertical dotted lines.
:param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
listed in the legend and associated with matplotlib colors).
:param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
"""
fig, ax = plt.subplots()
ax.grid()
x_error = qp.error.ae
y_error = getattr(qp.error, error_name)
# get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same
# order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to
# x_error function) and 'y' is the estim-test shift (computed as according to y_error)
data = _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order)
if method_order is None:
method_order = method_names
_set_colors(ax, n_methods=len(method_order))
bins = np.linspace(0, 1, n_bins+1)
binwidth = 1 / n_bins
min_x, max_x, min_y, max_y = None, None, None, None
npoints = np.zeros(len(bins), dtype=float)
for method in method_order:
tr_test_drifts = data[method]['x']
method_drifts = data[method]['y']
if logscale:
ax.set_yscale("log")
ax.yaxis.set_major_formatter(ScalarFormatter())
ax.yaxis.get_major_formatter().set_scientific(False)
ax.minorticks_off()
inds = np.digitize(tr_test_drifts, bins, right=True)
xs, ys, ystds = [], [], []
for p,ind in enumerate(range(len(bins))):
selected = inds==ind
if selected.sum() > 0:
xs.append(ind*binwidth-binwidth/2)
ys.append(np.mean(method_drifts[selected]))
ystds.append(np.std(method_drifts[selected]))
npoints[p] += len(method_drifts[selected])
xs = np.asarray(xs)
ys = np.asarray(ys)
ystds = np.asarray(ystds)
min_x_method, max_x_method, min_y_method, max_y_method = xs.min(), xs.max(), ys.min(), ys.max()
min_x = min_x_method if min_x is None or min_x_method < min_x else min_x
max_x = max_x_method if max_x is None or max_x_method > max_x else max_x
max_y = max_y_method if max_y is None or max_y_method > max_y else max_y
min_y = min_y_method if min_y is None or min_y_method < min_y else min_y
max_y = max_y_method if max_y is None or max_y_method > max_y else max_y
ax.errorbar(xs, ys, fmt='-', marker='o', color='w', markersize=8, linewidth=4, zorder=1)
ax.errorbar(xs, ys, fmt='-', marker='o', label=method, markersize=6, linewidth=2, zorder=2)
if show_std:
ax.fill_between(xs, ys-ystds, ys+ystds, alpha=0.25)
if show_density:
ax2 = ax.twinx()
densities = npoints/np.sum(npoints)
ax2.bar([ind * binwidth-binwidth/2 for ind in range(len(bins))],
densities, alpha=0.15, color='g', width=binwidth, label='density')
ax2.set_ylim(0,max(densities))
ax2.spines['right'].set_color('g')
ax2.tick_params(axis='y', colors='g')
ax.set(xlabel=f'Distribution shift between training set and test sample',
ylabel=f'{error_name.upper()} (true distribution, predicted distribution)',
title=title)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
if vlines:
for vline in vlines:
ax.axvline(vline, 0, 1, linestyle='--', color='k')
ax.set_xlim(min_x, max_x)
if logscale:
#nice scale for the logaritmic axis
ax.set_ylim(0,10 ** math.ceil(math.log10(max_y)))
if show_legend:
fig.legend(loc='lower center',
bbox_to_anchor=(1, 0.5),
ncol=(len(method_names)+1)//2)
_save_or_show(savepath)
[docs]
def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs,
n_bins=20, binning='isomerous',
x_error='ae', y_error='ae', ttest_alpha=0.005, tail_density_threshold=0.005,
method_order=None,
savepath=None):
"""
Displays (only) the top performing methods for different regions of the train-test shift in form of a broken
bar chart, in which each method has bars only for those regions in which either one of the following conditions
hold: (i) it is the best method (in average) for the bin, or (ii) it is not statistically significantly different
(in average) as according to a two-sided t-test on independent samples at confidence `ttest_alpha`.
The binning can be made "isometric" (same size), or "isomerous" (same number of experiments -- default). A second
plot is displayed on top, that displays the distribution of experiments for each bin (when binning="isometric") or
the percentiles points of the distribution (when binning="isomerous").
:param method_names: array-like with the method names for each experiment
:param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for
each experiment
:param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components)
for each experiment
:param tr_prevs: training prevalence of each experiment
:param n_bins: number of bins in which the y-axis is to be divided (default is 20)
:param binning: type of binning, either "isomerous" (default) or "isometric"
:param x_error: a string representing the name of an error function (as defined in `quapy.error`) to be used for
measuring the amount of train-test shift (default is "ae")
:param y_error: a string representing the name of an error function (as defined in `quapy.error`) to be used for
measuring the amount of error in the prevalence estimations (default is "ae")
:param ttest_alpha: the confidence interval above which a p-value (two-sided t-test on independent samples) is
to be considered as an indicator that the two means are not statistically significantly different. Default is
0.005, meaning that a `p-value > 0.005` indicates the two methods involved are to be considered similar
:param tail_density_threshold: sets a threshold on the density of experiments (over the total number of experiments)
below which a bin in the tail (i.e., the right-most ones) will be discarded. This is in order to avoid some
bins to be shown for train-test outliers.
:param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e.,
listed in the legend and associated with matplotlib colors).
:param savepath: path where to save the plot. If not indicated (as default), the plot is shown.
:return:
"""
assert binning in ['isomerous', 'isometric'], 'unknown binning type; valid types are "isomerous" and "isometric"'
x_error = getattr(qp.error, x_error)
y_error = getattr(qp.error, y_error)
# get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same
# order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to
# x_error function) and 'y' is the estim-test shift (computed as according to y_error)
data = _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order)
if method_order is None:
method_order = method_names
if binning == 'isomerous':
# take bins containing the same amount of examples
tr_test_drifts = np.concatenate([data[m]['x'] for m in method_order])
bins = np.quantile(tr_test_drifts, q=np.linspace(0, 1, n_bins+1)).flatten()
else:
# take equidistant bins
bins = np.linspace(0, 1, n_bins+1)
bins[0] = -0.001
bins[-1] += 0.001
# we use this to keep track of how many datapoits contribute to each bin
inds_histogram_global = np.zeros(n_bins, dtype=float)
n_methods = len(method_order)
buckets = np.zeros(shape=(n_methods, n_bins, 3))
for i, method in enumerate(method_order):
tr_test_drifts = data[method]['x']
method_drifts = data[method]['y']
inds = np.digitize(tr_test_drifts, bins, right=False)
inds_histogram_global += np.histogram(tr_test_drifts, density=False, bins=bins)[0]
for j in range(len(bins)):
selected = inds == j
if selected.sum() > 0:
buckets[i, j-1, 0] = np.mean(method_drifts[selected])
buckets[i, j-1, 1] = np.std(method_drifts[selected])
buckets[i, j-1, 2] = selected.sum()
# cancel last buckets with low density
histogram = inds_histogram_global / inds_histogram_global.sum()
for tail in reversed(range(len(histogram))):
if histogram[tail] < tail_density_threshold:
buckets[:,tail,2] = 0
else:
break
salient_methods = set()
best_methods = []
for bucket in range(buckets.shape[1]):
nc = buckets[:, bucket, 2].sum()
if nc == 0:
best_methods.append([])
continue
order = np.argsort(buckets[:, bucket, 0])
rank1 = order[0]
best_bucket_methods = [method_order[rank1]]
best_mean, best_std, best_nc = buckets[rank1, bucket, :]
for method_index in order[1:]:
method_mean, method_std, method_nc = buckets[method_index, bucket, :]
_, pval = ttest_ind_from_stats(best_mean, best_std, best_nc, method_mean, method_std, method_nc)
if pval > ttest_alpha:
best_bucket_methods.append(method_order[method_index])
best_methods.append(best_bucket_methods)
salient_methods.update(best_bucket_methods)
print(best_bucket_methods)
if binning=='isomerous':
fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [0.2, 1]}, figsize=(20, len(salient_methods)))
else:
fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [1, 1]}, figsize=(20, len(salient_methods)))
ax = axes[1]
high_from = 0
yticks, yticks_method_names = [], []
color = get_cmap('Accent').colors
vlines = []
bar_high = 1
for method in [m for m in method_order if m in salient_methods]:
broken_paths = []
path_start, path_end = None, None
for i, best_bucket_methods in enumerate(best_methods):
if method in best_bucket_methods:
if path_start is None:
path_start = bins[i]
path_end = bins[i+1]-path_start
else:
path_end += bins[i+1]-bins[i]
else:
if path_start is not None:
broken_paths.append(tuple((path_start, path_end)))
path_start, path_end = None, None
if path_start is not None:
broken_paths.append(tuple((path_start, path_end)))
ax.broken_barh(broken_paths, (high_from, bar_high), facecolors=color[len(yticks_method_names)])
yticks.append(high_from+bar_high/2)
high_from += bar_high
yticks_method_names.append(method)
for path_start, path_end in broken_paths:
vlines.extend([path_start, path_start+path_end])
vlines = np.unique(vlines)
vlines = sorted(vlines)
for v in vlines[1:-1]:
ax.axvline(x=v, color='k', linestyle='--')
ax.set_ylim(0, high_from)
ax.set_xlim(vlines[0], vlines[-1])
ax.set_xlabel('Distribution shift between training set and sample')
ax.set_yticks(yticks)
ax.set_yticklabels(yticks_method_names)
# upper plot (explaining distribution)
ax = axes[0]
if binning == 'isometric':
# show the density for each region
bins[0]=0
y_pos = [b+(bins[i+1]-b)/2 for i,b in enumerate(bins[:-1]) if histogram[i]>0]
bar_width = [bins[i+1]-bins[i] for i in range(len(bins[:-1])) if histogram[i]>0]
ax.bar(y_pos, [n for n in histogram if n>0], bar_width, align='center', alpha=0.5, color='silver')
ax.set_ylabel('shift\ndistribution', rotation=0, ha='right', va='center')
ax.set_xlim(vlines[0], vlines[-1])
ax.get_xaxis().set_visible(False)
plt.subplots_adjust(wspace=0, hspace=0.1)
else:
# show the percentiles of the distribution
cumsum = np.cumsum(histogram)
for i in range(len(bins[:-1])):
start, width = bins[i], bins[i+1]-bins[i]
ax.broken_barh([tuple((start, width))], (0, 1), facecolors='whitesmoke' if i%2==0 else 'silver')
if i < len(bins)-2:
ax.text(bins[i+1], 0.5, '$P_{'+f'{int(np.round(cumsum[i]*100))}'+'}$', ha='center')
ax.set_ylim(0, 1)
ax.set_xlim(vlines[0], vlines[-1])
ax.get_yaxis().set_visible(False)
ax.get_xaxis().set_visible(False)
plt.subplots_adjust(wspace=0, hspace=0)
_save_or_show(savepath)
def _merge(method_names, true_prevs, estim_prevs):
ndims = true_prevs[0].shape[1]
data = defaultdict(lambda: {'true': np.empty(shape=(0, ndims)), 'estim': np.empty(shape=(0, ndims))})
method_order=[]
for method, true_prev, estim_prev in zip(method_names, true_prevs, estim_prevs):
data[method]['true'] = np.concatenate([data[method]['true'], true_prev])
data[method]['estim'] = np.concatenate([data[method]['estim'], estim_prev])
if method not in method_order:
method_order.append(method)
true_prevs_ = [data[m]['true'] for m in method_order]
estim_prevs_ = [data[m]['estim'] for m in method_order]
return method_order, true_prevs_, estim_prevs_
def _set_colors(ax, n_methods):
NUM_COLORS = n_methods
cm = plt.get_cmap('tab20')
ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)])
def _save_or_show(savepath):
# if savepath is specified, then saves the plot in that path; otherwise the plot is shown
if savepath is not None:
qp.util.create_parent_dir(savepath)
# plt.tight_layout()
plt.savefig(savepath, bbox_inches='tight')
else:
plt.show()
def _join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order):
data = defaultdict(lambda: {'x': np.empty(shape=(0)), 'y': np.empty(shape=(0))})
if method_order is None:
method_order = []
for method, test_prevs_i, estim_prevs_i, tr_prev_i in zip(method_names, true_prevs, estim_prevs, tr_prevs):
tr_prev_i = np.repeat(tr_prev_i.reshape(1, -1), repeats=test_prevs_i.shape[0], axis=0)
tr_test_drifts = x_error(test_prevs_i, tr_prev_i)
data[method]['x'] = np.concatenate([data[method]['x'], tr_test_drifts])
method_drifts = y_error(test_prevs_i, estim_prevs_i)
data[method]['y'] = np.concatenate([data[method]['y'], method_drifts])
if method not in method_order:
method_order.append(method)
return data