From 1a3755eb58ee7e5c1713e7bb1eee054402ef07c2 Mon Sep 17 00:00:00 2001 From: Alex Moreo Date: Mon, 22 Nov 2021 18:10:48 +0100 Subject: [PATCH] adding documentation, adding brokenbar plots, merging plots from tweetsent with density --- docs/build/html/genindex.html | 2 + docs/build/html/index.html | 23 +- docs/build/html/modules.html | 7 +- docs/build/html/objects.inv | Bin 2712 -> 2727 bytes docs/build/html/quapy.classification.html | 27 +- docs/build/html/quapy.data.html | 31 +- docs/build/html/quapy.html | 116 ++++++-- docs/build/html/quapy.method.html | 35 +-- docs/build/html/searchindex.js | 2 +- quapy/plot.py | 328 ++++++++++++++++++---- 10 files changed, 436 insertions(+), 135 deletions(-) diff --git a/docs/build/html/genindex.html b/docs/build/html/genindex.html index 5d2c96f..36d0848 100644 --- a/docs/build/html/genindex.html +++ b/docs/build/html/genindex.html @@ -176,6 +176,8 @@
  • binary_diagonal() (in module quapy.plot)
  • BinaryQuantifier (class in quapy.method.base) +
    • +
  • brokenbar_supremacy_by_drift() (in module quapy.plot)
    • diff --git a/docs/build/html/index.html b/docs/build/html/index.html index 44c449a..8a3bc6e 100644 --- a/docs/build/html/index.html +++ b/docs/build/html/index.html @@ -5,7 +5,8 @@ - + + Welcome to QuaPy’s documentation! — QuaPy 0.1.6 documentation @@ -45,11 +46,11 @@
    -
    +

    Welcome to QuaPy’s documentation!

    QuaPy is an open source framework for Quantification (a.k.a. Supervised Prevalence Estimation) written in Python.

    -
    +

    Introduction

    QuaPy roots on the concept of data sample, and provides implementations of most important concepts in quantification literature, such as the most important quantification baselines, many advanced @@ -57,7 +58,7 @@ quantification methods, quantification-oriented model selection, many evaluation used for evaluating quantification methods. QuaPy also integrates commonly used datasets and offers visualization tools for facilitating the analysis and interpretation of results.

    -
    +

    A quick example:

    The following script fetchs a Twitter dataset, trains and evaluates an Adjusted Classify & Count model in terms of the Mean Absolute Error (MAE) @@ -87,8 +88,8 @@ class prevalence of the training set. For this reason, any Quantification model should be tested across samples characterized by different class prevalences. QuaPy implements sampling procedures and evaluation protocols that automates this endeavour. See the Evaluation for detailed examples.

    -
    -
    +
    +

    Features

    • Implementation of most popular quantification methods (Classify-&-Count variants, Expectation-Maximization, SVM-based variants for quantification, HDy, QuaNet, and Ensembles).

      • @@ -144,17 +145,17 @@ See the E
    -
    -
    -
    -
    + + + +

    Indices and tables

    -
    +
    diff --git a/docs/build/html/modules.html b/docs/build/html/modules.html index 6c5aa2e..b23a7dd 100644 --- a/docs/build/html/modules.html +++ b/docs/build/html/modules.html @@ -5,7 +5,8 @@ - + + quapy — QuaPy 0.1.6 documentation @@ -49,7 +50,7 @@
    -
    +

    quapy

      @@ -96,7 +97,7 @@
    -
    +
    diff --git a/docs/build/html/objects.inv b/docs/build/html/objects.inv index 318d51756e1a0bc26664fb0035d8a75d56e9414f..e7019681abe4c99cb19d4a5b48916f5e2d75577f 100644 GIT binary patch delta 2611 zcmV-33e5GG6{i)DbOD`_bsvAomV}rDxBz5Zd)n8?>*YxT@K2-&QN$1DFt$vgzs66a zy8(in-w!xH&hq0=8A}c+%zm8xuCGLUnWaCJ*KYqhyS*u|#wiwpY)OnIVbmbjEij4_ zWgzz~5n*Z3g1vvk^mV~@L`V{UgB=G|uB>$-Gy624#+@H+i7eVu6r6ub$qppSF~^yh zURTZ`A+h{!icrqk22Za}e8)bN98B=>F-6g7lmi9afaE_>cjW z2(k?%38A|%4X^p(?7z!>V=>+wQp}G?kUv1fl{MzT<GE8u9Is8!F#y| zPu0y&#jpG7JK)b86AFJkOt)WvM+v-zN zrh2sfjHV)VOtddR+XIc8rK^AA8wCMk^Eon5j6BDNiyY_VH=L3LrA!DY*jP$VHU|kK z@rg1`gEi^P7i_5bYNIUMQ7pPR(@xR@V$nKJiCn7?DYaKHdFOGQ4WY_m8GY3_Kv8)R z$OG~E^Zko+1GIlfaTcbis=i0&I|`h@7WW00Ce zm}~ot2~Lr~nX*~BLpxB}9hQt+ciKn=geLG2R*myCF|cBGpbi#eAURDO1IK9&88|-E z$%MmF=MfWGPLDo;*{DFOLKDz(iSA3QYZ{V;wwz^U(}RCv!x43$L&81MnV3#4+W@LE za1?T6&Y6x~NyN!#HL2%S(WDGZ!Xc}vMWbYkIOHiVjow~&v;`UO(Le7WTRD;s#vCD? ziE2(0co$v_`A(!P7eWpUwH~)@f^tO4AIJKRKxL(NgWHjh9<@b2*V{9uJ474g zB~20N|YQ(MJx7(h=SWWo8mkBivJE(h}h~b`XC7i_#Wlx1Vh5vi)R(*MmuMW+C#L zCgTfdHf_xSQ{*~og)oOyWt(r{PO)npwJXl8dP7(gwyTI$J-qwWrX3<aepSmEm307rAUb~;7c^S(IQh>(NJx4DqY2?%8F2T|HTGz$l8sHS*9~BS>MyOAhjq(lpM%d z#zTr5nH!Q4Nx*}p#z|#x)}13H*APpodS8FT-&UF(5~9Ra5@>@N+!i@2TP<$-6SVx| z!aS|M#Je4bT@mv93DMpCeSeYNhFp_w{x&V!tU%SBw4*P_ye_vws6;|16oog_ry;y} zl~1hhv>)$ia<4b)&9@WBV^5z~^`zD3Grv}z*R**O5FX`BHHs6)`#!_Dr$%VKa^Qa= ztghB|NFF%i&KFCMZc)-<`bC36_yEoE?BMzG zt51(Czp8(p`lI=3{kZt@r8E6IaAE0n4)BGccSQcy5B~~kF z@<;Kxn*jK9+=p6h#Y8Tc5KBUp0?{neJ+Qw%96gLwNrxFt4e}h#-tNr5J-bF(yhqn8 z*IvRA%cV9VKyuGP>{*)lbk1`9*{eFAL)S;mI!xE^0J-ldok*WOQC}#$h`WC@_2{V@ z5oT62w1IPQ_OLMNcNgawZ5x=U4`?`H^6CaR!E*k@4!4kQm?2W=1{Nl3Uis~%t%7Z) ztwQaltpaSPt^7JASf2!FN!yGk+STjm2+Ur;EbrzXW1-~#v3T$UJb4qo`iXz@>VoIL z{pcfI=M`VdOaq(G)cf(mOc#GMI%Zj}Z?JE3J{?@*O*&!SKduMYI?dH3TI9G>igbCleWf*^omxwrcFGi97iO(}i zRNTKkWDD{NDcLiYteUg4b3LRkJ5x?Nd?vV)K9`x`q{zdbo%A^yypuMU8Nm`rk$~2d z(xVQY(aYxkkH>bskz_JX(TCa|@|yI+-<;^M5ic(YOy zRqCgVT7rM|wYq9Mpyj&_beZBwFI}p>T9ztat8jBFR-sxY{sn2Qtxz)EmeuipfBsj@ z>NkECjrz-tn&k#f`SG^?qG^P75T(x2Gjuwxx>Ob}#`mVhxB_1tRnO9{j)KwFa59?m z<1EMVE8cjI4h1nWh^>TCn$TIDcKR9Tk$s-^h*=C|A~6%gjCvsBz~A7k;sRUIy#q3N~FQ*9wC7 zat)rUo1gSQ_tibZpE-Y~IpJZt{Q^8n$Qz9(Z$pTy zbYA|wI#7HsMS~zA6}jcg&N?_`zp>U&R!& zv9D#Sz|z!Mp&p{DZ@z_6McVbEZ#Ec)@`6nw4z1p zmt{@$X!{urB6LiZFF@Nv9yd!@|Hd~80>tKXWS|&%jtv(%&dF~$r3p%z5Ja=Fl$>r3 zl8nSBV44PNQkO5-kiNA6%XSotF3z-*^nh5jE>bGj5F(}W3MTJ7jWb+#-{p&=(l5# znnai@`-~}0k-!<)Y`#M~0@)pwj9YiwNCh-c$VXT;&eO!eirFD|uowf$Y2p|-PIJh> z@tIC09F{7Nn8bnHqhPClziJ+F!;WmpmpSxqe(C0oQvk>b+m?R7_6BI7;!=lx?VNAkg# zBcwAC<}@Ym!iyo_sg&hH$bkXtamyxvBLaUMt0w}LmD&w%M?QK~7WrIn&zSZ&$5o^U zig1#kcgBCMX89GQd41$TLqi)=u3YO$euXzh9Symv8@Xw$qX9>-K0U+)#VkE!x$q%D zD997cld3(g7)d_lXSt=x){eqSvmatNAxptBFSV|FC9Fe#EZ*FO1uW-a^` z>$Ah9BSjb7LL<`NeK;-kP_=z}DbKuEgNryO8HBnZFo;@=OG{MslURzuQaVVt)GPJL zSWbWaYaos9-1@%pIE~rtP9o8-u$+OcgYXBc@|STzqa}}%{~UxQ34%Mh>>R4Ip%@LU zy%e^9si%Os!W4|=QXpKwuzb+WrD8dxt2l+M2zB>gY!HX6-KdylD$|nnGhGW(i=w0C zK*lm2q90^#Xi6m|9xNCq$l$CyM@X(9mQsK9zJ|Z8G&>{|#32c^!3=JToRzH>H~k4( z-ncMNtDAVY<7B5poPh?YE>G^&M!orQ;&|-o_o|+>>U`$c%JZ5wPXfZDfJp zF2d?+T{k{>2&t`#n8%45HTbC9MNw#bYt=Q3XBk9SPBh5Kw(x_`u0ycH8A^-n>o|?j z-E;;Qr3CZpBy8wj9)9BJ2@gEigHLa-oFG(6;2+5YN8I^h>Cr799j0FtD1;AC9M2A( zFTeWq$nq=v@zfv9SL?^cmnYw`1=4@=?l<)KJpUE|6=t9T|I_?7&TQQXjCd88Fpg=T>bPw#W4@VCp1?e!OsX(5i+1s7@w`bRY#d~zk za_uD?u|g^%B1rBz5qp*rWJ0XSdD4j^3JyBnB@*;ok($u4; zYDAb>(a;9Y!P&#Yq~BefXS8i#o<5-AgvqNL+yu+{6Fb~Ox?zS$p&MA3ta;_Pm$nMF znYIeGo3;wDowoAplwf@lpe1cHo@iIEqa!eT{j$89dyECh|6}ps2YB)(eDxFm=G6ty zfBVr#y3Q-Ul$i!LpQ-obg_(aYW^~N5LOo#L7JNFm#G7=&y2~l*?&(>~(6ITaPo^)` zi?iy)@Sdc_g~O$TunX@qjd@t5&m8|9{&L=EHW2*we@#Hw{ME#APhU+m2ir7Z-1AKn z%`+u5Azd><6H9nDRvUw-!JaYvER2dXi?^S@!%w)!i1!BFjspK+M<;)xM5B3?pWADV z=I)Z!9;u6HwR7bny7FjiE=X2|U<;$|$VM-Bi^m7F`m%Vuf1Ix!64x#c$MJ#V_=pOY z(%5>6pTT^5e;4ICC%Y7Pf8XUKB1r!<7^HoEy)hvCs19MM3vdrWU5;nyIfF?aq|aT0 zSC=9Jb&+5Px4|M6;b(swOU7NjmmcY67wU)-7%J^ME}hsLL|**3*qFdkeQ*N8@Dq{C zQr^xl8VExlSF&bZ{)&n<3UYjj_B2UI?uEPffhuc+hiklmQ>XCbJ&9lSanKOW>_Nd^ zc@QuuSiBEHa&W#zj+&XR@QDccP^jx@CWu=#PjvcrTVC{}Z2Q zlt@3nJ!A{|N>aLKELk;YXXk22U3R9NbofkgCw(q6!AX&aJv-@hHh3p(E;E89Bt?R> z-jp8odcv7p^b4oSnqVI94O_v(I@_|*2Amht1q@BF;0#)@1;=ce7JxyEv)~L^k_AOr z)TT(89BmJ(aJ7H%LK}(&{59^~iJsstg&Z@r6-_AK)$iAglGAUFp$$lKx|O5F1cm?$ zf@06!#|S}sJpP>@RXCbso*vhd6<6nxE*^7&B|&tHawZY^fUJCMy=qx72y#l0MoA{@ zJy)NRL0g&pmQ#b}0_Px`-T&$98VGhArHPg9bP7P5aApz&);62u8C)_6R=dfKei z>(O={DoIwrOea^~;(Vw-^C4vNQ^Mle?Wb9NO_zc)^V6G=tx~6}b-Bk}Na&Pa$@cZS z?6UgadaGD|f5oi+Sf_5(Uv6NQ8#EQi z+xkY+2<0G3ouy~!bX;|*EUd@(rp34dUmc-m^R5ofXlpnbP5E - + + quapy.classification package — QuaPy 0.1.6 documentation @@ -51,12 +52,12 @@
    -
    +

    quapy.classification package

    -
    +

    Submodules

    -
    -
    +
    +

    quapy.classification.methods module

    @@ -165,8 +166,8 @@ and eventually also n_components for TruncatedSVD

    -
    -
    + +

    quapy.classification.neural module

    @@ -562,8 +563,8 @@ applied, meaning that if the longest document in the batch is shorter than
    -
    -
    + +

    quapy.classification.svmperf module

    @@ -653,11 +654,11 @@ for further details.

    -
    -
    + +

    Module contents

    -
    -
    + +
    diff --git a/docs/build/html/quapy.data.html b/docs/build/html/quapy.data.html index 39bc69a..355bc52 100644 --- a/docs/build/html/quapy.data.html +++ b/docs/build/html/quapy.data.html @@ -5,7 +5,8 @@ - + + quapy.data package — QuaPy 0.1.6 documentation @@ -51,12 +52,12 @@
    -
    +

    quapy.data package

    -
    +

    Submodules

    -
    -
    +
    +

    quapy.data.base module

    @@ -206,8 +207,8 @@ quapy.data.base.isbinary(data)
    -
    -
    + +

    quapy.data.datasets module

    @@ -270,8 +271,8 @@ faster subsequent invokations quapy.data.datasets.warn(*args, **kwargs)
    -
    -
    + +

    quapy.data.preprocessing module

    @@ -360,8 +361,8 @@ where the dimensions corresponding to infrequent instances have been removed

    where the instances are stored in a csr_matrix of real-valued tfidf scores

    -
    -
    + +

    quapy.data.reader module

    @@ -422,11 +423,11 @@ E.g., y=[‘B’, ‘B’, ‘A’, ‘C’] -> [1,1,0,2], [‘A’,’B’, :return: a ndarray (int) of class indexes, and a ndarray of classnames corresponding to the indexes.

    -
    -
    + +

    Module contents

    -
    -
    + +
    diff --git a/docs/build/html/quapy.html b/docs/build/html/quapy.html index 4782c8c..86287e8 100644 --- a/docs/build/html/quapy.html +++ b/docs/build/html/quapy.html @@ -5,7 +5,8 @@ - + + quapy package — QuaPy 0.1.6 documentation @@ -52,9 +53,9 @@
    -
    +

    quapy package

    -
    +

    Subpackages

      @@ -87,11 +88,11 @@
    -
    -
    +
    +

    Submodules

    -
    -
    + +

    quapy.error module

    @@ -508,8 +509,8 @@ will be taken from the environment variable SAMPLE_SIZE (which has
    -
    -
    + +

    quapy.evaluation module

    @@ -584,8 +585,8 @@ contains the the prevalence estimations

    quapy.evaluation.natural_prevalence_report(model: quapy.method.base.BaseQuantifier, test: quapy.data.base.LabelledCollection, sample_size, n_repetitions=1, n_jobs=1, random_seed=42, error_metrics: Iterable[Union[str, Callable]] = 'mae', verbose=False)
    -
    -
    + +

    quapy.functional module

    @@ -668,8 +669,8 @@ and with the limits smoothed, i.e.: quapy.functional.uniform_simplex_sampling(n_classes, size=1)
    -
    -
    + +

    quapy.model_selection module

    @@ -783,27 +784,92 @@ a float in [0,1] indicating the proportion of labelled data to extract from the
    -
    -
    + +

    quapy.plot module

    quapy.plot.binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=<matplotlib.colors.ListedColormap object>, vertical_xticks=False, legend=True, savepath=None)
    -
    +
    +
    Parameters
    +
      +

    • method_names – array-like with the method names for each experiment

      • +

    • true_prevs – array-like with the true prevalence values (each being a ndarray with n_classes components) for +each experiment

      • +

    • estim_prevs – array-like with the estimated prevalence values (each being a ndarray with n_classes components) +for each experiment

      • +

    • pos_class – index of the positive class

      • +

    • title – the title to be displayed in the plot

      • +

    • nbins – number of bins

      • +

    • colormap – the matplotlib colormap to use (default cm.tab10)

      • +

    • vertical_xticks –

      • +

    • legend – whether or not to display the legend (default is True)

      • +

    • savepath – path where to save the plot. If not indicated (as default), the plot is shown.

      • +
    +
    +
    +
    quapy.plot.binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None)
    -
    +

    Box-plots displaying the global bias (i.e., signed error computed as the estimated value minus the true value) +for each quantification method with respect to a given positive class.

    +
    +
    Parameters
    +
      +

    • method_names – array-like with the method names for each experiment

      • +

    • true_prevs – array-like with the true prevalence values (each being a ndarray with n_classes components) for +each experiment

      • +

    • estim_prevs – array-like with the estimated prevalence values (each being a ndarray with n_classes components) +for each experiment

      • +

    • pos_class – index of the positive class

      • +

    • title – the title to be displayed in the plot

      • +

    • savepath – path where to save the plot. If not indicated (as default), the plot is shown.

      • +
    +
    +
    +
    -quapy.plot.binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True, train_prev=None, savepath=None)
    +quapy.plot.binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True, train_prev=None, savepath=None, method_order=None) +

    The diagonal plot displays the predicted prevalence values (along the y-axis) as a function of the true prevalence +values (along the x-axis). The optimal quantifier is described by the diagonal (0,0)-(1,1) of the plot (hence the +name). It is convenient for binary quantification problems, though it can be used for multiclass problems by +indicating which class is to be taken as the positive class. (For multiclass quantification problems, other plots +like the error_by_drift() might be preferable though).

    +
    +
    Parameters
    +
      +

    • method_names – array-like with the method names for each experiment

      • +

    • true_prevs – array-like with the true prevalence values (each being a ndarray with n_classes components) for +each experiment

      • +

    • estim_prevs – array-like with the estimated prevalence values (each being a ndarray with n_classes components) +for each experiment

      • +

    • pos_class – index of the positive class

      • +

    • title – the title to be displayed in the plot

      • +

    • show_std – whether or not to show standard deviations (represented by color bands). This might be inconvenient +for cases in which many methods are compared, or when the standard deviations are high – default True)

      • +

    • legend – whether or not to display the leyend (default True)

      • +

    • train_prev – if indicated (default is None), the training prevalence (for the positive class) is hightlighted +in the plot. This is convenient when all the experiments have been conducted in the same dataset.

      • +

    • savepath – path where to save the plot. If not indicated (as default), the plot is shown.

      • +

    • method_order – if indicated (default is None), imposes the order in which the methods are processed (i.e., +listed in the legend and associated with matplotlib colors).

      • +
    +
    +
    +
    + +
    +
    +quapy.plot.brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, binning='isomerous', x_error='ae', y_error='ae', ttest_alpha=0.005, tail_density_threshold=0.005, method_order=None, savepath=None)
    -quapy.plot.error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, error_name='ae', show_std=True, logscale=False, title='Quantification error as a function of distribution shift', savepath=None)
    +quapy.plot.error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, error_name='ae', show_std=False, show_density=True, logscale=False, title='Quantification error as a function of distribution shift', savepath=None, vlines=None, method_order=None)
    @@ -811,8 +877,8 @@ a float in [0,1] indicating the proportion of labelled data to extract from the quapy.plot.save_or_show(savepath)
    -
    -
    + +

    quapy.util module

    @@ -901,16 +967,16 @@ with temp_seed(random_seed):

    -
    -
    + +

    Module contents

    quapy.isbinary(x)
    -
    -
    + +
    diff --git a/docs/build/html/quapy.method.html b/docs/build/html/quapy.method.html index 1b03669..41b2f85 100644 --- a/docs/build/html/quapy.method.html +++ b/docs/build/html/quapy.method.html @@ -5,7 +5,8 @@ - + + quapy.method package — QuaPy 0.1.6 documentation @@ -47,12 +48,12 @@
    -
    +

    quapy.method package

    -
    +

    Submodules

    -
    -
    +
    +

    quapy.method.aggregative module

    @@ -599,8 +600,8 @@ LabelledCollection, represents the validation split itself or None otherwise) to be used as a validation set for any subsequent parameter fitting

    -
    -
    + +

    quapy.method.base module

    @@ -674,8 +675,8 @@ or None otherwise) to be used as a validation set for any subsequent parameter f quapy.method.base.isprobabilistic(model: quapy.method.base.BaseQuantifier)
    -
    -
    + +

    quapy.method.meta module

    @@ -814,8 +815,8 @@ to a first approximation of the test prevalence as made by all models in the ens quapy.method.meta.get_probability_distribution(posterior_probabilities, bins=8)
    -
    -
    + +

    quapy.method.neural module

    @@ -912,8 +913,8 @@ fit_learner=False, the data will be split in 66/34 for training QuaNet and valid quapy.method.neural.mae_loss(output, target)
    -
    -
    + +

    quapy.method.non_aggregative module

    @@ -946,11 +947,11 @@ fit_learner=False, the data will be split in 66/34 for training QuaNet and valid
    -
    -
    + +

    Module contents

    -
    -
    + +
    diff --git a/docs/build/html/searchindex.js b/docs/build/html/searchindex.js index 0795a61..39defdc 100644 --- a/docs/build/html/searchindex.js +++ b/docs/build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["Datasets","Evaluation","Installation","Methods","Model-Selection","Plotting","index","modules","quapy","quapy.classification","quapy.data","quapy.method"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["Datasets.md","Evaluation.md","Installation.rst","Methods.md","Model-Selection.md","Plotting.md","index.rst","modules.rst","quapy.rst","quapy.classification.rst","quapy.data.rst","quapy.method.rst"],objects:{"":{quapy:[8,0,0,"-"]},"quapy.classification":{methods:[9,0,0,"-"],neural:[9,0,0,"-"],svmperf:[9,0,0,"-"]},"quapy.classification.methods":{LowRankLogisticRegression:[9,1,1,""]},"quapy.classification.methods.LowRankLogisticRegression":{fit:[9,2,1,""],get_params:[9,2,1,""],predict:[9,2,1,""],predict_proba:[9,2,1,""],set_params:[9,2,1,""],transform:[9,2,1,""]},"quapy.classification.neural":{CNNnet:[9,1,1,""],LSTMnet:[9,1,1,""],NeuralClassifierTrainer:[9,1,1,""],TextClassifierNet:[9,1,1,""],TorchDataset:[9,1,1,""]},"quapy.classification.neural.CNNnet":{document_embedding:[9,2,1,""],get_params:[9,2,1,""],vocabulary_size:[9,3,1,""]},"quapy.classification.neural.LSTMnet":{document_embedding:[9,2,1,""],get_params:[9,2,1,""],vocabulary_size:[9,3,1,""]},"quapy.classification.neural.NeuralClassifierTrainer":{device:[9,3,1,""],fit:[9,2,1,""],get_params:[9,2,1,""],predict:[9,2,1,""],predict_proba:[9,2,1,""],reset_net_params:[9,2,1,""],set_params:[9,2,1,""],transform:[9,2,1,""]},"quapy.classification.neural.TextClassifierNet":{dimensions:[9,2,1,""],document_embedding:[9,2,1,""],forward:[9,2,1,""],get_params:[9,2,1,""],predict_proba:[9,2,1,""],vocabulary_size:[9,3,1,""],xavier_uniform:[9,2,1,""]},"quapy.classification.neural.TorchDataset":{asDataloader:[9,2,1,""]},"quapy.classification.svmperf":{SVMperf:[9,1,1,""]},"quapy.classification.svmperf.SVMperf":{decision_function:[9,2,1,""],fit:[9,2,1,""],predict:[9,2,1,""],set_params:[9,2,1,""],valid_losses:[9,4,1,""]},"quapy.data":{base:[10,0,0,"-"],datasets:[10,0,0,"-"],preprocessing:[10,0,0,"-"],reader:[10,0,0,"-"]},"quapy.data.base":{Dataset:[10,1,1,""],LabelledCollection:[10,1,1,""],isbinary:[10,5,1,""]},"quapy.data.base.Dataset":{SplitStratified:[10,2,1,""],binary:[10,3,1,""],classes_:[10,3,1,""],kFCV:[10,2,1,""],load:[10,2,1,""],n_classes:[10,3,1,""],stats:[10,2,1,""],vocabulary_size:[10,3,1,""]},"quapy.data.base.LabelledCollection":{Xy:[10,3,1,""],artificial_sampling_generator:[10,2,1,""],artificial_sampling_index_generator:[10,2,1,""],binary:[10,3,1,""],counts:[10,2,1,""],kFCV:[10,2,1,""],load:[10,2,1,""],n_classes:[10,3,1,""],natural_sampling_generator:[10,2,1,""],natural_sampling_index_generator:[10,2,1,""],prevalence:[10,2,1,""],sampling:[10,2,1,""],sampling_from_index:[10,2,1,""],sampling_index:[10,2,1,""],split_stratified:[10,2,1,""],stats:[10,2,1,""],uniform_sampling:[10,2,1,""],uniform_sampling_index:[10,2,1,""]},"quapy.data.datasets":{df_replace:[10,5,1,""],fetch_UCIDataset:[10,5,1,""],fetch_UCILabelledCollection:[10,5,1,""],fetch_reviews:[10,5,1,""],fetch_twitter:[10,5,1,""],warn:[10,5,1,""]},"quapy.data.preprocessing":{IndexTransformer:[10,1,1,""],index:[10,5,1,""],reduce_columns:[10,5,1,""],standardize:[10,5,1,""],text2tfidf:[10,5,1,""]},"quapy.data.preprocessing.IndexTransformer":{add_word:[10,2,1,""],fit:[10,2,1,""],fit_transform:[10,2,1,""],index:[10,2,1,""],transform:[10,2,1,""],vocabulary_size:[10,2,1,""]},"quapy.data.reader":{binarize:[10,5,1,""],from_csv:[10,5,1,""],from_sparse:[10,5,1,""],from_text:[10,5,1,""],reindex_labels:[10,5,1,""]},"quapy.error":{absolute_error:[8,5,1,""],acc_error:[8,5,1,""],acce:[8,5,1,""],ae:[8,5,1,""],f1_error:[8,5,1,""],f1e:[8,5,1,""],from_name:[8,5,1,""],kld:[8,5,1,""],mae:[8,5,1,""],mean_absolute_error:[8,5,1,""],mean_relative_absolute_error:[8,5,1,""],mkld:[8,5,1,""],mnkld:[8,5,1,""],mrae:[8,5,1,""],mse:[8,5,1,""],nkld:[8,5,1,""],rae:[8,5,1,""],relative_absolute_error:[8,5,1,""],se:[8,5,1,""],smooth:[8,5,1,""]},"quapy.evaluation":{artificial_prevalence_prediction:[8,5,1,""],artificial_prevalence_protocol:[8,5,1,""],artificial_prevalence_report:[8,5,1,""],evaluate:[8,5,1,""],gen_prevalence_prediction:[8,5,1,""],natural_prevalence_prediction:[8,5,1,""],natural_prevalence_protocol:[8,5,1,""],natural_prevalence_report:[8,5,1,""]},"quapy.functional":{HellingerDistance:[8,5,1,""],adjusted_quantification:[8,5,1,""],artificial_prevalence_sampling:[8,5,1,""],get_nprevpoints_approximation:[8,5,1,""],normalize_prevalence:[8,5,1,""],num_prevalence_combinations:[8,5,1,""],prevalence_from_labels:[8,5,1,""],prevalence_from_probabilities:[8,5,1,""],prevalence_linspace:[8,5,1,""],strprev:[8,5,1,""],uniform_prevalence_sampling:[8,5,1,""],uniform_simplex_sampling:[8,5,1,""]},"quapy.method":{aggregative:[11,0,0,"-"],base:[11,0,0,"-"],meta:[11,0,0,"-"],neural:[11,0,0,"-"],non_aggregative:[11,0,0,"-"]},"quapy.method.aggregative":{ACC:[11,1,1,""],AdjustedClassifyAndCount:[11,4,1,""],AggregativeProbabilisticQuantifier:[11,1,1,""],AggregativeQuantifier:[11,1,1,""],CC:[11,1,1,""],ClassifyAndCount:[11,4,1,""],ELM:[11,1,1,""],EMQ:[11,1,1,""],ExpectationMaximizationQuantifier:[11,4,1,""],ExplicitLossMinimisation:[11,4,1,""],HDy:[11,1,1,""],HellingerDistanceY:[11,4,1,""],MAX:[11,1,1,""],MS2:[11,1,1,""],MS:[11,1,1,""],MedianSweep2:[11,4,1,""],MedianSweep:[11,4,1,""],OneVsAll:[11,1,1,""],PACC:[11,1,1,""],PCC:[11,1,1,""],ProbabilisticAdjustedClassifyAndCount:[11,4,1,""],ProbabilisticClassifyAndCount:[11,4,1,""],SVMAE:[11,1,1,""],SVMKLD:[11,1,1,""],SVMNKLD:[11,1,1,""],SVMQ:[11,1,1,""],SVMRAE:[11,1,1,""],T50:[11,1,1,""],ThresholdOptimization:[11,1,1,""],X:[11,1,1,""],training_helper:[11,5,1,""]},"quapy.method.aggregative.ACC":{aggregate:[11,2,1,""],classify:[11,2,1,""],fit:[11,2,1,""],solve_adjustment:[11,2,1,""]},"quapy.method.aggregative.AggregativeProbabilisticQuantifier":{posterior_probabilities:[11,2,1,""],predict_proba:[11,2,1,""],probabilistic:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.aggregative.AggregativeQuantifier":{aggregate:[11,2,1,""],aggregative:[11,3,1,""],classes_:[11,3,1,""],classify:[11,2,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],learner:[11,3,1,""],n_classes:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.aggregative.CC":{aggregate:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.ELM":{aggregate:[11,2,1,""],classify:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.EMQ":{EM:[11,2,1,""],EPSILON:[11,4,1,""],MAX_ITER:[11,4,1,""],aggregate:[11,2,1,""],fit:[11,2,1,""],predict_proba:[11,2,1,""]},"quapy.method.aggregative.HDy":{aggregate:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.MS":{optimize_threshold:[11,2,1,""]},"quapy.method.aggregative.MS2":{optimize_threshold:[11,2,1,""]},"quapy.method.aggregative.OneVsAll":{aggregate:[11,2,1,""],binary:[11,3,1,""],classes_:[11,3,1,""],classify:[11,2,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],posterior_probabilities:[11,2,1,""],probabilistic:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.aggregative.PACC":{aggregate:[11,2,1,""],classify:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.PCC":{aggregate:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.ThresholdOptimization":{aggregate:[11,2,1,""],compute_fpr:[11,2,1,""],compute_table:[11,2,1,""],compute_tpr:[11,2,1,""],fit:[11,2,1,""],optimize_threshold:[11,2,1,""]},"quapy.method.base":{BaseQuantifier:[11,1,1,""],BinaryQuantifier:[11,1,1,""],isaggregative:[11,5,1,""],isbinary:[11,5,1,""],isprobabilistic:[11,5,1,""]},"quapy.method.base.BaseQuantifier":{aggregative:[11,3,1,""],binary:[11,3,1,""],classes_:[11,3,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],probabilistic:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.base.BinaryQuantifier":{binary:[11,3,1,""]},"quapy.method.meta":{EACC:[11,5,1,""],ECC:[11,5,1,""],EEMQ:[11,5,1,""],EHDy:[11,5,1,""],EPACC:[11,5,1,""],Ensemble:[11,1,1,""],ensembleFactory:[11,5,1,""],get_probability_distribution:[11,5,1,""]},"quapy.method.meta.Ensemble":{VALID_POLICIES:[11,4,1,""],accuracy_policy:[11,2,1,""],aggregative:[11,3,1,""],binary:[11,3,1,""],classes_:[11,3,1,""],ds_policy:[11,2,1,""],ds_policy_get_posteriors:[11,2,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],probabilistic:[11,3,1,""],ptr_policy:[11,2,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""],sout:[11,2,1,""]},"quapy.method.neural":{QuaNetModule:[11,1,1,""],QuaNetTrainer:[11,1,1,""],mae_loss:[11,5,1,""]},"quapy.method.neural.QuaNetModule":{device:[11,3,1,""],forward:[11,2,1,""],init_hidden:[11,2,1,""]},"quapy.method.neural.QuaNetTrainer":{classes_:[11,3,1,""],clean_checkpoint:[11,2,1,""],clean_checkpoint_dir:[11,2,1,""],epoch:[11,2,1,""],fit:[11,2,1,""],get_aggregative_estims:[11,2,1,""],get_params:[11,2,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.non_aggregative":{MaximumLikelihoodPrevalenceEstimation:[11,1,1,""]},"quapy.method.non_aggregative.MaximumLikelihoodPrevalenceEstimation":{classes_:[11,3,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.model_selection":{GridSearchQ:[8,1,1,""]},"quapy.model_selection.GridSearchQ":{best_model:[8,2,1,""],classes_:[8,3,1,""],fit:[8,2,1,""],get_params:[8,2,1,""],quantify:[8,2,1,""],set_params:[8,2,1,""]},"quapy.plot":{binary_bias_bins:[8,5,1,""],binary_bias_global:[8,5,1,""],binary_diagonal:[8,5,1,""],error_by_drift:[8,5,1,""],save_or_show:[8,5,1,""]},"quapy.util":{EarlyStop:[8,1,1,""],create_if_not_exist:[8,5,1,""],create_parent_dir:[8,5,1,""],download_file:[8,5,1,""],download_file_if_not_exists:[8,5,1,""],get_quapy_home:[8,5,1,""],map_parallel:[8,5,1,""],parallel:[8,5,1,""],pickled_resource:[8,5,1,""],save_text_file:[8,5,1,""],temp_seed:[8,5,1,""]},quapy:{classification:[9,0,0,"-"],data:[10,0,0,"-"],error:[8,0,0,"-"],evaluation:[8,0,0,"-"],functional:[8,0,0,"-"],isbinary:[8,5,1,""],method:[11,0,0,"-"],model_selection:[8,0,0,"-"],plot:[8,0,0,"-"],util:[8,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","property","Python property"],"4":["py","attribute","Python attribute"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:property","4":"py:attribute","5":"py:function"},terms:{"0":[0,1,3,4,5,8,9,10,11],"00":[0,1,4,8],"000":1,"0001":[4,11],"000e":1,"001":[4,9,11],"009":1,"01":[8,9,11],"017":1,"018":0,"02":1,"021":0,"02552":4,"03":1,"034":1,"035":1,"037":1,"04":1,"041":1,"042":1,"046":1,"048":1,"05":[5,8],"055":1,"063":0,"065":0,"070":1,"073":1,"075":1,"078":0,"081":0,"082":[0,1],"083":0,"086":0,"091":1,"099":0,"1":[0,1,3,4,5,8,9,10,11],"10":[0,1,4,5,8,9,11],"100":[0,1,3,4,5,9,10,11],"1000":[0,4,11],"10000":4,"100000":4,"101":[4,10],"1010":4,"1024":11,"104":0,"108":1,"109":0,"11":[0,1,6],"11338":0,"114":1,"1145":[],"12":9,"120":0,"1215742":0,"1271":0,"13":[0,9],"139":0,"14":[3,11],"142":1,"146":[3,11],"1473":0,"148":0,"1484":0,"15":[3,8,11],"150":0,"153":0,"157":0,"158":0,"159":0,"1593":0,"1594":0,"1599":0,"161":0,"163":[0,1],"164":[0,3,11],"167":0,"17":0,"1771":1,"1775":[0,3],"1778":[0,3],"178":0,"1823":0,"1839":0,"18399":0,"1853":0,"19":[3,10,11],"193":0,"199151":0,"19982":4,"1e":9,"1st":0,"2":[0,1,3,5,8,10,11],"20":[5,8,11],"200":[1,9],"2000":0,"2002":[3,11],"2011":4,"2013":[3,11],"2015":[0,2,3,9,11],"2016":[3,10,11],"2017":[0,3,11],"2018":[0,3,10],"2019":[3,11],"2020":4,"20342":4,"206":0,"207":0,"208":0,"21":[1,3,5,8,11],"210":8,"211":0,"2126":0,"2155":0,"21591":0,"218":[3,11],"2184":0,"219e":1,"22":[0,3,9,10,11],"222":0,"222046":0,"226":0,"229":1,"229399":0,"23":9,"235":1,"238":0,"2390":0,"24":[0,9],"243":0,"248563":0,"24866":4,"24987":4,"25":[0,5,8,9,11],"25000":0,"256":[0,9],"26":9,"261":0,"265":0,"266":0,"267":0,"27":[1,3,9,11],"270":0,"2700406":[],"271":0,"272":0,"274":0,"275":1,"27th":[0,3,10],"28":3,"280":0,"281":0,"282":0,"283":[0,1],"288":0,"289":0,"2971":0,"2nd":0,"2t":[1,8],"2tp":8,"2x5fcv":0,"3":[0,1,3,5,6,8,9,10,11],"30":[0,1,3,11],"300":[0,1,9],"305":0,"306":0,"312":0,"32":[0,6],"33":[0,5],"331":0,"333":0,"335":0,"337":0,"34":[0,3,11],"341":0,"346":1,"347":0,"350":0,"351":0,"357":1,"359":0,"361":0,"366":1,"372":0,"373":0,"376132":0,"3765":0,"3813":0,"3821":0,"383e":1,"387e":1,"392":0,"394":0,"399":0,"3f":[1,6],"3rd":0,"4":[0,1,3,4,5,8,11],"40":[0,3,4,11],"404333":0,"407":0,"41":[3,11],"412":0,"412e":1,"413":0,"414":0,"417":0,"41734":4,"42":[1,8],"421":0,"4259":0,"426e":1,"427":0,"430":0,"434":0,"435":1,"43676":4,"437":0,"44":0,"446":0,"45":[3,5,11],"452":0,"459":1,"4601":0,"461":0,"463":0,"465":0,"466":0,"470":0,"48":[3,11],"481":0,"48135":4,"486":0,"4898":0,"492":0,"496":0,"4960":1,"497":0,"5":[0,1,3,4,5,8,9,10,11],"50":[0,5,8,11],"500":[0,1,4,5,11],"5000":[1,5],"5005":4,"507":0,"508":0,"512":[9,11],"514":0,"515e":1,"530":0,"534":0,"535":0,"535e":1,"5379":4,"539":0,"541":1,"546":0,"5473":0,"54it":4,"55":5,"55it":4,"565":1,"569":0,"57":0,"573":0,"578":1,"583":0,"591":[3,11],"5f":4,"5fcv":11,"6":[0,1,3,5,8,10,11],"60":0,"600":1,"601":0,"604":[3,11],"606":0,"625":0,"627":0,"633e":1,"634":1,"64":[9,11],"640":0,"641":0,"650":0,"653":0,"654":1,"66":[1,11],"665":0,"667":0,"669":0,"67":5,"683":0,"688":0,"691":0,"694582":0,"7":[1,5,9],"70":0,"700":0,"701e":1,"711":0,"717":1,"725":1,"730":0,"735":0,"740e":1,"748":0,"75":[0,5,8],"762":0,"774":0,"778":0,"787":0,"794":0,"798":0,"8":[0,1,5,10,11],"8000":0,"830":0,"837":1,"858":1,"861":0,"87":[0,3,11],"8788":0,"889504":0,"8d2fhsgcvn0aaaaa":[],"9":[0,1,3,5,11],"90":[5,8],"901":0,"909":1,"914":1,"917":0,"919":0,"922":0,"923":0,"935":0,"936":0,"937":0,"945":1,"95":8,"9533":0,"958":0,"97":0,"979":0,"982":0,"99":8,"abstract":[3,9,11],"case":[0,1,3,4,5,8,11],"class":[0,1,3,4,5,6,8,9,10,11],"d\u00edez":[3,11],"default":[1,3,8,9,10],"do":[0,1,3,4,8,9],"final":[1,3,5],"float":[0,3,8,9,10,11],"function":[0,1,3,4,5,6,7,9,11],"g\u00e1llego":[0,3,11],"gonz\u00e1lez":[3,11],"import":[0,1,3,4,5,6],"int":[0,5,8,10,11],"long":[4,9],"new":[0,3,10,11],"p\u00e9rez":[0,3,11],"return":[0,1,3,4,5,8,9,10,11],"rodr\u0131":[3,11],"short":9,"static":[3,11],"true":[0,1,3,4,5,6,8,9,10,11],"try":4,"while":[3,5,8,9,11],A:[0,3,8,9,10,11],As:[3,4],By:[1,3,8],For:[0,1,5,6,8,11],If:[3,5,8,11],In:[0,1,2,3,4,5,6,9,11],It:[3,4,5],One:[0,1,3,11],That:[1,4],The:[0,1,2,4,5,6,8,9,10,11],Then:3,These:0,To:[5,10],_:5,__:[],__class__:5,__name__:5,_adjust:[],_ae_:[],_classify_:11,_error_name_:11,_fit_learner_:11,_kld_:[],_labelledcollection_:11,_learner_:11,_mean:[],_min_df_:10,_my:[],_nkld_:[],_posterior_probabilities_:11,_q_:[],_rae_:[],_svmperf_:[],ab:[],aboud:3,about:[0,5],abov:[0,3,5],absolut:[1,3,5,6,8],absolute_error:8,abstractmethod:3,acc:[1,3,5,6,8,11],acc_error:8,accept:3,access:[0,3],accommod:0,accord:[1,3,4,8,9],accordingli:5,accuraci:[1,5,8],accuracy_polici:11,achiev:[1,3,4,5],acm:[0,3,10,11],across:[0,1,4,5,6,8],action:[0,11],acut:0,ad:6,add:[3,4,8],add_word:10,addit:3,addition:[0,11],adjust:[3,6,11],adjusted_quantif:8,adjustedclassifyandcount:11,adopt:[3,4],advanc:[0,6],advantag:3,ae:[1,2,5,8],ae_:1,affect:8,afterward:11,again:5,against:5,aggreg:[1,4,5,6,7,8],aggregativeprobabilisticquantifi:[3,11],aggregativequantifi:[3,11],aggregg:11,aim:[4,5],al:[0,2,9],alaiz:[3,11],alegr:[3,11],alejandro:4,alia:[3,11],all:[0,1,2,3,5,8,11],allia:3,alloc:9,allow:[0,1,2,3,5,8,9,10,11],almost:3,along:[0,3,11],alreadi:[3,11],also:[0,1,2,3,5,6,9],altern:4,although:[3,4,5,11],alwai:[3,4,5],among:3,an:[0,1,2,3,4,5,6,8,9,11],analys:[5,6],analysi:[0,3,6,10,11],analyz:5,ani:[0,1,3,4,5,6,8,9,10,11],anoth:[0,1,3,5],anyon:0,api:6,app:8,appeal:1,appear:5,append:5,appli:[2,3,4,5,8,9,10],appropri:4,approxim:[1,5,9,11],ar:[0,1,3,4,5,8,9,10,11],archive_filenam:8,archive_path:8,arg:[8,10,11],args_i:8,argu:4,argument:[0,1,3,5],arifici:8,aris:1,around:1,arrai:[1,3,5,8,9,10],articl:[3,4,11],artifici:[0,1,3,4,5,6,8],artificial_prevalence_predict:8,artificial_prevalence_protocol:8,artificial_prevalence_report:8,artificial_prevalence_sampl:8,artificial_sampling_ev:[1,4],artificial_sampling_gener:[0,10],artificial_sampling_index_gener:10,artificial_sampling_predict:[1,5],artificial_sampling_report:1,arxiv:4,asarrai:1,asdataload:9,asonam:0,assess:4,assign:[3,8],associ:10,assum:[1,6,11],assumpt:[1,5,6],astyp:10,attempt:3,attribut:11,august:0,autom:[0,3,6],automat:[0,1],av:[3,11],avail:[0,1,2,3,5,6,9],averag:[1,3,8],avoid:1,axi:5,b:[0,10],balanc:[0,4],band:5,bar:8,barranquero:[2,3,9,11],base:[0,3,6,7,8,9],base_classifi:5,base_estim:3,base_quantifier_class:11,baseestim:[9,11],baselin:6,basequantifi:[3,8,11],basic:[5,11],batch:9,batch_siz:9,batch_size_test:9,been:[0,3,4,5,10,11],befor:[3,9,11],beforehand:8,behav:[3,5],being:[4,8],belief:1,belong:3,below:[0,2,3,5,10],best:[4,8,9,11],best_model:8,best_model_:4,best_params_:4,better:4,between:[4,5,6,8,9],beyond:5,bia:6,bias:5,bidirect:11,bin:[5,11],bin_bia:5,bin_diag:5,binar:[8,10],binari:[3,5,6,9,10,11],binary_bias_bin:[5,8],binary_bias_glob:[5,8],binary_diagon:[5,8],binary_quantifi:11,binaryquantifi:11,block:0,bool:[8,11],both:5,bound:8,box:5,breast:0,brief:1,broken:5,budg:1,budget:[1,4],build:11,bypass:11,c:[3,4,9,10,11],calibr:3,calibratedclassifi:3,calibratedclassifiercv:3,calibratedcv:11,call:[0,1,5,8,11],callabl:[0,8,10],can:[0,1,2,3,4,5,8],cancer:0,cannot:11,cardiotocographi:0,care:11,carri:3,casa_token:[],castano:[3,11],castro:[3,11],categor:3,categori:[1,8],cc:[3,5,11],ceil:8,center:5,chang:[0,1,3,11],character:[3,6],characteriz:[0,3,11],charg:[0,8],check:[3,4],checkpoint:[9,11],checkpointdir:11,checkpointnam:11,checkpointpath:9,choic:4,chosen:[4,8],cl:0,class2int:10,class_weight:4,classes_:[8,10,11],classif:[0,1,3,7,8,10,11],classif_posterior:[3,11],classif_predict:[3,11],classif_predictions_bin:11,classifi:[1,4,5,6,9,11],classifier_net:9,classifiermixin:9,classifyandcount:[3,11],classmethod:[0,10,11],classnam:10,clean_checkpoint:11,clean_checkpoint_dir:11,clear:5,clearer:1,clearli:5,clip:8,close:1,closer:1,cmc:0,cnn:3,cnnnet:[3,9],code:[0,3,4,5,9],coincid:[0,6],col:[0,10],collect:[0,8,9,10],collet:10,color:[5,8],colormap:8,column:[0,10],com:[],combin:[0,1,4,8],combinatio:8,combinations_budget:8,come:0,commandlin:[],common:11,commonli:6,compar:[5,11],comparison:5,compil:[2,3],complet:[3,5],compon:9,compress:0,comput:[1,3,5,8,11],computation:4,compute_fpr:11,compute_t:11,compute_tpr:11,concept:6,concur:11,conduct:0,confer:[0,3,10],configur:[4,8],consecut:9,consid:[3,5,9,10],consist:[0,4,5,9,10],constrain:[1,5],constructor:3,consult:[0,1],contain:[1,2,3,5,8,9,10,11],contanin:8,content:7,context:8,contrast:1,control:[1,4],conv_block:[],conv_lay:[],convert:[1,3,9],convolut:9,copi:10,cornel:[],correct:11,correspond:[5,10],cost:1,costli:4,could:[0,1,3,4,5,6,11],count:[4,5,6,10,11],count_:[],counter:10,countvector:10,covari:10,cover:[1,4,9],coz:[0,3,11],cpu:[1,9],creat:[0,6,8],create_if_not_exist:8,create_parent_dir:8,crisp:3,criteria:4,cross:[3,11],cs:[],csr_matrix:10,csv:10,ctg:0,cuda:[3,9,11],cumbersom:1,curios:5,current:[3,8,9,10],custom:[3,6,8],customarili:[3,4],cv:[3,4],cyan:5,d_:8,dat:[0,9],data:[1,3,4,5,6,7,8,9,11],data_hom:10,datafram:1,dataload:9,dataset:[1,3,4,5,6,7,8,9,11],dataset_nam:10,deal:0,decaesteck:[3,11],decai:9,decim:1,decis:[3,9],decision_funct:9,decomposit:9,dedic:1,deep:[3,8,11],def:[0,1,3,5,8],defin:[0,3,8,9,11],degre:4,del:[0,3,11],delai:8,deliv:3,dens:0,depend:[0,1,4,5,8],describ:[3,11],descript:0,design:4,desir:[0,1],despit:1,detail:[0,1,3,6,9,11],determin:[1,4,5],detriment:5,devel:10,develop:[4,6],deviat:[0,1,5],devic:[0,3,5,9,11],df:[1,10],df_replac:10,diabet:0,diagon:6,dict:[8,10,11],dictionari:[8,9],differ:[0,1,3,4,5,6,8,10],difficult:5,digit:0,dimens:[8,9,10],dimension:[8,9,10],directli:[0,1,3],directori:[2,9,10],discoveri:[3,11],discuss:5,disjoint:9,displai:[1,5],displaystyl:8,distanc:11,distant:[1,8],distribut:[0,3,5,8,11],diverg:[1,3,8],dl:[],doabl:0,doc_embed:11,doc_embedding_s:11,doc_posterior:11,document:[0,1,3,5,9,10,11],document_embed:9,doe:[0,2,3,8],doi:[],done:3,dot:5,down:5,download:[0,2,3],download_fil:8,download_file_if_not_exist:8,drawn:[0,1,4],drift:6,drop:[9,11],drop_p:9,dropout:9,ds:[3,11],ds_polici:11,ds_policy_get_posterior:11,dtype:1,dump:10,dure:[1,5],dynam:[3,9,11],e:[0,1,3,4,5,6,8,9,10,11],eacc:11,each:[0,1,3,4,5,8,9,10,11],earli:9,early_stop:11,earlystop:8,easili:[0,2,5,9],ecc:11,edu:[],eemq:11,effect:3,effici:3,ehdi:11,either:[1,3,8,11],element:3,elm:[3,11],em:11,emb:9,embed:[3,9],embed_s:9,embedding_s:9,empti:10,emq:[5,11],enabl:9,encod:10,end:[4,8],endeavour:6,enough:5,ensembl:[0,6,11],ensemblefactori:11,ensure_probabilist:11,entir:[0,3,4,5],environ:[1,3,4,5,8],ep:[1,8],epacc:11,epoch:[9,11],epsilon:[1,8,11],equal:[1,8],equidist:[0,8],equip:[3,5],err:8,err_drift:5,err_nam:8,error:[3,4,6,7,9],error_:[],error_by_drift:[5,8],error_funct:1,error_metr:[1,4,8],error_nam:[5,8,11],establish:8,estim:[1,3,5,6,8,9,11],estim_prev:[1,5,8],estim_preval:[3,6],esuli:[0,2,3,9,10,11],et:[0,2,9],etc:6,eval_budget:[4,8],evalu:[0,3,4,5,6,7,9],eventu:9,everi:[3,11],everyth:3,evinc:5,ex:[],exact:0,exactli:0,exampl:[0,1,3,4,5,8,9,11],exce:8,excel:0,except:[3,8],exemplifi:0,exhibit:[4,5],exist:8,expand_frame_repr:1,expect:6,expectationmaximizationquantifi:[3,11],experi:[1,2,3,4,5,8],explain:[1,5],explicitlossminim:11,explicitlossminimis:11,explor:[4,8],express:10,ext:2,extend:[2,3,11],extens:[0,2,5],extern:3,extract:[1,8],f1:[1,8,9],f1_error:8,f1e:[1,8],f:[0,1,3,4,5,6,10,11],f_1:8,fabrizio:4,facilit:6,fact:[3,5],factor:8,fals:[1,3,5,8,9,10,11],famili:3,familiar:3,far:9,fast:8,faster:[0,10],feat1:10,feat2:10,featn:10,featur:0,feature_extract:10,fetch:[0,6],fetch_review:[0,1,3,4,5,10],fetch_twitt:[0,3,6,10],fetch_ucidataset:[0,3,10],fetch_ucilabelledcollect:[0,10],ff_layer:11,fhe:0,file:[0,5,9,10],fin:0,find:[0,4],finish:4,first:[0,1,2,3,5,8,10,11],fit:[1,3,4,5,6,8,9,10,11],fit_learn:[3,11],fit_transform:10,fix:[1,4],float64:1,fn:8,fold:[3,11],folder:0,follow:[0,1,3,4,5,6],fomart:10,for_model_select:[0,10],form:0,format:[0,5,10],former:[2,11],forward:[9,11],found:[0,3,4,9],four:3,fp:[8,11],fpr:8,frac:8,framework:6,frequenc:0,from:[0,1,3,4,5,6,8,10,11],from_csv:10,from_nam:[1,8],from_spars:10,from_text:10,full:1,fulli:0,func:8,further:[0,1,3,9],fusion:[0,3,11],futur:3,g:[0,1,3,4,6,8,10,11],gao:[0,3,10,11],gasp:[0,10],gen:8,gen_data:5,gen_fn:8,gen_prevalence_predict:8,gener:[0,1,3,4,5,8,9,10,11],generation_func:8,german:0,get:[0,1,5,8,9],get_aggregative_estim:11,get_nprevpoints_approxim:[1,8],get_param:[3,8,9,11],get_probability_distribut:11,get_quapy_hom:8,github:[],given:[1,3,4,9,11],goe:4,good:[4,5],got:4,govern:1,gpu:9,grant:11,grid:[4,8,11],gridsearchcv:4,gridsearchq:[4,8],group:3,guarante:11,guez:[3,11],gzip:0,ha:[3,4,5,8,9],haberman:[0,3],handl:0,happen:[4,5],hard:3,harder:5,harmon:8,harri:0,hat:8,have:[0,1,2,3,4,5,10,11],hcr:[0,3,10],hdy:[6,11],held:[3,4,9],helling:11,hellingerdist:8,hellingerdistancei:[3,11],help:5,here:1,hidden:[5,9],hidden_s:9,hide:5,high:5,higher:[1,5],hlt:[],hold:6,home:10,hook:11,how:[0,1,3,4,5,11],howev:[0,4,5,11],hp:[0,3,4,10],html:[],http:[],hyper:[4,8,9],hyperparam:4,hyperparamet:[3,8,11],i:[0,1,3,4,5,8,9,10,11],id:[0,3,10],idf:0,ieee:0,ignor:[8,10,11],iid:[1,5,6],illustr:[3,4,5],imdb:[0,5,10],implement:[0,1,3,4,5,6,8,9,11],impos:4,improv:[3,9],includ:[0,1,3,5,6],inde:[3,4],independ:8,index:[0,3,6,9,10],indextransform:10,indic:[0,1,3,4,5,8,10,11],individu:[1,3],infer:0,inform:[0,1,3,4,8,10,11],infrequ:10,inherit:3,init:3,init_hidden:11,initi:[0,9],inplac:[1,3,10],input:[3,5,8,9],insight:5,inspir:3,instal:[0,3,6,9],instanc:[0,3,4,5,6,8,9,10,11],instanti:[0,1,3,4,9],instead:[1,3,4,11],integ:[3,9,10],integr:6,interest:[1,5,6,8],interestingli:5,interfac:[0,1],intern:[0,3,10],interpret:[5,6],interv:[1,5,8],introduc:1,invok:[0,1,3,8,10],involv:[2,5],io:[],ionospher:0,iri:0,irrespect:5,isaggreg:11,isbinari:[8,10,11],isometr:5,isprobabilist:11,isti:[],item:8,iter:[0,8,11],its:[3,4,8,9],itself:[3,11],j:[0,3,11],joachim:[3,9],job:[2,8],joblib:2,just:[1,3],k:[3,6,11],kei:8,kept:10,kernel:9,kernel_height:9,kfcv:[0,10,11],kindl:[0,1,3,5,10],kl:8,kld:[1,2,8,9],know:3,knowledg:[0,3,10,11],known:[0,3,4],kullback:[1,3,8],kwarg:[9,10,11],l1:11,label:[0,3,4,5,6,8,9,10,11],labelledcollect:[0,3,4,8,10,11],larg:4,largest:8,last:[1,3,5,9],lastli:3,latex:5,latinn:[3,11],latter:11,layer:[3,9],lead:1,learn:[1,2,3,4,6,8,9,11],learner:[3,4,9,11],least:[0,10],leav:10,legend:8,leibler:[1,3,8],length:9,less:[8,10],let:[1,3],level:11,leverag:3,like:[0,1,3,5,8,9],limit:[5,8],line:[1,3],linear:5,linear_model:[1,3,4,6,9],linearsvc:[3,5],linspac:5,list:[0,5,8,9,10],listedcolormap:8,literatur:[0,1,4,6],load:[0,3,8,10],loader:0,loader_func:[0,10],local:8,log:[8,10],logist:[1,3,9,11],logisticregress:[1,3,4,6,9],logscal:8,logspac:4,longer:8,longest:9,look:[0,1,3,5],loss:[6,9,11],low:[5,9],lower:[5,8],lower_is_bett:8,lowest:5,lowranklogisticregress:9,lr:[1,3,9,11],lstm:[3,9],lstm_class_nlay:9,lstm_hidden_s:11,lstm_nlayer:11,lstmnet:9,m:[3,8,11],machin:[1,4,6],macro:8,made:[0,2,11],mae:[1,4,6,8,9,11],mae_loss:11,main:5,maintain:[3,11],make:[0,1,3],mammograph:0,manag:[0,3,10],mani:[1,3,4,5,6,11],manner:0,manual:0,map:[1,9],map_parallel:8,margin:9,math:[],mathcal:8,matplotlib:[2,8],matric:[0,5,10],matrix:5,max:11,max_it:11,max_sample_s:11,maxim:6,maximum:[1,8,9],maximumlikelihoodprevalenceestim:11,md:[],mean:[0,1,3,4,5,6,8,9,10,11],mean_absolute_error:8,mean_relative_absolute_error:8,measur:[2,3,4,5,6,11],mediansweep2:11,mediansweep:11,member:3,memori:9,mention:3,merg:5,meta:[6,7,8],meth:[],method:[0,1,4,5,6,7,8],method_data:5,method_nam:[5,8],metric:[1,3,4,6,8],might:1,min_df:[1,3,4,5,10],min_po:11,mine:[0,3,11],minim:8,minimum:10,minimun:10,mining6:10,mixtur:3,mkld:[1,8,11],mnkld:[1,8,11],mock:9,modal:4,model:[0,1,5,6,8,9,11],model_select:[4,7],modifi:[3,8],modul:[0,1,3,5,6,7],moment:[0,3],more:[3,5,8],moreo:[0,3,4,10],most:[0,3,5,6,11],movi:0,mrae:[1,6,8,9,11],ms2:11,ms:11,mse:[1,3,6,8,11],msg:11,multiprocess:8,multivari:[3,9,11],must:3,my:[],my_arrai:8,my_custom_load:0,my_data:0,mycustomloss:3,n:[0,1,8,9],n_bin:[5,8],n_class:[1,3,8,9,10,11],n_compon:9,n_dimens:9,n_epoch:11,n_featur:9,n_instanc:9,n_job:[1,3,4,8,10,11],n_preval:[0,8,10],n_prevpoint:[1,4,5,8],n_repeat:[1,8],n_repetit:[1,4,5,8],n_sampl:[8,9],name:[5,8,9,10],nativ:6,natur:[1,8],natural_prevalence_predict:8,natural_prevalence_protocol:8,natural_prevalence_report:8,natural_sampling_gener:10,natural_sampling_index_gener:10,nbin:[5,8],ndarrai:[1,3,8,10,11],necessarili:11,need:[0,3,11],neg:[0,5,8],nest:[],net:9,network:[0,9,10,11],neural:[0,7,8,10],neuralclassifiertrain:[3,9],neutral:0,next:[4,8,9],nfold:[0,10],nkld:[1,2,6,8,9],nn:[9,11],nogap:10,non:[3,11],non_aggreg:[7,8],none:[1,4,8,9,10,11],nonetheless:4,nor:3,normal:[0,1,3,8,11],normalize_preval:8,note:[1,3,4,5],now:5,nowadai:3,np:[1,3,4,5,8],npp:8,nprevpoint:8,nrepeat:[0,10],num_prevalence_combin:[1,8],number:[0,1,3,5,8,9,10,11],numer:[0,1,3,6,10],numpi:[2,4,8,9,11],o_l6x_pcf09mdetq4tu7jk98mxfbgsxp9zso14jkuiyudgfg0:[],object:[0,8,9,10,11],observ:1,obtain:[1,4],occur:[5,10],occurr:10,octob:[0,3],off:9,offer:[3,6],older:2,omd:[0,10],ommit:1,onc:[1,3,5,8],one:[0,1,3,4,5,8,11],ones:[1,3,5,8,10],onevsal:[3,11],onli:[0,3,5,8,9,11],open:[0,6],oper:3,opt:4,optim:[2,3,4,8,9,11],optimize_threshold:11,option:[0,1,3,5,8,10,11],order:[0,2,3,5,8,10,11],order_bi:11,org:[],orient:[3,6,8,11],origin:[0,3,10,11],os:0,other:[1,3,5,6,8],otherwis:[0,3,11],our:[],out:[3,4,5,9],outcom:5,outer:8,output:[0,1,3,4,9,11],over:[3,4],overal:1,overestim:5,overrid:3,overridden:[3,11],own:4,p:[0,3,8,11],p_hat:8,pacc:[1,3,5,11],packag:[0,2,3,6,7],pad:9,pad_length:9,padding_length:9,page:[0,2,6],pageblock:0,pair:[0,8],panda:[1,2],paper:[0,3,11],parallel:[1,3,8],param:[4,8,9,10,11],param_grid:[4,8,11],param_mod_sel:11,param_model_sel:11,paramet:[1,3,4,8,9,10,11],part:[3,10],particular:[0,1,3],particularli:1,pass:[0,1,5,9,11],past:1,patch:[2,3,9],path:[0,3,5,8,9,10],patienc:[8,9,11],pattern:[3,11],pca:[],pcalr:[],pcc:[3,4,5,11],pd:1,pdf:5,peopl:[],perf:[6,9],perform:[1,3,4,5,6,8,9,11],phonem:0,pick:4,pickl:[3,8,10],pickle_path:8,pickled_resourc:8,pii:[],pip:2,pipelin:[],pkl:8,plai:0,plan:3,pleas:3,plot:[6,7],png:5,point:[0,1,3,8],polici:[3,11],popular:6,portion:4,pos_class:[8,10],posit:[0,3,5,8],possibl:[1,3,8],posterior:[3,8,9,11],posterior_prob:[3,11],postpon:3,potter:0,pp:[0,3],practic:[0,4],pre:[0,3],prec:[0,8],precis:[0,1,8],preclassifi:3,predict:[3,4,5,8,9,11],predict_proba:[3,9,11],predictor:1,prepare_svmperf:[2,3],preprint:4,preprocess:[0,1,3,7,8],present:[0,3,10],preserv:[1,5],pretti:5,prev:[0,1,8,10],prevail:3,preval:[0,1,3,4,5,6,8,10,11],prevalence_estim:8,prevalence_from_label:8,prevalence_from_prob:8,prevalence_linspac:8,prevel:11,previou:3,previous:11,prevs_estim:11,prevs_hat:[1,8],princip:9,print:[0,1,3,4,6,9],prior:[1,3,4,5,6],priori:[3,11],probabilist:[3,11],probabilisticadjustedclassifyandcount:11,probabilisticclassifyandcount:11,probabl:[1,3,4,5,6,9,11],problem:[0,3,5,11],procedur:[3,6,11],proceed:[0,3,10],process:[3,4,8],processor:3,procol:1,produc:[0,1,5,8],product:3,progress:8,properli:0,properti:[3,8,9,10,11],proport:[3,4,8,9,11],propos:[2,3,11],protocl:8,protocol:[0,3,4,5,6,8],provid:[0,3,5,6],ptecondestim:11,ptr:[3,11],ptr_polici:11,purpos:[0,11],python:[0,6],pytorch:2,q:[0,2,3,8,9],qacc:9,qdrop_p:11,qf1:9,qgm:9,qp:[0,1,3,4,5,6,8],quanet:[2,6,9,11],quanetmodul:11,quanettrain:11,quantif:[0,1,6,8,9,10,11],quantifi:[3,4,5,6,8,11],quantification_error:8,quantiti:8,quapi:[0,1,2,3,4,5],quapy_data:0,quay_data:10,quevedo:[0,3,11],quick:[],r:[0,3,11],rac:[],rae:[1,2,8],rais:[3,8],rand:8,random:[1,3,4,5,8],random_se:[1,8],random_st:10,randomli:0,rang:[0,5],rank:[3,9],rare:10,rate:[3,9],rather:[1,4],raw:10,rb:0,re:[3,4,10],read:10,reader:[7,8],readm:[],real:[9,10],reason:[3,5,6],recal:8,receiv:[0,3,5],recip:11,recognit:[3,11],recommend:[1,5],recurr:[0,3,10],red:0,red_siz:[3,11],reduc:[0,10],reduce_column:[0,10],refer:[9,10],refit:[4,8],regard:4,regist:11,regress:9,regressor:[1,3,11],reindex_label:10,reiniti:9,rel:[1,3,8],relative_absolute_error:8,reli:[1,3],reliabl:[3,11],rememb:5,remov:10,repeat:[8,10],repetit:8,repl:10,replac:[0,3,10],replic:[1,4,8],report:1,repositori:0,repr_siz:9,repres:[1,3,5,10,11],represent:[0,3,9],request:[0,8,11],requir:[0,1,3,6,9],reset_net_param:9,resourc:8,respect:[0,1,5,8,11],respond:3,rest:[10,11],result:[1,2,3,4,5,6,11],retain:[0,3,9],retrain:4,return_constrained_dim:8,reus:[0,3,8],review:[5,6,10],reviews_sentiment_dataset:0,rewrit:5,right:4,role:0,root:6,roughli:0,routin:8,row:10,run:[0,1,2,3,4,5,8,11],s003132031400291x:[],s:[0,1,3,4,5,8,9,10],saeren:[3,11],sai:11,said:3,same:[0,3,5,10],sampl:[0,1,3,4,5,6,8,9,10,11],sample_s:[0,1,3,4,5,8,10,11],sampling_from_index:[0,10],sampling_index:[0,10],sander:[0,10],save:[5,8],save_or_show:8,save_text_fil:8,savepath:[5,8],scall:10,scenario:[1,3,4,5,6],scienc:[3,11],sciencedirect:[],scikit:[2,3,4],scipi:[2,10],score:[0,1,4,9,10],script:[1,2,3,6],se:[1,8],search:[3,4,6,8,11],sebastiani:[0,3,4,10,11],second:[0,1,3,5,8],section:4,see:[0,1,2,3,4,5,6,8,9],seed:[1,4,8],seem:3,seemingli:5,seen:5,select:[0,3,6,8,11],selector:3,self:[3,9,10,11],semeion:0,semev:0,semeval13:[0,10],semeval14:[0,10],semeval15:[0,10],semeval16:[0,6,10],sentenc:10,sentiment:[3,6,10,11],separ:[8,10],seri:0,serv:3,set:[0,1,3,4,5,6,8,9,10,11],set_opt:1,set_param:[3,8,9,11],set_siz:[],sever:0,sh:[2,3],shape:[5,8,9],share:[0,10],shift:[1,4,6,8],shorter:9,shoud:3,should:[0,1,3,4,5,6,9,10,11],show:[0,1,3,4,5,8,9,10],show_std:[5,8],showcas:5,shown:[1,5],shuffl:[9,10],signific:1,silent:[8,11],similar:11,simpl:[0,3,5,11],simplest:3,simplex:[0,8],simpli:[1,2,3,4,5,6,8,11],sinc:[0,1,3,5,8,11],singl:[1,3,6,11],size:[0,1,3,8,9,10,11],sklearn:[1,3,4,5,6,9,10,11],sld:3,slice:8,smooth:[1,8],smooth_limits_epsilon:8,so:[0,1,3,5,8,9,11],social:[0,3,10,11],soft:3,softwar:0,solid:5,solv:4,solve_adjust:11,some:[0,1,3,5],some_arrai:8,sometim:1,sonar:0,sourc:[2,3,6,9],sout:11,space:[0,4,9],spambas:0,spars:[0,10],special:[0,5,10],specif:[3,4],specifi:[0,1,3,5,8,9,10,11],spectf:0,spectrum:[0,1,4,5],speed:3,split:[0,3,4,5,9,10,11],split_stratifi:10,splitstratifi:10,spmatrix:10,squar:[1,3,8],sst:[0,10],stabil:1,stand:8,standard:[0,1,5,10],start:4,stat:10,state:8,statist:[0,1,11],stats_siz:11,std:9,stdout:8,step:[5,8],stop:9,store:[0,9,10],str:[0,8,10],strategi:[3,4],stratifi:[0,3],stride:9,string:[1,8,10],strongli:[4,5],strprev:[0,1,8],structur:3,studi:[0,3,11],subclass:11,subinterv:5,sublinear_tf:10,submit:0,submodul:7,subobject:[],suboptim:4,subpackag:7,subsequ:[10,11],subtract:[0,8],subtyp:10,suffic:5,suffici:11,sum:11,sum_:8,summar:0,supervis:[4,6],support:[3,6,9],surpass:1,svm:[3,5,6,9],svm_light:[],svm_perf:[],svm_perf_classifi:9,svm_perf_learn:9,svm_perf_quantif:[2,3],svmae:[3,11],svmkld:[3,11],svmnkld:[3,11],svmperf:[2,3,7,8],svmperf_bas:[9,11],svmperf_hom:3,svmq:[3,11],svmrae:[3,11],syntax:5,system:4,t50:11,t:[0,1,3,8],take:[0,3,5,8,11],taken:[3,8,9],target:[3,5,6,8,9,11],task:[3,4,11],temp_se:8,tempor:8,tend:5,tendenc:5,tensor:9,term:[0,1,3,4,5,6,8,9,10,11],test:[0,1,3,4,5,6,8,9,10,11],test_bas:[],test_dataset:[],test_method:[],test_path:[0,10],test_sampl:8,test_split:10,text2tfidf:[0,1,3,10],text:[0,3,8,9,10,11],textclassifiernet:9,textual:[0,6,10],tf:[0,10],tfidf:[0,4,5,10],tfidfvector:10,than:[1,4,5,8,9,10],thei:[0,3],them:[0,3,11],theoret:4,thereaft:1,thi:[0,1,2,3,4,5,6,8,9,11],thing:3,third:[1,5],thorsten:9,those:[1,3,4,5,8,9],though:3,three:[0,5],thresholdoptim:11,through:[3,8],thu:[3,4,5,8,11],tictacto:0,time:[0,1,3,8,10],timeout:8,timeouterror:8,timer:8,titl:8,tj:[],tn:[8,11],token:[0,9,10],tool:[1,6],top:[3,11],torch:[3,9,11],torchdataset:9,toward:5,tp:[8,11],tpr:8,tqdm:2,tr_iter_per_poch:11,tr_prev:[5,8,11],trade:9,tradition:1,train:[0,1,3,4,5,6,8,9,10,11],train_path:[0,10],train_prev:[5,8],train_prop:10,train_siz:10,train_val_split:11,trainer:9,training_help:11,training_preval:5,training_s:5,transact:[3,11],transform:[0,9,10],transfus:0,trivial:3,true_prev:[1,5,8],true_preval:6,truncatedsvd:9,turn:4,tweet:[0,3,10,11],twitter:[6,10],twitter_sentiment_datasets_test:0,twitter_sentiment_datasets_train:0,two:[0,1,3,4,5,8],type:[0,3],typic:[1,4,5,8,9],uci:6,unabl:0,unadjust:5,unalt:9,unbias:5,uncompress:0,under:1,underestim:5,underlin:8,unfortun:5,unifi:0,uniform_prevalence_sampl:8,uniform_sampl:10,uniform_sampling_index:10,uniform_simplex_sampl:8,uniformli:8,union:[8,11],uniqu:10,unit:0,unix:0,unk:10,unless:11,unlik:[1,4],unus:[8,9,11],up:[3,4,8,9,11],updat:[],url:8,us:[0,1,3,4,5,6,8,9,10,11],user:[0,1,5],utf:10,util:[7,9],v:[3,11],va_iter_per_poch:11,val:[0,10],val_split:[3,4,8,9,11],valid:[0,1,3,4,5,8,9,10,11],valid_loss:[3,9],valid_polici:11,valu:[0,1,3,8,9,10,11],variabl:[1,3,5,8],varianc:[0,5],variant:[5,6,11],varieti:4,variou:[1,5],vector:[0,8,9,10],verbos:[0,1,4,8,9,10,11],veri:[3,5],versatil:6,version:[2,9],vertical_xtick:8,via:[0,2,3,11],view:5,visual:[5,6],vocab_s:9,vocabulari:[9,10],vocabulary_s:[3,9,10],vs:3,w:[0,3,10,11],wa:[0,3,5,10,11],wai:[1,11],wait:9,want:[3,4],warn:10,wb:[0,10],wdbc:0,we:[0,1,3,4,5,6],weight:[9,10],weight_decai:9,well:[0,3,4,5],were:0,what:3,when:[0,1,3,4,5,8,9],whenev:[5,8],where:[3,5,8,9,10,11],wherebi:4,whether:[8,9,10,11],which:[0,1,3,4,5,8,9,10,11],white:0,whole:[0,1,3,4,8],why:3,wide:5,wiki:[0,3],wine:0,within:[8,11],without:[1,3,8],word:[1,3,6,9,10],work:[1,3,4,5],worker:1,wors:[4,5],would:[0,1,3,5,6,8,11],wrapper:[8,9],written:6,www:[],x:[5,8,9,10,11],xavier:9,xavier_uniform:9,xlrd:[0,2],xy:10,y:[5,8,9,10,11],y_:11,y_pred:8,y_true:8,ye:10,yeast:0,yield:[5,8],yin:[],you:[2,3],your:3,z:0,zero:[0,8],zfthyovrzwxmgfzylqw_y8cagg:[],zip:[0,5]},titles:["Datasets","Evaluation","Installation","Quantification Methods","Model Selection","Plotting","Welcome to QuaPy\u2019s documentation!","quapy","quapy package","quapy.classification package","quapy.data package","quapy.method package"],titleterms:{"function":8,A:6,The:3,ad:0,aggreg:[3,11],base:[10,11],bia:5,classif:[4,9],classifi:3,content:[6,8,9,10,11],count:3,custom:0,data:[0,10],dataset:[0,10],diagon:5,distanc:3,document:6,drift:5,emq:3,ensembl:3,error:[1,5,8],evalu:[1,8],ex:[],exampl:6,expect:3,explicit:3,featur:6,get:[],hdy:3,helling:3,indic:6,instal:2,introduct:6,issu:0,learn:0,loss:[2,3,4],machin:0,maxim:3,measur:1,meta:[3,11],method:[3,9,11],minim:3,model:[3,4],model_select:8,modul:[8,9,10,11],network:3,neural:[3,9,11],non_aggreg:11,orient:[2,4],packag:[8,9,10,11],perf:2,plot:[5,8],preprocess:10,process:0,protocol:1,quanet:3,quantif:[2,3,4,5],quapi:[6,7,8,9,10,11],quick:6,reader:10,readm:[],requir:2,review:0,s:6,select:4,sentiment:0,start:[],submodul:[8,9,10,11],subpackag:8,svm:2,svmperf:9,tabl:6,target:4,test:[],test_bas:[],test_dataset:[],test_method:[],titl:[],twitter:0,uci:0,util:8,variant:3,welcom:6,y:3}}) \ No newline at end of file +Search.setIndex({docnames:["Datasets","Evaluation","Installation","Methods","Model-Selection","Plotting","index","modules","quapy","quapy.classification","quapy.data","quapy.method"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["Datasets.md","Evaluation.md","Installation.rst","Methods.md","Model-Selection.md","Plotting.md","index.rst","modules.rst","quapy.rst","quapy.classification.rst","quapy.data.rst","quapy.method.rst"],objects:{"":{quapy:[8,0,0,"-"]},"quapy.classification":{methods:[9,0,0,"-"],neural:[9,0,0,"-"],svmperf:[9,0,0,"-"]},"quapy.classification.methods":{LowRankLogisticRegression:[9,1,1,""]},"quapy.classification.methods.LowRankLogisticRegression":{fit:[9,2,1,""],get_params:[9,2,1,""],predict:[9,2,1,""],predict_proba:[9,2,1,""],set_params:[9,2,1,""],transform:[9,2,1,""]},"quapy.classification.neural":{CNNnet:[9,1,1,""],LSTMnet:[9,1,1,""],NeuralClassifierTrainer:[9,1,1,""],TextClassifierNet:[9,1,1,""],TorchDataset:[9,1,1,""]},"quapy.classification.neural.CNNnet":{document_embedding:[9,2,1,""],get_params:[9,2,1,""],vocabulary_size:[9,3,1,""]},"quapy.classification.neural.LSTMnet":{document_embedding:[9,2,1,""],get_params:[9,2,1,""],vocabulary_size:[9,3,1,""]},"quapy.classification.neural.NeuralClassifierTrainer":{device:[9,3,1,""],fit:[9,2,1,""],get_params:[9,2,1,""],predict:[9,2,1,""],predict_proba:[9,2,1,""],reset_net_params:[9,2,1,""],set_params:[9,2,1,""],transform:[9,2,1,""]},"quapy.classification.neural.TextClassifierNet":{dimensions:[9,2,1,""],document_embedding:[9,2,1,""],forward:[9,2,1,""],get_params:[9,2,1,""],predict_proba:[9,2,1,""],vocabulary_size:[9,3,1,""],xavier_uniform:[9,2,1,""]},"quapy.classification.neural.TorchDataset":{asDataloader:[9,2,1,""]},"quapy.classification.svmperf":{SVMperf:[9,1,1,""]},"quapy.classification.svmperf.SVMperf":{decision_function:[9,2,1,""],fit:[9,2,1,""],predict:[9,2,1,""],set_params:[9,2,1,""],valid_losses:[9,4,1,""]},"quapy.data":{base:[10,0,0,"-"],datasets:[10,0,0,"-"],preprocessing:[10,0,0,"-"],reader:[10,0,0,"-"]},"quapy.data.base":{Dataset:[10,1,1,""],LabelledCollection:[10,1,1,""],isbinary:[10,5,1,""]},"quapy.data.base.Dataset":{SplitStratified:[10,2,1,""],binary:[10,3,1,""],classes_:[10,3,1,""],kFCV:[10,2,1,""],load:[10,2,1,""],n_classes:[10,3,1,""],stats:[10,2,1,""],vocabulary_size:[10,3,1,""]},"quapy.data.base.LabelledCollection":{Xy:[10,3,1,""],artificial_sampling_generator:[10,2,1,""],artificial_sampling_index_generator:[10,2,1,""],binary:[10,3,1,""],counts:[10,2,1,""],kFCV:[10,2,1,""],load:[10,2,1,""],n_classes:[10,3,1,""],natural_sampling_generator:[10,2,1,""],natural_sampling_index_generator:[10,2,1,""],prevalence:[10,2,1,""],sampling:[10,2,1,""],sampling_from_index:[10,2,1,""],sampling_index:[10,2,1,""],split_stratified:[10,2,1,""],stats:[10,2,1,""],uniform_sampling:[10,2,1,""],uniform_sampling_index:[10,2,1,""]},"quapy.data.datasets":{df_replace:[10,5,1,""],fetch_UCIDataset:[10,5,1,""],fetch_UCILabelledCollection:[10,5,1,""],fetch_reviews:[10,5,1,""],fetch_twitter:[10,5,1,""],warn:[10,5,1,""]},"quapy.data.preprocessing":{IndexTransformer:[10,1,1,""],index:[10,5,1,""],reduce_columns:[10,5,1,""],standardize:[10,5,1,""],text2tfidf:[10,5,1,""]},"quapy.data.preprocessing.IndexTransformer":{add_word:[10,2,1,""],fit:[10,2,1,""],fit_transform:[10,2,1,""],index:[10,2,1,""],transform:[10,2,1,""],vocabulary_size:[10,2,1,""]},"quapy.data.reader":{binarize:[10,5,1,""],from_csv:[10,5,1,""],from_sparse:[10,5,1,""],from_text:[10,5,1,""],reindex_labels:[10,5,1,""]},"quapy.error":{absolute_error:[8,5,1,""],acc_error:[8,5,1,""],acce:[8,5,1,""],ae:[8,5,1,""],f1_error:[8,5,1,""],f1e:[8,5,1,""],from_name:[8,5,1,""],kld:[8,5,1,""],mae:[8,5,1,""],mean_absolute_error:[8,5,1,""],mean_relative_absolute_error:[8,5,1,""],mkld:[8,5,1,""],mnkld:[8,5,1,""],mrae:[8,5,1,""],mse:[8,5,1,""],nkld:[8,5,1,""],rae:[8,5,1,""],relative_absolute_error:[8,5,1,""],se:[8,5,1,""],smooth:[8,5,1,""]},"quapy.evaluation":{artificial_prevalence_prediction:[8,5,1,""],artificial_prevalence_protocol:[8,5,1,""],artificial_prevalence_report:[8,5,1,""],evaluate:[8,5,1,""],gen_prevalence_prediction:[8,5,1,""],natural_prevalence_prediction:[8,5,1,""],natural_prevalence_protocol:[8,5,1,""],natural_prevalence_report:[8,5,1,""]},"quapy.functional":{HellingerDistance:[8,5,1,""],adjusted_quantification:[8,5,1,""],artificial_prevalence_sampling:[8,5,1,""],get_nprevpoints_approximation:[8,5,1,""],normalize_prevalence:[8,5,1,""],num_prevalence_combinations:[8,5,1,""],prevalence_from_labels:[8,5,1,""],prevalence_from_probabilities:[8,5,1,""],prevalence_linspace:[8,5,1,""],strprev:[8,5,1,""],uniform_prevalence_sampling:[8,5,1,""],uniform_simplex_sampling:[8,5,1,""]},"quapy.method":{aggregative:[11,0,0,"-"],base:[11,0,0,"-"],meta:[11,0,0,"-"],neural:[11,0,0,"-"],non_aggregative:[11,0,0,"-"]},"quapy.method.aggregative":{ACC:[11,1,1,""],AdjustedClassifyAndCount:[11,4,1,""],AggregativeProbabilisticQuantifier:[11,1,1,""],AggregativeQuantifier:[11,1,1,""],CC:[11,1,1,""],ClassifyAndCount:[11,4,1,""],ELM:[11,1,1,""],EMQ:[11,1,1,""],ExpectationMaximizationQuantifier:[11,4,1,""],ExplicitLossMinimisation:[11,4,1,""],HDy:[11,1,1,""],HellingerDistanceY:[11,4,1,""],MAX:[11,1,1,""],MS2:[11,1,1,""],MS:[11,1,1,""],MedianSweep2:[11,4,1,""],MedianSweep:[11,4,1,""],OneVsAll:[11,1,1,""],PACC:[11,1,1,""],PCC:[11,1,1,""],ProbabilisticAdjustedClassifyAndCount:[11,4,1,""],ProbabilisticClassifyAndCount:[11,4,1,""],SVMAE:[11,1,1,""],SVMKLD:[11,1,1,""],SVMNKLD:[11,1,1,""],SVMQ:[11,1,1,""],SVMRAE:[11,1,1,""],T50:[11,1,1,""],ThresholdOptimization:[11,1,1,""],X:[11,1,1,""],training_helper:[11,5,1,""]},"quapy.method.aggregative.ACC":{aggregate:[11,2,1,""],classify:[11,2,1,""],fit:[11,2,1,""],solve_adjustment:[11,2,1,""]},"quapy.method.aggregative.AggregativeProbabilisticQuantifier":{posterior_probabilities:[11,2,1,""],predict_proba:[11,2,1,""],probabilistic:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.aggregative.AggregativeQuantifier":{aggregate:[11,2,1,""],aggregative:[11,3,1,""],classes_:[11,3,1,""],classify:[11,2,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],learner:[11,3,1,""],n_classes:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.aggregative.CC":{aggregate:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.ELM":{aggregate:[11,2,1,""],classify:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.EMQ":{EM:[11,2,1,""],EPSILON:[11,4,1,""],MAX_ITER:[11,4,1,""],aggregate:[11,2,1,""],fit:[11,2,1,""],predict_proba:[11,2,1,""]},"quapy.method.aggregative.HDy":{aggregate:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.MS":{optimize_threshold:[11,2,1,""]},"quapy.method.aggregative.MS2":{optimize_threshold:[11,2,1,""]},"quapy.method.aggregative.OneVsAll":{aggregate:[11,2,1,""],binary:[11,3,1,""],classes_:[11,3,1,""],classify:[11,2,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],posterior_probabilities:[11,2,1,""],probabilistic:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.aggregative.PACC":{aggregate:[11,2,1,""],classify:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.PCC":{aggregate:[11,2,1,""],fit:[11,2,1,""]},"quapy.method.aggregative.ThresholdOptimization":{aggregate:[11,2,1,""],compute_fpr:[11,2,1,""],compute_table:[11,2,1,""],compute_tpr:[11,2,1,""],fit:[11,2,1,""],optimize_threshold:[11,2,1,""]},"quapy.method.base":{BaseQuantifier:[11,1,1,""],BinaryQuantifier:[11,1,1,""],isaggregative:[11,5,1,""],isbinary:[11,5,1,""],isprobabilistic:[11,5,1,""]},"quapy.method.base.BaseQuantifier":{aggregative:[11,3,1,""],binary:[11,3,1,""],classes_:[11,3,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],probabilistic:[11,3,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.base.BinaryQuantifier":{binary:[11,3,1,""]},"quapy.method.meta":{EACC:[11,5,1,""],ECC:[11,5,1,""],EEMQ:[11,5,1,""],EHDy:[11,5,1,""],EPACC:[11,5,1,""],Ensemble:[11,1,1,""],ensembleFactory:[11,5,1,""],get_probability_distribution:[11,5,1,""]},"quapy.method.meta.Ensemble":{VALID_POLICIES:[11,4,1,""],accuracy_policy:[11,2,1,""],aggregative:[11,3,1,""],binary:[11,3,1,""],classes_:[11,3,1,""],ds_policy:[11,2,1,""],ds_policy_get_posteriors:[11,2,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],probabilistic:[11,3,1,""],ptr_policy:[11,2,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""],sout:[11,2,1,""]},"quapy.method.neural":{QuaNetModule:[11,1,1,""],QuaNetTrainer:[11,1,1,""],mae_loss:[11,5,1,""]},"quapy.method.neural.QuaNetModule":{device:[11,3,1,""],forward:[11,2,1,""],init_hidden:[11,2,1,""]},"quapy.method.neural.QuaNetTrainer":{classes_:[11,3,1,""],clean_checkpoint:[11,2,1,""],clean_checkpoint_dir:[11,2,1,""],epoch:[11,2,1,""],fit:[11,2,1,""],get_aggregative_estims:[11,2,1,""],get_params:[11,2,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.method.non_aggregative":{MaximumLikelihoodPrevalenceEstimation:[11,1,1,""]},"quapy.method.non_aggregative.MaximumLikelihoodPrevalenceEstimation":{classes_:[11,3,1,""],fit:[11,2,1,""],get_params:[11,2,1,""],quantify:[11,2,1,""],set_params:[11,2,1,""]},"quapy.model_selection":{GridSearchQ:[8,1,1,""]},"quapy.model_selection.GridSearchQ":{best_model:[8,2,1,""],classes_:[8,3,1,""],fit:[8,2,1,""],get_params:[8,2,1,""],quantify:[8,2,1,""],set_params:[8,2,1,""]},"quapy.plot":{binary_bias_bins:[8,5,1,""],binary_bias_global:[8,5,1,""],binary_diagonal:[8,5,1,""],brokenbar_supremacy_by_drift:[8,5,1,""],error_by_drift:[8,5,1,""],save_or_show:[8,5,1,""]},"quapy.util":{EarlyStop:[8,1,1,""],create_if_not_exist:[8,5,1,""],create_parent_dir:[8,5,1,""],download_file:[8,5,1,""],download_file_if_not_exists:[8,5,1,""],get_quapy_home:[8,5,1,""],map_parallel:[8,5,1,""],parallel:[8,5,1,""],pickled_resource:[8,5,1,""],save_text_file:[8,5,1,""],temp_seed:[8,5,1,""]},quapy:{classification:[9,0,0,"-"],data:[10,0,0,"-"],error:[8,0,0,"-"],evaluation:[8,0,0,"-"],functional:[8,0,0,"-"],isbinary:[8,5,1,""],method:[11,0,0,"-"],model_selection:[8,0,0,"-"],plot:[8,0,0,"-"],util:[8,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","property","Python property"],"4":["py","attribute","Python attribute"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:property","4":"py:attribute","5":"py:function"},terms:{"0":[0,1,3,4,5,8,9,10,11],"00":[0,1,4,8],"000":1,"0001":[4,11],"000e":1,"001":[4,9,11],"005":8,"009":1,"01":[8,9,11],"017":1,"018":0,"02":1,"021":0,"02552":4,"03":1,"034":1,"035":1,"037":1,"04":1,"041":1,"042":1,"046":1,"048":1,"05":[5,8],"055":1,"063":0,"065":0,"070":1,"073":1,"075":1,"078":0,"081":0,"082":[0,1],"083":0,"086":0,"091":1,"099":0,"1":[0,1,3,4,5,8,9,10,11],"10":[0,1,4,5,8,9,11],"100":[0,1,3,4,5,9,10,11],"1000":[0,4,11],"10000":4,"100000":4,"101":[4,10],"1010":4,"1024":11,"104":0,"108":1,"109":0,"11":[0,1,6],"11338":0,"114":1,"1145":[],"12":9,"120":0,"1215742":0,"1271":0,"13":[0,9],"139":0,"14":[3,11],"142":1,"146":[3,11],"1473":0,"148":0,"1484":0,"15":[3,8,11],"150":0,"153":0,"157":0,"158":0,"159":0,"1593":0,"1594":0,"1599":0,"161":0,"163":[0,1],"164":[0,3,11],"167":0,"17":0,"1771":1,"1775":[0,3],"1778":[0,3],"178":0,"1823":0,"1839":0,"18399":0,"1853":0,"19":[3,10,11],"193":0,"199151":0,"19982":4,"1e":9,"1st":0,"2":[0,1,3,5,8,10,11],"20":[5,8,11],"200":[1,9],"2000":0,"2002":[3,11],"2011":4,"2013":[3,11],"2015":[0,2,3,9,11],"2016":[3,10,11],"2017":[0,3,11],"2018":[0,3,10],"2019":[3,11],"2020":4,"20342":4,"206":0,"207":0,"208":0,"21":[1,3,5,8,11],"210":8,"211":0,"2126":0,"2155":0,"21591":0,"218":[3,11],"2184":0,"219e":1,"22":[0,3,9,10,11],"222":0,"222046":0,"226":0,"229":1,"229399":0,"23":9,"235":1,"238":0,"2390":0,"24":[0,9],"243":0,"248563":0,"24866":4,"24987":4,"25":[0,5,8,9,11],"25000":0,"256":[0,9],"26":9,"261":0,"265":0,"266":0,"267":0,"27":[1,3,9,11],"270":0,"2700406":[],"271":0,"272":0,"274":0,"275":1,"27th":[0,3,10],"28":3,"280":0,"281":0,"282":0,"283":[0,1],"288":0,"289":0,"2971":0,"2nd":0,"2t":[1,8],"2tp":8,"2x5fcv":0,"3":[0,1,3,5,6,8,9,10,11],"30":[0,1,3,11],"300":[0,1,9],"305":0,"306":0,"312":0,"32":[0,6],"33":[0,5],"331":0,"333":0,"335":0,"337":0,"34":[0,3,11],"341":0,"346":1,"347":0,"350":0,"351":0,"357":1,"359":0,"361":0,"366":1,"372":0,"373":0,"376132":0,"3765":0,"3813":0,"3821":0,"383e":1,"387e":1,"392":0,"394":0,"399":0,"3f":[1,6],"3rd":0,"4":[0,1,3,4,5,8,11],"40":[0,3,4,11],"404333":0,"407":0,"41":[3,11],"412":0,"412e":1,"413":0,"414":0,"417":0,"41734":4,"42":[1,8],"421":0,"4259":0,"426e":1,"427":0,"430":0,"434":0,"435":1,"43676":4,"437":0,"44":0,"446":0,"45":[3,5,11],"452":0,"459":1,"4601":0,"461":0,"463":0,"465":0,"466":0,"470":0,"48":[3,11],"481":0,"48135":4,"486":0,"4898":0,"492":0,"496":0,"4960":1,"497":0,"5":[0,1,3,4,5,8,9,10,11],"50":[0,5,8,11],"500":[0,1,4,5,11],"5000":[1,5],"5005":4,"507":0,"508":0,"512":[9,11],"514":0,"515e":1,"530":0,"534":0,"535":0,"535e":1,"5379":4,"539":0,"541":1,"546":0,"5473":0,"54it":4,"55":5,"55it":4,"565":1,"569":0,"57":0,"573":0,"578":1,"583":0,"591":[3,11],"5f":4,"5fcv":11,"6":[0,1,3,5,8,10,11],"60":0,"600":1,"601":0,"604":[3,11],"606":0,"625":0,"627":0,"633e":1,"634":1,"64":[9,11],"640":0,"641":0,"650":0,"653":0,"654":1,"66":[1,11],"665":0,"667":0,"669":0,"67":5,"683":0,"688":0,"691":0,"694582":0,"7":[1,5,9],"70":0,"700":0,"701e":1,"711":0,"717":1,"725":1,"730":0,"735":0,"740e":1,"748":0,"75":[0,5,8],"762":0,"774":0,"778":0,"787":0,"794":0,"798":0,"8":[0,1,5,10,11],"8000":0,"830":0,"837":1,"858":1,"861":0,"87":[0,3,11],"8788":0,"889504":0,"8d2fhsgcvn0aaaaa":[],"9":[0,1,3,5,11],"90":[5,8],"901":0,"909":1,"914":1,"917":0,"919":0,"922":0,"923":0,"935":0,"936":0,"937":0,"945":1,"95":8,"9533":0,"958":0,"97":0,"979":0,"982":0,"99":8,"abstract":[3,9,11],"case":[0,1,3,4,5,8,11],"class":[0,1,3,4,5,6,8,9,10,11],"d\u00edez":[3,11],"default":[1,3,8,9,10],"do":[0,1,3,4,8,9],"final":[1,3,5],"float":[0,3,8,9,10,11],"function":[0,1,3,4,5,6,7,9,11],"g\u00e1llego":[0,3,11],"gonz\u00e1lez":[3,11],"import":[0,1,3,4,5,6],"int":[0,5,8,10,11],"long":[4,9],"new":[0,3,10,11],"p\u00e9rez":[0,3,11],"return":[0,1,3,4,5,8,9,10,11],"rodr\u0131":[3,11],"short":9,"static":[3,11],"true":[0,1,3,4,5,6,8,9,10,11],"try":4,"while":[3,5,8,9,11],A:[0,3,8,9,10,11],As:[3,4],By:[1,3,8],For:[0,1,5,6,8,11],If:[3,5,8,11],In:[0,1,2,3,4,5,6,9,11],It:[3,4,5,8],One:[0,1,3,11],That:[1,4],The:[0,1,2,4,5,6,8,9,10,11],Then:3,These:0,To:[5,10],_:5,__:[],__class__:5,__name__:5,_adjust:[],_ae_:[],_classify_:11,_error_name_:11,_fit_learner_:11,_kld_:[],_labelledcollection_:11,_learner_:11,_mean:[],_min_df_:10,_my:[],_nkld_:[],_posterior_probabilities_:11,_q_:[],_rae_:[],_svmperf_:[],ab:[],aboud:3,about:[0,5],abov:[0,3,5],absolut:[1,3,5,6,8],absolute_error:8,abstractmethod:3,acc:[1,3,5,6,8,11],acc_error:8,accept:3,access:[0,3],accommod:0,accord:[1,3,4,8,9],accordingli:5,accuraci:[1,5,8],accuracy_polici:11,achiev:[1,3,4,5],acm:[0,3,10,11],across:[0,1,4,5,6,8],action:[0,11],acut:0,ad:6,add:[3,4,8],add_word:10,addit:3,addition:[0,11],adjust:[3,6,11],adjusted_quantif:8,adjustedclassifyandcount:11,adopt:[3,4],advanc:[0,6],advantag:3,ae:[1,2,5,8],ae_:1,affect:8,afterward:11,again:5,against:5,aggreg:[1,4,5,6,7,8],aggregativeprobabilisticquantifi:[3,11],aggregativequantifi:[3,11],aggregg:11,aim:[4,5],al:[0,2,9],alaiz:[3,11],alegr:[3,11],alejandro:4,alia:[3,11],all:[0,1,2,3,5,8,11],allia:3,alloc:9,allow:[0,1,2,3,5,8,9,10,11],almost:3,along:[0,3,8,11],alreadi:[3,11],also:[0,1,2,3,5,6,9],altern:4,although:[3,4,5,11],alwai:[3,4,5],among:3,an:[0,1,2,3,4,5,6,8,9,11],analys:[5,6],analysi:[0,3,6,10,11],analyz:5,ani:[0,1,3,4,5,6,8,9,10,11],anoth:[0,1,3,5],anyon:0,api:6,app:8,appeal:1,appear:5,append:5,appli:[2,3,4,5,8,9,10],appropri:4,approxim:[1,5,9,11],ar:[0,1,3,4,5,8,9,10,11],archive_filenam:8,archive_path:8,arg:[8,10,11],args_i:8,argu:4,argument:[0,1,3,5],arifici:8,aris:1,around:1,arrai:[1,3,5,8,9,10],articl:[3,4,11],artifici:[0,1,3,4,5,6,8],artificial_prevalence_predict:8,artificial_prevalence_protocol:8,artificial_prevalence_report:8,artificial_prevalence_sampl:8,artificial_sampling_ev:[1,4],artificial_sampling_gener:[0,10],artificial_sampling_index_gener:10,artificial_sampling_predict:[1,5],artificial_sampling_report:1,arxiv:4,asarrai:1,asdataload:9,asonam:0,assess:4,assign:[3,8],associ:[8,10],assum:[1,6,11],assumpt:[1,5,6],astyp:10,attempt:3,attribut:11,august:0,autom:[0,3,6],automat:[0,1],av:[3,11],avail:[0,1,2,3,5,6,9],averag:[1,3,8],avoid:1,axi:[5,8],b:[0,10],balanc:[0,4],band:[5,8],bar:8,barranquero:[2,3,9,11],base:[0,3,6,7,8,9],base_classifi:5,base_estim:3,base_quantifier_class:11,baseestim:[9,11],baselin:6,basequantifi:[3,8,11],basic:[5,11],batch:9,batch_siz:9,batch_size_test:9,been:[0,3,4,5,8,10,11],befor:[3,9,11],beforehand:8,behav:[3,5],being:[4,8],belief:1,belong:3,below:[0,2,3,5,10],best:[4,8,9,11],best_model:8,best_model_:4,best_params_:4,better:4,between:[4,5,6,8,9],beyond:5,bia:[6,8],bias:5,bidirect:11,bin:[5,8,11],bin_bia:5,bin_diag:5,binar:[8,10],binari:[3,5,6,8,9,10,11],binary_bias_bin:[5,8],binary_bias_glob:[5,8],binary_diagon:[5,8],binary_quantifi:11,binaryquantifi:11,block:0,bool:[8,11],both:5,bound:8,box:[5,8],breast:0,brief:1,broken:5,brokenbar_supremacy_by_drift:8,budg:1,budget:[1,4],build:11,bypass:11,c:[3,4,9,10,11],calibr:3,calibratedclassifi:3,calibratedclassifiercv:3,calibratedcv:11,call:[0,1,5,8,11],callabl:[0,8,10],can:[0,1,2,3,4,5,8],cancer:0,cannot:11,cardiotocographi:0,care:11,carri:3,casa_token:[],castano:[3,11],castro:[3,11],categor:3,categori:[1,8],cc:[3,5,11],ceil:8,center:5,chang:[0,1,3,11],character:[3,6],characteriz:[0,3,11],charg:[0,8],check:[3,4],checkpoint:[9,11],checkpointdir:11,checkpointnam:11,checkpointpath:9,choic:4,chosen:[4,8],cl:0,class2int:10,class_weight:4,classes_:[8,10,11],classif:[0,1,3,7,8,10,11],classif_posterior:[3,11],classif_predict:[3,11],classif_predictions_bin:11,classifi:[1,4,5,6,9,11],classifier_net:9,classifiermixin:9,classifyandcount:[3,11],classmethod:[0,10,11],classnam:10,clean_checkpoint:11,clean_checkpoint_dir:11,clear:5,clearer:1,clearli:5,clip:8,close:1,closer:1,cm:8,cmc:0,cnn:3,cnnnet:[3,9],code:[0,3,4,5,9],coincid:[0,6],col:[0,10],collect:[0,8,9,10],collet:10,color:[5,8],colormap:8,column:[0,10],com:[],combin:[0,1,4,8],combinatio:8,combinations_budget:8,come:0,commandlin:[],common:11,commonli:6,compar:[5,8,11],comparison:5,compil:[2,3],complet:[3,5],compon:[8,9],compress:0,comput:[1,3,5,8,11],computation:4,compute_fpr:11,compute_t:11,compute_tpr:11,concept:6,concur:11,conduct:[0,8],confer:[0,3,10],configur:[4,8],consecut:9,consid:[3,5,9,10],consist:[0,4,5,9,10],constrain:[1,5],constructor:3,consult:[0,1],contain:[1,2,3,5,8,9,10,11],contanin:8,content:7,context:8,contrast:1,control:[1,4],conv_block:[],conv_lay:[],conveni:8,convert:[1,3,9],convolut:9,copi:10,cornel:[],correct:11,correspond:[5,10],cost:1,costli:4,could:[0,1,3,4,5,6,11],count:[4,5,6,10,11],count_:[],counter:10,countvector:10,covari:10,cover:[1,4,9],coz:[0,3,11],cpu:[1,9],creat:[0,6,8],create_if_not_exist:8,create_parent_dir:8,crisp:3,criteria:4,cross:[3,11],cs:[],csr_matrix:10,csv:10,ctg:0,cuda:[3,9,11],cumbersom:1,curios:5,current:[3,8,9,10],custom:[3,6,8],customarili:[3,4],cv:[3,4],cyan:5,d_:8,dat:[0,9],data:[1,3,4,5,6,7,8,9,11],data_hom:10,datafram:1,dataload:9,dataset:[1,3,4,5,6,7,8,9,11],dataset_nam:10,deal:0,decaesteck:[3,11],decai:9,decim:1,decis:[3,9],decision_funct:9,decomposit:9,dedic:1,deep:[3,8,11],def:[0,1,3,5,8],defin:[0,3,8,9,11],degre:4,del:[0,3,11],delai:8,deliv:3,dens:0,depend:[0,1,4,5,8],describ:[3,8,11],descript:0,design:4,desir:[0,1],despit:1,detail:[0,1,3,6,9,11],determin:[1,4,5],detriment:5,devel:10,develop:[4,6],deviat:[0,1,5,8],devic:[0,3,5,9,11],df:[1,10],df_replac:10,diabet:0,diagon:[6,8],dict:[8,10,11],dictionari:[8,9],differ:[0,1,3,4,5,6,8,10],difficult:5,digit:0,dimens:[8,9,10],dimension:[8,9,10],directli:[0,1,3],directori:[2,9,10],discoveri:[3,11],discuss:5,disjoint:9,displai:[1,5,8],displaystyl:8,distanc:11,distant:[1,8],distribut:[0,3,5,8,11],diverg:[1,3,8],dl:[],doabl:0,doc_embed:11,doc_embedding_s:11,doc_posterior:11,document:[0,1,3,5,9,10,11],document_embed:9,doe:[0,2,3,8],doi:[],done:3,dot:5,down:5,download:[0,2,3],download_fil:8,download_file_if_not_exist:8,drawn:[0,1,4],drift:6,drop:[9,11],drop_p:9,dropout:9,ds:[3,11],ds_polici:11,ds_policy_get_posterior:11,dtype:1,dump:10,dure:[1,5],dynam:[3,9,11],e:[0,1,3,4,5,6,8,9,10,11],eacc:11,each:[0,1,3,4,5,8,9,10,11],earli:9,early_stop:11,earlystop:8,easili:[0,2,5,9],ecc:11,edu:[],eemq:11,effect:3,effici:3,ehdi:11,either:[1,3,8,11],element:3,elm:[3,11],em:11,emb:9,embed:[3,9],embed_s:9,embedding_s:9,empti:10,emq:[5,11],enabl:9,encod:10,end:[4,8],endeavour:6,enough:5,ensembl:[0,6,11],ensemblefactori:11,ensure_probabilist:11,entir:[0,3,4,5],environ:[1,3,4,5,8],ep:[1,8],epacc:11,epoch:[9,11],epsilon:[1,8,11],equal:[1,8],equidist:[0,8],equip:[3,5],err:8,err_drift:5,err_nam:8,error:[3,4,6,7,9],error_:[],error_by_drift:[5,8],error_funct:1,error_metr:[1,4,8],error_nam:[5,8,11],establish:8,estim:[1,3,5,6,8,9,11],estim_prev:[1,5,8],estim_preval:[3,6],esuli:[0,2,3,9,10,11],et:[0,2,9],etc:6,eval_budget:[4,8],evalu:[0,3,4,5,6,7,9],eventu:9,everi:[3,11],everyth:3,evinc:5,ex:[],exact:0,exactli:0,exampl:[0,1,3,4,5,8,9,11],exce:8,excel:0,except:[3,8],exemplifi:0,exhibit:[4,5],exist:8,expand_frame_repr:1,expect:6,expectationmaximizationquantifi:[3,11],experi:[1,2,3,4,5,8],explain:[1,5],explicitlossminim:11,explicitlossminimis:11,explor:[4,8],express:10,ext:2,extend:[2,3,11],extens:[0,2,5],extern:3,extract:[1,8],f1:[1,8,9],f1_error:8,f1e:[1,8],f:[0,1,3,4,5,6,10,11],f_1:8,fabrizio:4,facilit:6,fact:[3,5],factor:8,fals:[1,3,5,8,9,10,11],famili:3,familiar:3,far:9,fast:8,faster:[0,10],feat1:10,feat2:10,featn:10,featur:0,feature_extract:10,fetch:[0,6],fetch_review:[0,1,3,4,5,10],fetch_twitt:[0,3,6,10],fetch_ucidataset:[0,3,10],fetch_ucilabelledcollect:[0,10],ff_layer:11,fhe:0,file:[0,5,9,10],fin:0,find:[0,4],finish:4,first:[0,1,2,3,5,8,10,11],fit:[1,3,4,5,6,8,9,10,11],fit_learn:[3,11],fit_transform:10,fix:[1,4],float64:1,fn:8,fold:[3,11],folder:0,follow:[0,1,3,4,5,6],fomart:10,for_model_select:[0,10],form:0,format:[0,5,10],former:[2,11],forward:[9,11],found:[0,3,4,9],four:3,fp:[8,11],fpr:8,frac:8,framework:6,frequenc:0,from:[0,1,3,4,5,6,8,10,11],from_csv:10,from_nam:[1,8],from_spars:10,from_text:10,full:1,fulli:0,func:8,further:[0,1,3,9],fusion:[0,3,11],futur:3,g:[0,1,3,4,6,8,10,11],gao:[0,3,10,11],gasp:[0,10],gen:8,gen_data:5,gen_fn:8,gen_prevalence_predict:8,gener:[0,1,3,4,5,8,9,10,11],generation_func:8,german:0,get:[0,1,5,8,9],get_aggregative_estim:11,get_nprevpoints_approxim:[1,8],get_param:[3,8,9,11],get_probability_distribut:11,get_quapy_hom:8,github:[],given:[1,3,4,8,9,11],global:8,goe:4,good:[4,5],got:4,govern:1,gpu:9,grant:11,grid:[4,8,11],gridsearchcv:4,gridsearchq:[4,8],group:3,guarante:11,guez:[3,11],gzip:0,ha:[3,4,5,8,9],haberman:[0,3],handl:0,happen:[4,5],hard:3,harder:5,harmon:8,harri:0,hat:8,have:[0,1,2,3,4,5,8,10,11],hcr:[0,3,10],hdy:[6,11],held:[3,4,9],helling:11,hellingerdist:8,hellingerdistancei:[3,11],help:5,henc:8,here:1,hidden:[5,9],hidden_s:9,hide:5,high:[5,8],higher:[1,5],hightlight:8,hlt:[],hold:6,home:10,hook:11,how:[0,1,3,4,5,11],howev:[0,4,5,11],hp:[0,3,4,10],html:[],http:[],hyper:[4,8,9],hyperparam:4,hyperparamet:[3,8,11],i:[0,1,3,4,5,8,9,10,11],id:[0,3,10],idf:0,ieee:0,ignor:[8,10,11],iid:[1,5,6],illustr:[3,4,5],imdb:[0,5,10],implement:[0,1,3,4,5,6,8,9,11],impos:[4,8],improv:[3,9],includ:[0,1,3,5,6],inconveni:8,inde:[3,4],independ:8,index:[0,3,6,8,9,10],indextransform:10,indic:[0,1,3,4,5,8,10,11],individu:[1,3],infer:0,inform:[0,1,3,4,8,10,11],infrequ:10,inherit:3,init:3,init_hidden:11,initi:[0,9],inplac:[1,3,10],input:[3,5,8,9],insight:5,inspir:3,instal:[0,3,6,9],instanc:[0,3,4,5,6,8,9,10,11],instanti:[0,1,3,4,9],instead:[1,3,4,11],integ:[3,9,10],integr:6,interest:[1,5,6,8],interestingli:5,interfac:[0,1],intern:[0,3,10],interpret:[5,6],interv:[1,5,8],introduc:1,invok:[0,1,3,8,10],involv:[2,5],io:[],ionospher:0,iri:0,irrespect:5,isaggreg:11,isbinari:[8,10,11],isomer:8,isometr:5,isprobabilist:11,isti:[],item:8,iter:[0,8,11],its:[3,4,8,9],itself:[3,11],j:[0,3,11],joachim:[3,9],job:[2,8],joblib:2,just:[1,3],k:[3,6,11],kei:8,kept:10,kernel:9,kernel_height:9,kfcv:[0,10,11],kindl:[0,1,3,5,10],kl:8,kld:[1,2,8,9],know:3,knowledg:[0,3,10,11],known:[0,3,4],kullback:[1,3,8],kwarg:[9,10,11],l1:11,label:[0,3,4,5,6,8,9,10,11],labelledcollect:[0,3,4,8,10,11],larg:4,largest:8,last:[1,3,5,9],lastli:3,latex:5,latinn:[3,11],latter:11,layer:[3,9],lead:1,learn:[1,2,3,4,6,8,9,11],learner:[3,4,9,11],least:[0,10],leav:10,legend:8,leibler:[1,3,8],length:9,less:[8,10],let:[1,3],level:11,leverag:3,leyend:8,like:[0,1,3,5,8,9],limit:[5,8],line:[1,3],linear:5,linear_model:[1,3,4,6,9],linearsvc:[3,5],linspac:5,list:[0,5,8,9,10],listedcolormap:8,literatur:[0,1,4,6],load:[0,3,8,10],loader:0,loader_func:[0,10],local:8,log:[8,10],logist:[1,3,9,11],logisticregress:[1,3,4,6,9],logscal:8,logspac:4,longer:8,longest:9,look:[0,1,3,5],loss:[6,9,11],low:[5,9],lower:[5,8],lower_is_bett:8,lowest:5,lowranklogisticregress:9,lr:[1,3,9,11],lstm:[3,9],lstm_class_nlay:9,lstm_hidden_s:11,lstm_nlayer:11,lstmnet:9,m:[3,8,11],machin:[1,4,6],macro:8,made:[0,2,11],mae:[1,4,6,8,9,11],mae_loss:11,main:5,maintain:[3,11],make:[0,1,3],mammograph:0,manag:[0,3,10],mani:[1,3,4,5,6,8,11],manner:0,manual:0,map:[1,9],map_parallel:8,margin:9,math:[],mathcal:8,matplotlib:[2,8],matric:[0,5,10],matrix:5,max:11,max_it:11,max_sample_s:11,maxim:6,maximum:[1,8,9],maximumlikelihoodprevalenceestim:11,md:[],mean:[0,1,3,4,5,6,8,9,10,11],mean_absolute_error:8,mean_relative_absolute_error:8,measur:[2,3,4,5,6,11],mediansweep2:11,mediansweep:11,member:3,memori:9,mention:3,merg:5,meta:[6,7,8],meth:[],method:[0,1,4,5,6,7,8],method_data:5,method_nam:[5,8],method_ord:8,metric:[1,3,4,6,8],might:[1,8],min_df:[1,3,4,5,10],min_po:11,mine:[0,3,11],minim:8,minimum:10,minimun:10,mining6:10,minu:8,mixtur:3,mkld:[1,8,11],mnkld:[1,8,11],mock:9,modal:4,model:[0,1,5,6,8,9,11],model_select:[4,7],modifi:[3,8],modul:[0,1,3,5,6,7],moment:[0,3],more:[3,5,8],moreo:[0,3,4,10],most:[0,3,5,6,11],movi:0,mrae:[1,6,8,9,11],ms2:11,ms:11,mse:[1,3,6,8,11],msg:11,multiclass:8,multiprocess:8,multivari:[3,9,11],must:3,my:[],my_arrai:8,my_custom_load:0,my_data:0,mycustomloss:3,n:[0,1,8,9],n_bin:[5,8],n_class:[1,3,8,9,10,11],n_compon:9,n_dimens:9,n_epoch:11,n_featur:9,n_instanc:9,n_job:[1,3,4,8,10,11],n_preval:[0,8,10],n_prevpoint:[1,4,5,8],n_repeat:[1,8],n_repetit:[1,4,5,8],n_sampl:[8,9],name:[5,8,9,10],nativ:6,natur:[1,8],natural_prevalence_predict:8,natural_prevalence_protocol:8,natural_prevalence_report:8,natural_sampling_gener:10,natural_sampling_index_gener:10,nbin:[5,8],ndarrai:[1,3,8,10,11],necessarili:11,need:[0,3,11],neg:[0,5,8],nest:[],net:9,network:[0,9,10,11],neural:[0,7,8,10],neuralclassifiertrain:[3,9],neutral:0,next:[4,8,9],nfold:[0,10],nkld:[1,2,6,8,9],nn:[9,11],nogap:10,non:[3,11],non_aggreg:[7,8],none:[1,4,8,9,10,11],nonetheless:4,nor:3,normal:[0,1,3,8,11],normalize_preval:8,note:[1,3,4,5],now:5,nowadai:3,np:[1,3,4,5,8],npp:8,nprevpoint:8,nrepeat:[0,10],num_prevalence_combin:[1,8],number:[0,1,3,5,8,9,10,11],numer:[0,1,3,6,10],numpi:[2,4,8,9,11],o_l6x_pcf09mdetq4tu7jk98mxfbgsxp9zso14jkuiyudgfg0:[],object:[0,8,9,10,11],observ:1,obtain:[1,4],occur:[5,10],occurr:10,octob:[0,3],off:9,offer:[3,6],older:2,omd:[0,10],ommit:1,onc:[1,3,5,8],one:[0,1,3,4,5,8,11],ones:[1,3,5,8,10],onevsal:[3,11],onli:[0,3,5,8,9,11],open:[0,6],oper:3,opt:4,optim:[2,3,4,8,9,11],optimize_threshold:11,option:[0,1,3,5,8,10,11],order:[0,2,3,5,8,10,11],order_bi:11,org:[],orient:[3,6,8,11],origin:[0,3,10,11],os:0,other:[1,3,5,6,8],otherwis:[0,3,11],our:[],out:[3,4,5,9],outcom:5,outer:8,output:[0,1,3,4,9,11],over:[3,4],overal:1,overestim:5,overrid:3,overridden:[3,11],own:4,p:[0,3,8,11],p_hat:8,pacc:[1,3,5,11],packag:[0,2,3,6,7],pad:9,pad_length:9,padding_length:9,page:[0,2,6],pageblock:0,pair:[0,8],panda:[1,2],paper:[0,3,11],parallel:[1,3,8],param:[4,8,9,10,11],param_grid:[4,8,11],param_mod_sel:11,param_model_sel:11,paramet:[1,3,4,8,9,10,11],part:[3,10],particular:[0,1,3],particularli:1,pass:[0,1,5,9,11],past:1,patch:[2,3,9],path:[0,3,5,8,9,10],patienc:[8,9,11],pattern:[3,11],pca:[],pcalr:[],pcc:[3,4,5,11],pd:1,pdf:5,peopl:[],perf:[6,9],perform:[1,3,4,5,6,8,9,11],phonem:0,pick:4,pickl:[3,8,10],pickle_path:8,pickled_resourc:8,pii:[],pip:2,pipelin:[],pkl:8,plai:0,plan:3,pleas:3,plot:[6,7],png:5,point:[0,1,3,8],polici:[3,11],popular:6,portion:4,pos_class:[8,10],posit:[0,3,5,8],possibl:[1,3,8],posterior:[3,8,9,11],posterior_prob:[3,11],postpon:3,potter:0,pp:[0,3],practic:[0,4],pre:[0,3],prec:[0,8],precis:[0,1,8],preclassifi:3,predict:[3,4,5,8,9,11],predict_proba:[3,9,11],predictor:1,prefer:8,prepare_svmperf:[2,3],preprint:4,preprocess:[0,1,3,7,8],present:[0,3,10],preserv:[1,5],pretti:5,prev:[0,1,8,10],prevail:3,preval:[0,1,3,4,5,6,8,10,11],prevalence_estim:8,prevalence_from_label:8,prevalence_from_prob:8,prevalence_linspac:8,prevel:11,previou:3,previous:11,prevs_estim:11,prevs_hat:[1,8],princip:9,print:[0,1,3,4,6,9],prior:[1,3,4,5,6],priori:[3,11],probabilist:[3,11],probabilisticadjustedclassifyandcount:11,probabilisticclassifyandcount:11,probabl:[1,3,4,5,6,9,11],problem:[0,3,5,8,11],procedur:[3,6,11],proceed:[0,3,10],process:[3,4,8],processor:3,procol:1,produc:[0,1,5,8],product:3,progress:8,properli:0,properti:[3,8,9,10,11],proport:[3,4,8,9,11],propos:[2,3,11],protocl:8,protocol:[0,3,4,5,6,8],provid:[0,3,5,6],ptecondestim:11,ptr:[3,11],ptr_polici:11,purpos:[0,11],python:[0,6],pytorch:2,q:[0,2,3,8,9],qacc:9,qdrop_p:11,qf1:9,qgm:9,qp:[0,1,3,4,5,6,8],quanet:[2,6,9,11],quanetmodul:11,quanettrain:11,quantif:[0,1,6,8,9,10,11],quantifi:[3,4,5,6,8,11],quantification_error:8,quantiti:8,quapi:[0,1,2,3,4,5],quapy_data:0,quay_data:10,quevedo:[0,3,11],quick:[],r:[0,3,11],rac:[],rae:[1,2,8],rais:[3,8],rand:8,random:[1,3,4,5,8],random_se:[1,8],random_st:10,randomli:0,rang:[0,5],rank:[3,9],rare:10,rate:[3,9],rather:[1,4],raw:10,rb:0,re:[3,4,10],read:10,reader:[7,8],readm:[],real:[9,10],reason:[3,5,6],recal:8,receiv:[0,3,5],recip:11,recognit:[3,11],recommend:[1,5],recurr:[0,3,10],red:0,red_siz:[3,11],reduc:[0,10],reduce_column:[0,10],refer:[9,10],refit:[4,8],regard:4,regist:11,regress:9,regressor:[1,3,11],reindex_label:10,reiniti:9,rel:[1,3,8],relative_absolute_error:8,reli:[1,3],reliabl:[3,11],rememb:5,remov:10,repeat:[8,10],repetit:8,repl:10,replac:[0,3,10],replic:[1,4,8],report:1,repositori:0,repr_siz:9,repres:[1,3,5,8,10,11],represent:[0,3,9],request:[0,8,11],requir:[0,1,3,6,9],reset_net_param:9,resourc:8,respect:[0,1,5,8,11],respond:3,rest:[10,11],result:[1,2,3,4,5,6,11],retain:[0,3,9],retrain:4,return_constrained_dim:8,reus:[0,3,8],review:[5,6,10],reviews_sentiment_dataset:0,rewrit:5,right:4,role:0,root:6,roughli:0,routin:8,row:10,run:[0,1,2,3,4,5,8,11],s003132031400291x:[],s:[0,1,3,4,5,8,9,10],saeren:[3,11],sai:11,said:3,same:[0,3,5,8,10],sampl:[0,1,3,4,5,6,8,9,10,11],sample_s:[0,1,3,4,5,8,10,11],sampling_from_index:[0,10],sampling_index:[0,10],sander:[0,10],save:[5,8],save_or_show:8,save_text_fil:8,savepath:[5,8],scall:10,scenario:[1,3,4,5,6],scienc:[3,11],sciencedirect:[],scikit:[2,3,4],scipi:[2,10],score:[0,1,4,9,10],script:[1,2,3,6],se:[1,8],search:[3,4,6,8,11],sebastiani:[0,3,4,10,11],second:[0,1,3,5,8],section:4,see:[0,1,2,3,4,5,6,8,9],seed:[1,4,8],seem:3,seemingli:5,seen:5,select:[0,3,6,8,11],selector:3,self:[3,9,10,11],semeion:0,semev:0,semeval13:[0,10],semeval14:[0,10],semeval15:[0,10],semeval16:[0,6,10],sentenc:10,sentiment:[3,6,10,11],separ:[8,10],seri:0,serv:3,set:[0,1,3,4,5,6,8,9,10,11],set_opt:1,set_param:[3,8,9,11],set_siz:[],sever:0,sh:[2,3],shape:[5,8,9],share:[0,10],shift:[1,4,6,8],shorter:9,shoud:3,should:[0,1,3,4,5,6,9,10,11],show:[0,1,3,4,5,8,9,10],show_dens:8,show_std:[5,8],showcas:5,shown:[1,5,8],shuffl:[9,10],sign:8,signific:1,silent:[8,11],similar:11,simpl:[0,3,5,11],simplest:3,simplex:[0,8],simpli:[1,2,3,4,5,6,8,11],sinc:[0,1,3,5,8,11],singl:[1,3,6,11],size:[0,1,3,8,9,10,11],sklearn:[1,3,4,5,6,9,10,11],sld:3,slice:8,smooth:[1,8],smooth_limits_epsilon:8,so:[0,1,3,5,8,9,11],social:[0,3,10,11],soft:3,softwar:0,solid:5,solv:4,solve_adjust:11,some:[0,1,3,5],some_arrai:8,sometim:1,sonar:0,sourc:[2,3,6,9],sout:11,space:[0,4,9],spambas:0,spars:[0,10],special:[0,5,10],specif:[3,4],specifi:[0,1,3,5,8,9,10,11],spectf:0,spectrum:[0,1,4,5],speed:3,split:[0,3,4,5,9,10,11],split_stratifi:10,splitstratifi:10,spmatrix:10,squar:[1,3,8],sst:[0,10],stabil:1,stand:8,standard:[0,1,5,8,10],start:4,stat:10,state:8,statist:[0,1,11],stats_siz:11,std:9,stdout:8,step:[5,8],stop:9,store:[0,9,10],str:[0,8,10],strategi:[3,4],stratifi:[0,3],stride:9,string:[1,8,10],strongli:[4,5],strprev:[0,1,8],structur:3,studi:[0,3,11],subclass:11,subinterv:5,sublinear_tf:10,submit:0,submodul:7,subobject:[],suboptim:4,subpackag:7,subsequ:[10,11],subtract:[0,8],subtyp:10,suffic:5,suffici:11,sum:11,sum_:8,summar:0,supervis:[4,6],support:[3,6,9],surpass:1,svm:[3,5,6,9],svm_light:[],svm_perf:[],svm_perf_classifi:9,svm_perf_learn:9,svm_perf_quantif:[2,3],svmae:[3,11],svmkld:[3,11],svmnkld:[3,11],svmperf:[2,3,7,8],svmperf_bas:[9,11],svmperf_hom:3,svmq:[3,11],svmrae:[3,11],syntax:5,system:4,t50:11,t:[0,1,3,8],tab10:8,tail_density_threshold:8,take:[0,3,5,8,11],taken:[3,8,9],target:[3,5,6,8,9,11],task:[3,4,11],temp_se:8,tempor:8,tend:5,tendenc:5,tensor:9,term:[0,1,3,4,5,6,8,9,10,11],test:[0,1,3,4,5,6,8,9,10,11],test_bas:[],test_dataset:[],test_method:[],test_path:[0,10],test_sampl:8,test_split:10,text2tfidf:[0,1,3,10],text:[0,3,8,9,10,11],textclassifiernet:9,textual:[0,6,10],tf:[0,10],tfidf:[0,4,5,10],tfidfvector:10,than:[1,4,5,8,9,10],thei:[0,3],them:[0,3,11],theoret:4,thereaft:1,thi:[0,1,2,3,4,5,6,8,9,11],thing:3,third:[1,5],thorsten:9,those:[1,3,4,5,8,9],though:[3,8],three:[0,5],thresholdoptim:11,through:[3,8],thu:[3,4,5,8,11],tictacto:0,time:[0,1,3,8,10],timeout:8,timeouterror:8,timer:8,titl:8,tj:[],tn:[8,11],token:[0,9,10],tool:[1,6],top:[3,11],torch:[3,9,11],torchdataset:9,toward:5,tp:[8,11],tpr:8,tqdm:2,tr_iter_per_poch:11,tr_prev:[5,8,11],trade:9,tradition:1,train:[0,1,3,4,5,6,8,9,10,11],train_path:[0,10],train_prev:[5,8],train_prop:10,train_siz:10,train_val_split:11,trainer:9,training_help:11,training_preval:5,training_s:5,transact:[3,11],transform:[0,9,10],transfus:0,trivial:3,true_prev:[1,5,8],true_preval:6,truncatedsvd:9,ttest_alpha:8,turn:4,tweet:[0,3,10,11],twitter:[6,10],twitter_sentiment_datasets_test:0,twitter_sentiment_datasets_train:0,two:[0,1,3,4,5,8],type:[0,3],typic:[1,4,5,8,9],uci:6,unabl:0,unadjust:5,unalt:9,unbias:5,uncompress:0,under:1,underestim:5,underlin:8,unfortun:5,unifi:0,uniform_prevalence_sampl:8,uniform_sampl:10,uniform_sampling_index:10,uniform_simplex_sampl:8,uniformli:8,union:[8,11],uniqu:10,unit:0,unix:0,unk:10,unless:11,unlik:[1,4],unus:[8,9,11],up:[3,4,8,9,11],updat:[],url:8,us:[0,1,3,4,5,6,8,9,10,11],user:[0,1,5],utf:10,util:[7,9],v:[3,11],va_iter_per_poch:11,val:[0,10],val_split:[3,4,8,9,11],valid:[0,1,3,4,5,8,9,10,11],valid_loss:[3,9],valid_polici:11,valu:[0,1,3,8,9,10,11],variabl:[1,3,5,8],varianc:[0,5],variant:[5,6,11],varieti:4,variou:[1,5],vector:[0,8,9,10],verbos:[0,1,4,8,9,10,11],veri:[3,5],versatil:6,version:[2,9],vertical_xtick:8,via:[0,2,3,11],view:5,visual:[5,6],vline:8,vocab_s:9,vocabulari:[9,10],vocabulary_s:[3,9,10],vs:3,w:[0,3,10,11],wa:[0,3,5,10,11],wai:[1,11],wait:9,want:[3,4],warn:10,wb:[0,10],wdbc:0,we:[0,1,3,4,5,6],weight:[9,10],weight_decai:9,well:[0,3,4,5],were:0,what:3,when:[0,1,3,4,5,8,9],whenev:[5,8],where:[3,5,8,9,10,11],wherebi:4,whether:[8,9,10,11],which:[0,1,3,4,5,8,9,10,11],white:0,whole:[0,1,3,4,8],why:3,wide:5,wiki:[0,3],wine:0,within:[8,11],without:[1,3,8],word:[1,3,6,9,10],work:[1,3,4,5],worker:1,wors:[4,5],would:[0,1,3,5,6,8,11],wrapper:[8,9],written:6,www:[],x:[5,8,9,10,11],x_error:8,xavier:9,xavier_uniform:9,xlrd:[0,2],xy:10,y:[5,8,9,10,11],y_:11,y_error:8,y_pred:8,y_true:8,ye:10,yeast:0,yield:[5,8],yin:[],you:[2,3],your:3,z:0,zero:[0,8],zfthyovrzwxmgfzylqw_y8cagg:[],zip:[0,5]},titles:["Datasets","Evaluation","Installation","Quantification Methods","Model Selection","Plotting","Welcome to QuaPy\u2019s documentation!","quapy","quapy package","quapy.classification package","quapy.data package","quapy.method package"],titleterms:{"function":8,A:6,The:3,ad:0,aggreg:[3,11],base:[10,11],bia:5,classif:[4,9],classifi:3,content:[6,8,9,10,11],count:3,custom:0,data:[0,10],dataset:[0,10],diagon:5,distanc:3,document:6,drift:5,emq:3,ensembl:3,error:[1,5,8],evalu:[1,8],ex:[],exampl:6,expect:3,explicit:3,featur:6,get:[],hdy:3,helling:3,indic:6,instal:2,introduct:6,issu:0,learn:0,loss:[2,3,4],machin:0,maxim:3,measur:1,meta:[3,11],method:[3,9,11],minim:3,model:[3,4],model_select:8,modul:[8,9,10,11],network:3,neural:[3,9,11],non_aggreg:11,orient:[2,4],packag:[8,9,10,11],perf:2,plot:[5,8],preprocess:10,process:0,protocol:1,quanet:3,quantif:[2,3,4,5],quapi:[6,7,8,9,10,11],quick:6,reader:10,readm:[],requir:2,review:0,s:6,select:4,sentiment:0,start:[],submodul:[8,9,10,11],subpackag:8,svm:2,svmperf:9,tabl:6,target:4,test:[],test_bas:[],test_dataset:[],test_method:[],titl:[],twitter:0,uci:0,util:8,variant:3,welcom:6,y:3}}) \ No newline at end of file diff --git a/quapy/plot.py b/quapy/plot.py index 8d46ea5..bc93fae 100644 --- a/quapy/plot.py +++ b/quapy/plot.py @@ -1,34 +1,58 @@ from collections import defaultdict - import matplotlib.pyplot as plt +from matplotlib.cm import get_cmap import numpy as np from matplotlib import cm +from scipy.stats import ttest_ind_from_stats import quapy as qp -from matplotlib.font_manager import FontProperties plt.rcParams['figure.figsize'] = [12, 8] plt.rcParams['figure.dpi'] = 200 plt.rcParams['font.size'] = 16 -def _set_colors(ax, n_methods): - NUM_COLORS = n_methods - cm = plt.get_cmap('tab20') - ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)]) - - def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=None, show_std=True, legend=True, - train_prev=None, savepath=None): + train_prev=None, savepath=None, method_order=None): + """ + The diagonal plot displays the predicted prevalence values (along the y-axis) as a function of the true prevalence + values (along the x-axis). The optimal quantifier is described by the diagonal (0,0)-(1,1) of the plot (hence the + name). It is convenient for binary quantification problems, though it can be used for multiclass problems by + indicating which class is to be taken as the positive class. (For multiclass quantification problems, other plots + like the :meth:`error_by_drift` might be preferable though). + + :param method_names: array-like with the method names for each experiment + :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for + each experiment + :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) + for each experiment + :param pos_class: index of the positive class + :param title: the title to be displayed in the plot + :param show_std: whether or not to show standard deviations (represented by color bands). This might be inconvenient + for cases in which many methods are compared, or when the standard deviations are high -- default True) + :param legend: whether or not to display the leyend (default True) + :param train_prev: if indicated (default is None), the training prevalence (for the positive class) is hightlighted + in the plot. This is convenient when all the experiments have been conducted in the same dataset. + :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. + :param method_order: if indicated (default is None), imposes the order in which the methods are processed (i.e., + listed in the legend and associated with matplotlib colors). + """ fig, ax = plt.subplots() ax.set_aspect('equal') ax.grid() ax.plot([0, 1], [0, 1], '--k', label='ideal', zorder=1) method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs) - _set_colors(ax, n_methods=len(method_names)) - for method, true_prev, estim_prev in zip(method_names, true_prevs, estim_prevs): + order = list(zip(method_names, true_prevs, estim_prevs)) + if method_order is not None: + table = {method_name:[true_prev, estim_prev] for method_name, true_prev, estim_prev in order} + order = [(method_name, *table[method_name]) for method_name in method_order] + + cm = plt.get_cmap('tab20') + NUM_COLORS = len(method_names) + ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)]) + for method, true_prev, estim_prev in order: true_prev = true_prev[:,pos_class] estim_prev = estim_prev[:,pos_class] @@ -50,14 +74,32 @@ def binary_diagonal(method_names, true_prevs, estim_prevs, pos_class=1, title=No ax.set_xlim(0, 1) if legend: - box = ax.get_position() - ax.set_position([box.x0, box.y0, box.width * 0.8, box.height]) - ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + # box = ax.get_position() + # ax.set_position([box.x0, box.y0, box.width * 0.8, box.height]) + # ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + # ax.set_position([box.x0, box.y0, box.width * 0.8, box.height]) + ax.legend(loc='lower center', + bbox_to_anchor=(1, -0.5), + ncol=(len(method_names)+1)//2) save_or_show(savepath) def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title=None, savepath=None): + """ + Box-plots displaying the global bias (i.e., signed error computed as the estimated value minus the true value) + for each quantification method with respect to a given positive class. + + :param method_names: array-like with the method names for each experiment + :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for + each experiment + :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) + for each experiment + :param pos_class: index of the positive class + :param title: the title to be displayed in the plot + :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. + """ + method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs) fig, ax = plt.subplots() @@ -79,33 +121,47 @@ def binary_bias_global(method_names, true_prevs, estim_prevs, pos_class=1, title def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=None, nbins=5, colormap=cm.tab10, vertical_xticks=False, legend=True, savepath=None): + """ + + :param method_names: array-like with the method names for each experiment + :param true_prevs: array-like with the true prevalence values (each being a ndarray with n_classes components) for + each experiment + :param estim_prevs: array-like with the estimated prevalence values (each being a ndarray with n_classes components) + for each experiment + :param pos_class: index of the positive class + :param title: the title to be displayed in the plot + :param nbins: number of bins + :param colormap: the matplotlib colormap to use (default cm.tab10) + :param vertical_xticks: + :param legend: whether or not to display the legend (default is True) + :param savepath: path where to save the plot. If not indicated (as default), the plot is shown. + """ from pylab import boxplot, plot, setp fig, ax = plt.subplots() ax.grid() method_names, true_prevs, estim_prevs = _merge(method_names, true_prevs, estim_prevs) - _set_colors(ax, n_methods=len(method_names)) bins = np.linspace(0, 1, nbins+1) binwidth = 1/nbins data = {} for method, true_prev, estim_prev in zip(method_names, true_prevs, estim_prevs): - true_prev = true_prev[:, pos_class] - estim_prev = estim_prev[:, pos_class] + true_prev = true_prev[:,pos_class] + estim_prev = estim_prev[:,pos_class] data[method] = [] - inds = np.digitize(true_prev, bins[1:], right=True) + inds = np.digitize(true_prev, bins, right=True) for ind in range(len(bins)): selected = inds==ind data[method].append(estim_prev[selected] - true_prev[selected]) nmethods = len(method_names) boxwidth = binwidth/(nmethods+4) - for i,bin in enumerate(bins): + for i,bin in enumerate(bins[:-1]): boxdata = [data[method][i] for method in method_names] positions = [bin+(i*boxwidth)+2*boxwidth for i,_ in enumerate(method_names)] - box = boxplot(boxdata, showmeans=False, positions=positions, widths=boxwidth, sym='+', patch_artist=True) + box = boxplot(boxdata, showmeans=False, positions=positions, widths = boxwidth, sym='+', patch_artist=True) for boxid in range(len(method_names)): c = colormap.colors[boxid%len(colormap.colors)] setp(box['fliers'][boxid], color=c, marker='+', markersize=3., markeredgecolor=c) @@ -118,7 +174,7 @@ def binary_bias_bins(method_names, true_prevs, estim_prevs, pos_class=1, title=N major_xticks_positions.append(b) minor_xticks_positions.append(b + binwidth / 2) major_xticks_labels.append('') - minor_xticks_labels.append(f'[{bins[i]:.2f}-{bins[i + 1]:.2f}' + (')' if i < len(bins)-2 else ']')) + minor_xticks_labels.append(f'[{bins[i]:.2f}-{bins[i + 1]:.2f})') ax.set_xticks(major_xticks_positions) ax.set_xticks(minor_xticks_positions, minor=True) ax.set_xticklabels(major_xticks_labels) @@ -166,10 +222,19 @@ def _merge(method_names, true_prevs, estim_prevs): return method_order, true_prevs_, estim_prevs_ -def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, error_name='ae', show_std=True, +def _set_colors(ax, n_methods): + NUM_COLORS = n_methods + cm = plt.get_cmap('tab20') + ax.set_prop_cycle(color=[cm(1. * i / NUM_COLORS) for i in range(NUM_COLORS)]) + + +def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, error_name='ae', show_std=False, + show_density=True, logscale=False, title=f'Quantification error as a function of distribution shift', - savepath=None): + savepath=None, + vlines=None, + method_order=None): fig, ax = plt.subplots() ax.grid() @@ -177,28 +242,17 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, e x_error = qp.error.ae y_error = getattr(qp.error, error_name) - # join all data, and keep the order in which the methods appeared for the first time - data = defaultdict(lambda:{'x':np.empty(shape=(0)), 'y':np.empty(shape=(0))}) - method_order = [] - - for method, test_prevs_i, estim_prevs_i, tr_prev_i in zip(method_names, true_prevs, estim_prevs, tr_prevs): - tr_prev_i = np.repeat(tr_prev_i.reshape(1,-1), repeats=test_prevs_i.shape[0], axis=0) - - tr_test_drifts = x_error(test_prevs_i, tr_prev_i) - data[method]['x'] = np.concatenate([data[method]['x'], tr_test_drifts]) - - method_drifts = y_error(test_prevs_i, estim_prevs_i) - data[method]['y'] = np.concatenate([data[method]['y'], method_drifts]) - - if method not in method_order: - method_order.append(method) + # get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same + # order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to + # x_error function) and 'y' is the estim-test shift (computed as according to y_error) + data = __join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order) _set_colors(ax, n_methods=len(method_order)) bins = np.linspace(0, 1, n_bins+1) - inds_histogram_global = np.zeros(n_bins, dtype=np.float) # we use this to keep track of how many datapoits contribute to each bin binwidth = 1 / n_bins - min_x, max_x = None, None + min_x, max_x, min_y, max_y = None, None, None, None + npoints = np.zeros(len(bins), dtype=float) for method in method_order: tr_test_drifts = data[method]['x'] method_drifts = data[method]['y'] @@ -206,40 +260,194 @@ def error_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, e method_drifts=np.log(1+method_drifts) inds = np.digitize(tr_test_drifts, bins, right=True) - inds_histogram_global += np.histogram(tr_test_drifts, density=True, bins=bins)[0] xs, ys, ystds = [], [], [] - for ind in range(len(bins)): + for p,ind in enumerate(range(len(bins))): selected = inds==ind if selected.sum() > 0: - xs.append(ind*binwidth) + xs.append(ind*binwidth-binwidth/2) ys.append(np.mean(method_drifts[selected])) ystds.append(np.std(method_drifts[selected])) + npoints[p] += len(method_drifts[selected]) xs = np.asarray(xs) ys = np.asarray(ys) ystds = np.asarray(ystds) - min_x_method, max_x_method = xs.min(), xs.max() + min_x_method, max_x_method, min_y_method, max_y_method = xs.min(), xs.max(), ys.min(), ys.max() min_x = min_x_method if min_x is None or min_x_method < min_x else min_x max_x = max_x_method if max_x is None or max_x_method > max_x else max_x + max_y = max_y_method if max_y is None or max_y_method > max_y else max_y + min_y = min_y_method if min_y is None or min_y_method < min_y else min_y + max_y = max_y_method if max_y is None or max_y_method > max_y else max_y + + ax.errorbar(xs, ys, fmt='-', marker='o', color='w', markersize=8, linewidth=4, zorder=1) + ax.errorbar(xs, ys, fmt='-', marker='o', label=method, markersize=6, linewidth=2, zorder=2) - ax.errorbar(xs, ys, fmt='-', marker='o', label=method, markersize=3, zorder=2) if show_std: ax.fill_between(xs, ys-ystds, ys+ystds, alpha=0.25) - # xs = bins[:-1] - # ys = inds_histogram_global - # print(xs.shape, ys.shape) - # ax.errorbar(xs, ys, label='density') + if show_density: + ax.bar([ind * binwidth-binwidth/2 for ind in range(len(bins))], + max_y*npoints/np.max(npoints), alpha=0.15, color='g', width=binwidth, label='density') ax.set(xlabel=f'Distribution shift between training set and test sample', ylabel=f'{error_name.upper()} (true distribution, predicted distribution)', title=title) box = ax.get_position() ax.set_position([box.x0, box.y0, box.width * 0.8, box.height]) + if vlines: + for vline in vlines: + ax.axvline(vline, 0, 1, linestyle='--', color='k') + ax.set_xlim(0, max_x) ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) - ax.set_xlim(min_x, max_x) + + save_or_show(savepath) + + +def brokenbar_supremacy_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, n_bins=20, binning='isomerous', + x_error='ae', y_error='ae', ttest_alpha=0.005, tail_density_threshold=0.005, + method_order=None, + savepath=None): + assert binning in ['isomerous', 'isometric'], 'unknown binning type; valid types are "isomerous" and "isometric"' + + x_error = getattr(qp.error, x_error) + y_error = getattr(qp.error, y_error) + + # get all data as a dictionary {'m':{'x':ndarray, 'y':ndarray}} where 'm' is a method name (in the same + # order as in method_order (if specified), and where 'x' are the train-test shifts (computed as according to + # x_error function) and 'y' is the estim-test shift (computed as according to y_error) + data = __join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order) + + if binning == 'isomerous': + # take bins containing the same amount of examples + tr_test_drifts = np.concatenate([data[m]['x'] for m in method_order]) + bins = np.quantile(tr_test_drifts, q=np.linspace(0, 1, n_bins+1)).flatten() + else: + # take equidistant bins + bins = np.linspace(0, 1, n_bins+1) + bins[0] = -0.001 + bins[-1] += 0.001 + + # we use this to keep track of how many datapoits contribute to each bin + inds_histogram_global = np.zeros(n_bins, dtype=np.float) + n_methods = len(method_order) + buckets = np.zeros(shape=(n_methods, n_bins, 3)) + for i, method in enumerate(method_order): + tr_test_drifts = data[method]['x'] + method_drifts = data[method]['y'] + + inds = np.digitize(tr_test_drifts, bins, right=False) + inds_histogram_global += np.histogram(tr_test_drifts, density=False, bins=bins)[0] + + for j in range(len(bins)): + selected = inds == j + if selected.sum() > 0: + buckets[i, j-1, 0] = np.mean(method_drifts[selected]) + buckets[i, j-1, 1] = np.std(method_drifts[selected]) + buckets[i, j-1, 2] = selected.sum() + + # cancel last buckets with low density + histogram = inds_histogram_global / inds_histogram_global.sum() + for tail in reversed(range(len(histogram))): + if histogram[tail] < tail_density_threshold: + buckets[:,tail,2] = 0 + else: + break + + salient_methods = set() + best_methods = [] + for bucket in range(buckets.shape[1]): + nc = buckets[:, bucket, 2].sum() + if nc == 0: + best_methods.append([]) + continue + + order = np.argsort(buckets[:, bucket, 0]) + rank1 = order[0] + best_bucket_methods = [method_order[rank1]] + best_mean, best_std, best_nc = buckets[rank1, bucket, :] + for method_index in order[1:]: + method_mean, method_std, method_nc = buckets[method_index, bucket, :] + _, pval = ttest_ind_from_stats(best_mean, best_std, best_nc, method_mean, method_std, method_nc) + if pval > ttest_alpha: + best_bucket_methods.append(method_order[method_index]) + best_methods.append(best_bucket_methods) + salient_methods.update(best_bucket_methods) + print(best_bucket_methods) + + if binning=='isomerous': + fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [0.2, 1]}, figsize=(20, len(salient_methods))) + else: + fig, axes = plt.subplots(2, 1, gridspec_kw={'height_ratios': [1, 1]}, figsize=(20, len(salient_methods))) + + ax = axes[1] + high_from = 0 + yticks, yticks_method_names = [], [] + color = get_cmap('Accent').colors + vlines = [] + bar_high = 1 + for method in [m for m in method_order if m in salient_methods]: + broken_paths = [] + path_start, path_end = None, None + for i, best_bucket_methods in enumerate(best_methods): + if method in best_bucket_methods: + if path_start is None: + path_start = bins[i] + path_end = bins[i+1]-path_start + else: + path_end += bins[i+1]-bins[i] + else: + if path_start is not None: + broken_paths.append(tuple((path_start, path_end))) + path_start, path_end = None, None + if path_start is not None: + broken_paths.append(tuple((path_start, path_end))) + + ax.broken_barh(broken_paths, (high_from, bar_high), facecolors=color[len(yticks_method_names)]) + yticks.append(high_from+bar_high/2) + high_from += bar_high + yticks_method_names.append(method) + for path_start, path_end in broken_paths: + vlines.extend([path_start, path_start+path_end]) + + vlines = np.unique(vlines) + vlines = sorted(vlines) + for v in vlines[1:-1]: + ax.axvline(x=v, color='k', linestyle='--') + + ax.set_ylim(0, high_from) + ax.set_xlim(vlines[0], vlines[-1]) + ax.set_xlabel('Distribution shift between training set and sample') + + ax.set_yticks(yticks) + ax.set_yticklabels(yticks_method_names) + + # upper plot (explaining distribution) + ax = axes[0] + if binning == 'isometric': + # show the density for each region + bins[0]=0 + y_pos = [b+(bins[i+1]-b)/2 for i,b in enumerate(bins[:-1]) if histogram[i]>0] + bar_width = [bins[i+1]-bins[i] for i in range(len(bins[:-1])) if histogram[i]>0] + ax.bar(y_pos, [n for n in histogram if n>0], bar_width, align='center', alpha=0.5, color='silver') + ax.set_ylabel('shift\ndistribution', rotation=0, ha='right', va='center') + ax.set_xlim(vlines[0], vlines[-1]) + ax.get_xaxis().set_visible(False) + plt.subplots_adjust(wspace=0, hspace=0.1) + else: + # show the percentiles of the distribution + cumsum = np.cumsum(histogram) + for i in range(len(bins[:-1])): + start, width = bins[i], bins[i+1]-bins[i] + ax.broken_barh([tuple((start, width))], (0, 1), facecolors='whitesmoke' if i%2==0 else 'silver') + if i < len(bins)-2: + ax.text(bins[i+1], 0.5, '$P_{'+f'{int(np.round(cumsum[i]*100))}'+'}$', ha='center') + ax.set_ylim(0, 1) + ax.set_xlim(vlines[0], vlines[-1]) + ax.get_yaxis().set_visible(False) + ax.get_xaxis().set_visible(False) + plt.subplots_adjust(wspace=0, hspace=0) save_or_show(savepath) @@ -253,3 +461,23 @@ def save_or_show(savepath): else: plt.show() + +def __join_data_by_drift(method_names, true_prevs, estim_prevs, tr_prevs, x_error, y_error, method_order): + data = defaultdict(lambda: {'x': np.empty(shape=(0)), 'y': np.empty(shape=(0))}) + + if method_order is None: + method_order = [] + + for method, test_prevs_i, estim_prevs_i, tr_prev_i in zip(method_names, true_prevs, estim_prevs, tr_prevs): + tr_prev_i = np.repeat(tr_prev_i.reshape(1, -1), repeats=test_prevs_i.shape[0], axis=0) + + tr_test_drifts = x_error(test_prevs_i, tr_prev_i) + data[method]['x'] = np.concatenate([data[method]['x'], tr_test_drifts]) + + method_drifts = y_error(test_prevs_i, estim_prevs_i) + data[method]['y'] = np.concatenate([data[method]['y'], method_drifts]) + + if method not in method_order: + method_order.append(method) + + return data \ No newline at end of file