lorenzo.volpi
/
QuAcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added kde implementation, switched to dataclass

This commit is contained in:
Lorenzo Volpi 2023-11-26 16:31:23 +01:00
parent d6736d194b
commit 58661e7f93
1 changed files with 83 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
quacc/evaluation/method.py
View File

@ -1,26 +1,35 @@
from dataclasses import dataclass
from typing import List
import numpy as np import numpy as np
from quapy.method.aggregative import PACC, SLD from quapy.method.aggregative import PACC, SLD, BaseQuantifier
from quapy.protocol import UPP, AbstractProtocol from quapy.protocol import UPP, AbstractProtocol
from sklearn.linear_model import LogisticRegression from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC from sklearn.svm import LinearSVC
import quacc as qc import quacc as qc
from quacc.evaluation.report import EvaluationReport from quacc.evaluation.report import EvaluationReport
from quacc.method.base import BQAE, MCAE, BaseAccuracyEstimator
from quacc.method.model_selection import GridSearchAE from quacc.method.model_selection import GridSearchAE
from quacc.quantification import KDEy
from ..method.base import BQAE, MCAE, BaseAccuracyEstimator
_param_grid = { _param_grid = {
"sld": { "sld": {
"q__classifier__C": np.logspace(-3, 3, 7), "q__classifier__C": np.logspace(-3, 3, 7),
"q__classifier__class_weight": [None, "balanced"], "q__classifier__class_weight": [None, "balanced"],
"q__recalib": [None, "bcts"], "q__recalib": [None, "bcts"],
"confidence": [["isoft"], ["max_conf", "entropy"]], "confidence": [None, ["isoft"], ["max_conf", "entropy"]],
}, },
"pacc": { "pacc": {
"q__classifier__C": np.logspace(-3, 3, 7), "q__classifier__C": np.logspace(-3, 3, 7),
"q__classifier__class_weight": [None, "balanced"], "q__classifier__class_weight": [None, "balanced"],
"confidence": [["isoft"], ["max_conf", "entropy"]], "confidence": [None, ["isoft"], ["max_conf", "entropy"]],
},
"kde": {
"q__classifier__C": np.logspace(-3, 3, 7),
"q__classifier__class_weight": [None, "balanced"],
"q__bandwidth": np.linspace(0.01, 0.2, 5),
"confidence": [None, ["isoft"]],
}, },
} }
@ -56,38 +65,42 @@ def evaluation_report(
return report return report
@dataclass(frozen=True)
class EvaluationMethod: class EvaluationMethod:
def __init__(self, name, q, est_c, conf=None, cf=False): name: str
self.name = name q: BaseQuantifier
self.__name__ = name est_n: str
self.q = q conf: List[str] | str = None
self.est_c = est_c cf: bool = False
self.conf = conf
self.cf = cf def get_est(self, c_model):
match self.est_n:
case "mul":
return MCAE(
c_model,
self.q,
confidence=self.conf,
collapse_false=self.cf,
)
case "bin":
return BQAE(c_model, self.q, confidence=self.conf)
def __call__(self, c_model, validation, protocol) -> EvaluationReport: def __call__(self, c_model, validation, protocol) -> EvaluationReport:
est = self.est_c( est = self.get_est(c_model).fit(validation)
c_model,
self.q,
confidence=self.conf,
collapse_false=self.cf,
).fit(validation)
return evaluation_report( return evaluation_report(
estimator=est, protocol=protocol, method_name=self.name estimator=est, protocol=protocol, method_name=self.name
) )
@dataclass(frozen=True)
class EvaluationMethodGridSearch(EvaluationMethod): class EvaluationMethodGridSearch(EvaluationMethod):
def __init__(self, name, q, est_c, cf=False, pg="sld"): pg: str = "sld"
super().__init__(name, q, est_c, cf=cf)
self.pg = pg
def __call__(self, c_model, validation, protocol) -> EvaluationReport: def __call__(self, c_model, validation, protocol) -> EvaluationReport:
v_train, v_val = validation.split_stratified(0.6, random_state=0) v_train, v_val = validation.split_stratified(0.6, random_state=0)
model = self.est_c(c_model, self.q, collapse_false=self.cf)
__grid = _param_grid.get(self.pg, {}) __grid = _param_grid.get(self.pg, {})
est = GridSearchAE( est = GridSearchAE(
model=model, model=self.get_est(c_model),
param_grid=__grid, param_grid=__grid,
refit=False, refit=False,
protocol=UPP(v_val, repeats=100), protocol=UPP(v_val, repeats=100),
@ -108,6 +121,10 @@ def __sld_lr():
return SLD(LogisticRegression()) return SLD(LogisticRegression())
def __kde_lr():
return KDEy(LogisticRegression())
def __sld_lsvc(): def __sld_lsvc():
return SLD(LinearSVC()) return SLD(LinearSVC())
@ -119,37 +136,54 @@ def __pacc_lr():
# fmt: off # fmt: off
__methods_set = [ __methods_set = [
# base sld # base sld
M("bin_sld", __sld_lr(), BQAE ), M("bin_sld", __sld_lr(), "bin" ),
M("mul_sld", __sld_lr(), MCAE ), M("mul_sld", __sld_lr(), "mul" ),
M("m3w_sld", __sld_lr(), MCAE, cf=True), M("m3w_sld", __sld_lr(), "mul", cf=True),
# max_conf + entropy sld # max_conf + entropy sld
M("binc_sld", __sld_lr(), BQAE, conf=["max_conf", "entropy"] ), M("binc_sld", __sld_lr(), "bin", conf=["max_conf", "entropy"] ),
M("mulc_sld", __sld_lr(), MCAE, conf=["max_conf", "entropy"] ), M("mulc_sld", __sld_lr(), "mul", conf=["max_conf", "entropy"] ),
M("m3wc_sld", __sld_lr(), MCAE, conf=["max_conf", "entropy"], cf=True), M("m3wc_sld", __sld_lr(), "mul", conf=["max_conf", "entropy"], cf=True),
# max_conf sld # max_conf sld
M("binmc_sld", __sld_lr(), BQAE, conf="max_conf", ), M("binmc_sld", __sld_lr(), "bin", conf="max_conf", ),
M("mulmc_sld", __sld_lr(), MCAE, conf="max_conf", ), M("mulmc_sld", __sld_lr(), "mul", conf="max_conf", ),
M("m3wmc_sld", __sld_lr(), MCAE, conf="max_conf", cf=True), M("m3wmc_sld", __sld_lr(), "mul", conf="max_conf", cf=True),
# entropy sld # entropy sld
M("binne_sld", __sld_lr(), BQAE, conf="entropy", ), M("binne_sld", __sld_lr(), "bin", conf="entropy", ),
M("mulne_sld", __sld_lr(), MCAE, conf="entropy", ), M("mulne_sld", __sld_lr(), "mul", conf="entropy", ),
M("m3wne_sld", __sld_lr(), MCAE, conf="entropy", cf=True), M("m3wne_sld", __sld_lr(), "mul", conf="entropy", cf=True),
# inverse softmax sld # inverse softmax sld
M("binis_sld", __sld_lr(), BQAE, conf="isoft", ), M("binis_sld", __sld_lr(), "bin", conf="isoft", ),
M("mulis_sld", __sld_lr(), MCAE, conf="isoft", ), M("mulis_sld", __sld_lr(), "mul", conf="isoft", ),
M("m3wis_sld", __sld_lr(), MCAE, conf="isoft", cf=True), M("m3wis_sld", __sld_lr(), "mul", conf="isoft", cf=True),
# inverse softmax sld
M("binis_pacc", __pacc_lr(), BQAE, conf="isoft", ),
M("mulis_pacc", __pacc_lr(), MCAE, conf="isoft", ),
M("m3wis_pacc", __pacc_lr(), MCAE, conf="isoft", cf=True),
# gs sld # gs sld
G("bin_sld_gs", __sld_lr(), BQAE, pg="sld" ), G("bin_sld_gs", __sld_lr(), "bin", pg="sld" ),
G("mul_sld_gs", __sld_lr(), MCAE, pg="sld" ), G("mul_sld_gs", __sld_lr(), "mul", pg="sld" ),
G("m3w_sld_gs", __sld_lr(), MCAE, pg="sld", cf=True), G("m3w_sld_gs", __sld_lr(), "mul", pg="sld", cf=True),
# gs pacc
G("bin_pacc_gs", __pacc_lr(), BQAE, pg="pacc" ), # base kde
G("mul_pacc_gs", __pacc_lr(), MCAE, pg="pacc" ), M("bin_kde", __kde_lr(), "bin" ),
G("m3w_pacc_gs", __pacc_lr(), MCAE, pg="pacc", cf=True), M("mul_kde", __kde_lr(), "mul" ),
M("m3w_kde", __kde_lr(), "mul", cf=True),
# max_conf + entropy kde
M("binc_kde", __kde_lr(), "bin", conf=["max_conf", "entropy"] ),
M("mulc_kde", __kde_lr(), "mul", conf=["max_conf", "entropy"] ),
M("m3wc_kde", __kde_lr(), "mul", conf=["max_conf", "entropy"], cf=True),
# max_conf kde
M("binmc_kde", __kde_lr(), "bin", conf="max_conf", ),
M("mulmc_kde", __kde_lr(), "mul", conf="max_conf", ),
M("m3wmc_kde", __kde_lr(), "mul", conf="max_conf", cf=True),
# entropy kde
M("binne_kde", __kde_lr(), "bin", conf="entropy", ),
M("mulne_kde", __kde_lr(), "mul", conf="entropy", ),
M("m3wne_kde", __kde_lr(), "mul", conf="entropy", cf=True),
# inverse softmax kde
M("binis_kde", __kde_lr(), "bin", conf="isoft", ),
M("mulis_kde", __kde_lr(), "mul", conf="isoft", ),
M("m3wis_kde", __kde_lr(), "mul", conf="isoft", cf=True),
# gs kde
G("bin_kde_gs", __kde_lr(), "bin", pg="kde", ),
G("mul_kde_gs", __kde_lr(), "mul", pg="kde", ),
G("m3w_kde_gs", __kde_lr(), "mul", pg="kde", cf=True),
] ]
# fmt: on # fmt: on