lorenzo.volpi
/
QuAcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

prev filtering added to DatasetReport.to_md

This commit is contained in:
Lorenzo Volpi 2023-11-16 17:13:26 +01:00
parent 04bc7576dd
commit 75c83de7d3
1 changed files with 33 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
quacc/evaluation/report.py
View File

@ -54,6 +54,15 @@ class EvaluationReport:
class CompReport: class CompReport:
_default_modes = [
"delta",
"delta_stdev",
"diagonal",
"shift",
"table",
"shift_table",
]
def __init__( def __init__(
self, self,
reports: List[EvaluationReport], reports: List[EvaluationReport],
@ -233,6 +242,8 @@ class CompReport:
res += fmt_line_md(f"train: {str(self.train_prev)}") res += fmt_line_md(f"train: {str(self.train_prev)}")
res += fmt_line_md(f"validation: {str(self.valid_prev)}") res += fmt_line_md(f"validation: {str(self.valid_prev)}")
for k, v in self.times.items(): for k, v in self.times.items():
if estimators is not None and k not in estimators:
continue
res += fmt_line_md(f"{k}: {v:.3f}s") res += fmt_line_md(f"{k}: {v:.3f}s")
res += "\n" res += "\n"
if "table" in modes: if "table" in modes:
@ -261,6 +272,18 @@ class CompReport:
class DatasetReport: class DatasetReport:
_default_dr_modes = [
"delta_train",
"stdev_train",
"delta_test",
"stdev_test",
"shift",
"train_table",
"test_table",
"shift_table",
]
_default_cr_modes = CompReport._default_modes
def __init__(self, name, crs=None): def __init__(self, name, crs=None):
self.name = name self.name = name
self.crs: List[CompReport] = [] if crs is None else crs self.crs: List[CompReport] = [] if crs is None else crs
@ -421,28 +444,18 @@ class DatasetReport:
conf="default", conf="default",
metric="acc", metric="acc",
estimators=[], estimators=[],
dr_modes=[ dr_modes=_default_dr_modes,
"delta_train", cr_modes=_default_cr_modes,
"stdev_train", cr_prevs: List[str] = None,
"delta_test",
"stdev_test",
"shift",
"train_table",
"test_table",
"shift_table",
],
cr_modes=[
"delta",
"delta_stdev",
"diagonal",
"shift",
"table",
"shift_table",
],
plot_path=None, plot_path=None,
): ):
res = f"# {self.name}\n\n" res = f"# {self.name}\n\n"
for cr in self.crs: for cr in self.crs:
if (
cr_prevs is not None
and str(round(cr.train_prev[1] * 100)) not in cr_prevs
):
continue
res += f"{cr.to_md(conf, metric=metric, estimators=estimators, modes=cr_modes, plot_path=plot_path)}\n\n" res += f"{cr.to_md(conf, metric=metric, estimators=estimators, modes=cr_modes, plot_path=plot_path)}\n\n"
_data = self.data(metric=metric, estimators=estimators) _data = self.data(metric=metric, estimators=estimators)
@ -535,6 +548,8 @@ class DatasetReport:
with open(pickle_path, "wb") as f: with open(pickle_path, "wb") as f:
pickle.dump(self, f) pickle.dump(self, f)
return self
@classmethod @classmethod
def unpickle(cls, pickle_path: Path): def unpickle(cls, pickle_path: Path):
with open(pickle_path, "rb") as f: with open(pickle_path, "rb") as f: