diff --git a/.gitignore b/.gitignore
index f450568..1ae9719 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,5 +11,4 @@ lipton_bbse/__pycache__/*
elsahar19_rca/__pycache__/*
*.coverage
.coverage
-scp_sync.py
-out/*
\ No newline at end of file
+scp_sync.py
\ No newline at end of file
diff --git a/out/plot/rcv1_CCAT_10_acc.png b/out/plot/rcv1_CCAT_10_acc.png
new file mode 100644
index 0000000..2994b60
Binary files /dev/null and b/out/plot/rcv1_CCAT_10_acc.png differ
diff --git a/out/plot/rcv1_CCAT_20_acc.png b/out/plot/rcv1_CCAT_20_acc.png
new file mode 100644
index 0000000..83a7991
Binary files /dev/null and b/out/plot/rcv1_CCAT_20_acc.png differ
diff --git a/out/plot/rcv1_CCAT_30_acc.png b/out/plot/rcv1_CCAT_30_acc.png
new file mode 100644
index 0000000..2e34308
Binary files /dev/null and b/out/plot/rcv1_CCAT_30_acc.png differ
diff --git a/out/plot/rcv1_CCAT_40_acc.png b/out/plot/rcv1_CCAT_40_acc.png
new file mode 100644
index 0000000..031feda
Binary files /dev/null and b/out/plot/rcv1_CCAT_40_acc.png differ
diff --git a/out/plot/rcv1_CCAT_50_acc.png b/out/plot/rcv1_CCAT_50_acc.png
new file mode 100644
index 0000000..86d23e7
Binary files /dev/null and b/out/plot/rcv1_CCAT_50_acc.png differ
diff --git a/out/plot/rcv1_CCAT_60_acc.png b/out/plot/rcv1_CCAT_60_acc.png
new file mode 100644
index 0000000..374cb70
Binary files /dev/null and b/out/plot/rcv1_CCAT_60_acc.png differ
diff --git a/out/plot/rcv1_CCAT_70_acc.png b/out/plot/rcv1_CCAT_70_acc.png
new file mode 100644
index 0000000..3af314e
Binary files /dev/null and b/out/plot/rcv1_CCAT_70_acc.png differ
diff --git a/out/plot/rcv1_CCAT_80_acc.png b/out/plot/rcv1_CCAT_80_acc.png
new file mode 100644
index 0000000..2f525d0
Binary files /dev/null and b/out/plot/rcv1_CCAT_80_acc.png differ
diff --git a/out/plot/rcv1_CCAT_90_acc.png b/out/plot/rcv1_CCAT_90_acc.png
new file mode 100644
index 0000000..0150b84
Binary files /dev/null and b/out/plot/rcv1_CCAT_90_acc.png differ
diff --git a/out/rcv1_CCAT.md b/out/rcv1_CCAT.md
new file mode 100644
index 0000000..1eff4e7
--- /dev/null
+++ b/out/rcv1_CCAT.md
@@ -0,0 +1,1955 @@
+rcv1_CCAT
+
+> train: [0.09996662 0.90003338]
+> validation: [0.09996662 0.90003338]
+> evaluate_bin_sld: 198.301s
+> evaluate_mul_sld: 53.156s
+> kfcv: 41.095s
+> atc_mc: 42.167s
+> atc_ne: 41.909s
+> doc_feat: 35.796s
+> tot: 202.108s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0048 |
+ 0.0040 |
+ 0.0866 |
+ 0.0243 |
+ 0.0243 |
+ 0.0830 |
+
+
+ (0.05, 0.95) |
+ 0.0060 |
+ 0.0072 |
+ 0.0441 |
+ 0.0134 |
+ 0.0134 |
+ 0.0407 |
+
+
+ (0.1, 0.9) |
+ 0.0084 |
+ 0.0103 |
+ 0.0032 |
+ 0.0070 |
+ 0.0070 |
+ 0.0036 |
+
+
+ (0.15, 0.85) |
+ 0.0127 |
+ 0.0172 |
+ 0.0418 |
+ 0.0090 |
+ 0.0090 |
+ 0.0450 |
+
+
+ (0.2, 0.8) |
+ 0.0184 |
+ 0.0246 |
+ 0.0841 |
+ 0.0168 |
+ 0.0168 |
+ 0.0872 |
+
+
+ (0.25, 0.75) |
+ 0.0231 |
+ 0.0318 |
+ 0.1246 |
+ 0.0239 |
+ 0.0239 |
+ 0.1276 |
+
+
+ (0.3, 0.7) |
+ 0.0313 |
+ 0.0426 |
+ 0.1678 |
+ 0.0334 |
+ 0.0334 |
+ 0.1706 |
+
+
+ (0.35, 0.65) |
+ 0.0392 |
+ 0.0536 |
+ 0.2110 |
+ 0.0422 |
+ 0.0422 |
+ 0.2137 |
+
+
+ (0.4, 0.6) |
+ 0.0418 |
+ 0.0563 |
+ 0.2528 |
+ 0.0541 |
+ 0.0541 |
+ 0.2555 |
+
+
+ (0.45, 0.55) |
+ 0.0527 |
+ 0.0715 |
+ 0.2966 |
+ 0.0622 |
+ 0.0622 |
+ 0.2991 |
+
+
+ (0.5, 0.5) |
+ 0.0569 |
+ 0.0771 |
+ 0.3383 |
+ 0.0749 |
+ 0.0749 |
+ 0.3407 |
+
+
+ (0.55, 0.45) |
+ 0.0637 |
+ 0.0867 |
+ 0.3817 |
+ 0.0847 |
+ 0.0847 |
+ 0.3840 |
+
+
+ (0.6, 0.4) |
+ 0.0727 |
+ 0.0999 |
+ 0.4250 |
+ 0.0943 |
+ 0.0943 |
+ 0.4272 |
+
+
+ (0.65, 0.35) |
+ 0.0778 |
+ 0.1062 |
+ 0.4662 |
+ 0.1040 |
+ 0.1040 |
+ 0.4683 |
+
+
+ (0.7, 0.3) |
+ 0.0825 |
+ 0.1118 |
+ 0.5099 |
+ 0.1131 |
+ 0.1131 |
+ 0.5119 |
+
+
+ (0.75, 0.25) |
+ 0.0879 |
+ 0.1197 |
+ 0.5519 |
+ 0.1217 |
+ 0.1217 |
+ 0.5537 |
+
+
+ (0.8, 0.2) |
+ 0.0887 |
+ 0.1192 |
+ 0.5945 |
+ 0.1334 |
+ 0.1334 |
+ 0.5963 |
+
+
+ (0.85, 0.15) |
+ 0.0926 |
+ 0.1269 |
+ 0.6368 |
+ 0.1426 |
+ 0.1426 |
+ 0.6384 |
+
+
+ (0.9, 0.1) |
+ 0.0887 |
+ 0.1250 |
+ 0.6791 |
+ 0.1528 |
+ 0.1528 |
+ 0.6806 |
+
+
+ (0.95, 0.05) |
+ 0.0501 |
+ 0.0961 |
+ 0.7227 |
+ 0.1614 |
+ 0.1614 |
+ 0.7241 |
+
+
+ (1.0, 0.0) |
+ 0.0004 |
+ 0.0358 |
+ 0.7631 |
+ 0.1704 |
+ 0.1704 |
+ 0.7643 |
+
+
+
+
+
+
+> train: [0.19993324 0.80006676]
+> validation: [0.20010013 0.79989987]
+> evaluate_bin_sld: 199.250s
+> evaluate_mul_sld: 55.414s
+> kfcv: 41.131s
+> atc_mc: 42.125s
+> atc_ne: 41.892s
+> doc_feat: 35.279s
+> tot: 202.707s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0055 |
+ 0.0058 |
+ 0.0915 |
+ 0.0147 |
+ 0.0147 |
+ 0.0775 |
+
+
+ (0.05, 0.95) |
+ 0.0157 |
+ 0.0084 |
+ 0.0719 |
+ 0.0130 |
+ 0.0130 |
+ 0.0581 |
+
+
+ (0.1, 0.9) |
+ 0.0154 |
+ 0.0099 |
+ 0.0503 |
+ 0.0108 |
+ 0.0108 |
+ 0.0365 |
+
+
+ (0.15, 0.85) |
+ 0.0141 |
+ 0.0111 |
+ 0.0292 |
+ 0.0104 |
+ 0.0104 |
+ 0.0158 |
+
+
+ (0.2, 0.8) |
+ 0.0120 |
+ 0.0116 |
+ 0.0103 |
+ 0.0100 |
+ 0.0100 |
+ 0.0068 |
+
+
+ (0.25, 0.75) |
+ 0.0098 |
+ 0.0124 |
+ 0.0115 |
+ 0.0091 |
+ 0.0091 |
+ 0.0243 |
+
+
+ (0.3, 0.7) |
+ 0.0079 |
+ 0.0131 |
+ 0.0312 |
+ 0.0106 |
+ 0.0106 |
+ 0.0445 |
+
+
+ (0.35, 0.65) |
+ 0.0087 |
+ 0.0154 |
+ 0.0529 |
+ 0.0097 |
+ 0.0097 |
+ 0.0660 |
+
+
+ (0.4, 0.6) |
+ 0.0074 |
+ 0.0143 |
+ 0.0729 |
+ 0.0110 |
+ 0.0110 |
+ 0.0859 |
+
+
+ (0.45, 0.55) |
+ 0.0082 |
+ 0.0148 |
+ 0.0933 |
+ 0.0111 |
+ 0.0111 |
+ 0.1062 |
+
+
+ (0.5, 0.5) |
+ 0.0081 |
+ 0.0152 |
+ 0.1152 |
+ 0.0136 |
+ 0.0136 |
+ 0.1280 |
+
+
+ (0.55, 0.45) |
+ 0.0104 |
+ 0.0164 |
+ 0.1384 |
+ 0.0147 |
+ 0.0147 |
+ 0.1511 |
+
+
+ (0.6, 0.4) |
+ 0.0108 |
+ 0.0193 |
+ 0.1567 |
+ 0.0168 |
+ 0.0168 |
+ 0.1692 |
+
+
+ (0.65, 0.35) |
+ 0.0129 |
+ 0.0212 |
+ 0.1806 |
+ 0.0196 |
+ 0.0196 |
+ 0.1930 |
+
+
+ (0.7, 0.3) |
+ 0.0134 |
+ 0.0242 |
+ 0.2005 |
+ 0.0178 |
+ 0.0178 |
+ 0.2128 |
+
+
+ (0.75, 0.25) |
+ 0.0162 |
+ 0.0238 |
+ 0.2196 |
+ 0.0201 |
+ 0.0201 |
+ 0.2318 |
+
+
+ (0.8, 0.2) |
+ 0.0161 |
+ 0.0248 |
+ 0.2425 |
+ 0.0214 |
+ 0.0214 |
+ 0.2546 |
+
+
+ (0.85, 0.15) |
+ 0.0207 |
+ 0.0320 |
+ 0.2620 |
+ 0.0227 |
+ 0.0227 |
+ 0.2740 |
+
+
+ (0.9, 0.1) |
+ 0.0233 |
+ 0.0340 |
+ 0.2841 |
+ 0.0267 |
+ 0.0267 |
+ 0.2960 |
+
+
+ (0.95, 0.05) |
+ 0.0261 |
+ 0.0393 |
+ 0.3034 |
+ 0.0274 |
+ 0.0274 |
+ 0.3151 |
+
+
+ (1.0, 0.0) |
+ 0.0019 |
+ 0.0162 |
+ 0.3217 |
+ 0.0311 |
+ 0.0311 |
+ 0.3333 |
+
+
+
+
+
+
+> train: [0.29989987 0.70010013]
+> validation: [0.30006676 0.69993324]
+> evaluate_bin_sld: 197.848s
+> evaluate_mul_sld: 55.610s
+> kfcv: 40.783s
+> atc_mc: 42.124s
+> atc_ne: 41.370s
+> doc_feat: 35.340s
+> tot: 199.287s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0051 |
+ 0.0059 |
+ 0.0530 |
+ 0.0059 |
+ 0.0059 |
+ 0.0422 |
+
+
+ (0.05, 0.95) |
+ 0.0108 |
+ 0.0082 |
+ 0.0455 |
+ 0.0063 |
+ 0.0063 |
+ 0.0347 |
+
+
+ (0.1, 0.9) |
+ 0.0127 |
+ 0.0110 |
+ 0.0356 |
+ 0.0062 |
+ 0.0062 |
+ 0.0250 |
+
+
+ (0.15, 0.85) |
+ 0.0147 |
+ 0.0145 |
+ 0.0265 |
+ 0.0076 |
+ 0.0076 |
+ 0.0160 |
+
+
+ (0.2, 0.8) |
+ 0.0158 |
+ 0.0162 |
+ 0.0173 |
+ 0.0071 |
+ 0.0071 |
+ 0.0086 |
+
+
+ (0.25, 0.75) |
+ 0.0147 |
+ 0.0158 |
+ 0.0091 |
+ 0.0070 |
+ 0.0070 |
+ 0.0075 |
+
+
+ (0.3, 0.7) |
+ 0.0134 |
+ 0.0162 |
+ 0.0073 |
+ 0.0080 |
+ 0.0080 |
+ 0.0127 |
+
+
+ (0.35, 0.65) |
+ 0.0138 |
+ 0.0178 |
+ 0.0132 |
+ 0.0100 |
+ 0.0100 |
+ 0.0230 |
+
+
+ (0.4, 0.6) |
+ 0.0130 |
+ 0.0180 |
+ 0.0204 |
+ 0.0096 |
+ 0.0096 |
+ 0.0306 |
+
+
+ (0.45, 0.55) |
+ 0.0102 |
+ 0.0149 |
+ 0.0297 |
+ 0.0102 |
+ 0.0102 |
+ 0.0397 |
+
+
+ (0.5, 0.5) |
+ 0.0094 |
+ 0.0160 |
+ 0.0405 |
+ 0.0111 |
+ 0.0111 |
+ 0.0504 |
+
+
+ (0.55, 0.45) |
+ 0.0095 |
+ 0.0135 |
+ 0.0516 |
+ 0.0123 |
+ 0.0123 |
+ 0.0615 |
+
+
+ (0.6, 0.4) |
+ 0.0086 |
+ 0.0132 |
+ 0.0596 |
+ 0.0122 |
+ 0.0122 |
+ 0.0693 |
+
+
+ (0.65, 0.35) |
+ 0.0102 |
+ 0.0123 |
+ 0.0717 |
+ 0.0149 |
+ 0.0149 |
+ 0.0814 |
+
+
+ (0.7, 0.3) |
+ 0.0098 |
+ 0.0115 |
+ 0.0797 |
+ 0.0160 |
+ 0.0160 |
+ 0.0894 |
+
+
+ (0.75, 0.25) |
+ 0.0111 |
+ 0.0108 |
+ 0.0880 |
+ 0.0160 |
+ 0.0160 |
+ 0.0975 |
+
+
+ (0.8, 0.2) |
+ 0.0112 |
+ 0.0093 |
+ 0.0996 |
+ 0.0206 |
+ 0.0206 |
+ 0.1091 |
+
+
+ (0.85, 0.15) |
+ 0.0149 |
+ 0.0119 |
+ 0.1094 |
+ 0.0197 |
+ 0.0197 |
+ 0.1187 |
+
+
+ (0.9, 0.1) |
+ 0.0167 |
+ 0.0137 |
+ 0.1178 |
+ 0.0216 |
+ 0.0216 |
+ 0.1271 |
+
+
+ (0.95, 0.05) |
+ 0.0184 |
+ 0.0145 |
+ 0.1275 |
+ 0.0222 |
+ 0.0222 |
+ 0.1367 |
+
+
+ (1.0, 0.0) |
+ 0.0007 |
+ 0.0099 |
+ 0.1371 |
+ 0.0238 |
+ 0.0238 |
+ 0.1462 |
+
+
+
+
+
+
+> train: [0.40003338 0.59996662]
+> validation: [0.40003338 0.59996662]
+> evaluate_bin_sld: 197.597s
+> evaluate_mul_sld: 55.556s
+> kfcv: 40.650s
+> atc_mc: 41.687s
+> atc_ne: 41.375s
+> doc_feat: 34.998s
+> tot: 198.892s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0013 |
+ 0.0048 |
+ 0.0194 |
+ 0.0071 |
+ 0.0071 |
+ 0.0126 |
+
+
+ (0.05, 0.95) |
+ 0.0076 |
+ 0.0084 |
+ 0.0184 |
+ 0.0071 |
+ 0.0071 |
+ 0.0111 |
+
+
+ (0.1, 0.9) |
+ 0.0092 |
+ 0.0107 |
+ 0.0161 |
+ 0.0078 |
+ 0.0078 |
+ 0.0093 |
+
+
+ (0.15, 0.85) |
+ 0.0127 |
+ 0.0149 |
+ 0.0134 |
+ 0.0070 |
+ 0.0070 |
+ 0.0077 |
+
+
+ (0.2, 0.8) |
+ 0.0183 |
+ 0.0200 |
+ 0.0110 |
+ 0.0066 |
+ 0.0066 |
+ 0.0075 |
+
+
+ (0.25, 0.75) |
+ 0.0208 |
+ 0.0230 |
+ 0.0090 |
+ 0.0075 |
+ 0.0075 |
+ 0.0069 |
+
+
+ (0.3, 0.7) |
+ 0.0235 |
+ 0.0260 |
+ 0.0080 |
+ 0.0076 |
+ 0.0076 |
+ 0.0073 |
+
+
+ (0.35, 0.65) |
+ 0.0273 |
+ 0.0306 |
+ 0.0065 |
+ 0.0079 |
+ 0.0079 |
+ 0.0095 |
+
+
+ (0.4, 0.6) |
+ 0.0296 |
+ 0.0335 |
+ 0.0074 |
+ 0.0072 |
+ 0.0072 |
+ 0.0099 |
+
+
+ (0.45, 0.55) |
+ 0.0283 |
+ 0.0313 |
+ 0.0080 |
+ 0.0085 |
+ 0.0085 |
+ 0.0116 |
+
+
+ (0.5, 0.5) |
+ 0.0267 |
+ 0.0317 |
+ 0.0087 |
+ 0.0085 |
+ 0.0085 |
+ 0.0147 |
+
+
+ (0.55, 0.45) |
+ 0.0273 |
+ 0.0331 |
+ 0.0131 |
+ 0.0086 |
+ 0.0086 |
+ 0.0196 |
+
+
+ (0.6, 0.4) |
+ 0.0239 |
+ 0.0320 |
+ 0.0136 |
+ 0.0082 |
+ 0.0082 |
+ 0.0202 |
+
+
+ (0.65, 0.35) |
+ 0.0208 |
+ 0.0290 |
+ 0.0171 |
+ 0.0084 |
+ 0.0084 |
+ 0.0241 |
+
+
+ (0.7, 0.3) |
+ 0.0186 |
+ 0.0288 |
+ 0.0213 |
+ 0.0084 |
+ 0.0084 |
+ 0.0281 |
+
+
+ (0.75, 0.25) |
+ 0.0158 |
+ 0.0261 |
+ 0.0219 |
+ 0.0090 |
+ 0.0090 |
+ 0.0288 |
+
+
+ (0.8, 0.2) |
+ 0.0130 |
+ 0.0235 |
+ 0.0269 |
+ 0.0089 |
+ 0.0089 |
+ 0.0338 |
+
+
+ (0.85, 0.15) |
+ 0.0084 |
+ 0.0180 |
+ 0.0284 |
+ 0.0083 |
+ 0.0083 |
+ 0.0352 |
+
+
+ (0.9, 0.1) |
+ 0.0057 |
+ 0.0134 |
+ 0.0322 |
+ 0.0092 |
+ 0.0092 |
+ 0.0390 |
+
+
+ (0.95, 0.05) |
+ 0.0050 |
+ 0.0091 |
+ 0.0339 |
+ 0.0101 |
+ 0.0101 |
+ 0.0406 |
+
+
+ (1.0, 0.0) |
+ 0.0007 |
+ 0.0064 |
+ 0.0379 |
+ 0.0106 |
+ 0.0106 |
+ 0.0447 |
+
+
+
+
+
+
+> train: [0.5 0.5]
+> validation: [0.5 0.5]
+> evaluate_bin_sld: 197.283s
+> evaluate_mul_sld: 54.736s
+> kfcv: 40.375s
+> atc_mc: 41.898s
+> atc_ne: 41.366s
+> doc_feat: 35.145s
+> tot: 198.630s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0004 |
+ 0.0035 |
+ 0.0257 |
+ 0.0289 |
+ 0.0289 |
+ 0.0344 |
+
+
+ (0.05, 0.95) |
+ 0.0075 |
+ 0.0085 |
+ 0.0224 |
+ 0.0253 |
+ 0.0253 |
+ 0.0310 |
+
+
+ (0.1, 0.9) |
+ 0.0081 |
+ 0.0122 |
+ 0.0205 |
+ 0.0239 |
+ 0.0239 |
+ 0.0292 |
+
+
+ (0.15, 0.85) |
+ 0.0102 |
+ 0.0148 |
+ 0.0180 |
+ 0.0205 |
+ 0.0205 |
+ 0.0267 |
+
+
+ (0.2, 0.8) |
+ 0.0139 |
+ 0.0198 |
+ 0.0165 |
+ 0.0211 |
+ 0.0211 |
+ 0.0248 |
+
+
+ (0.25, 0.75) |
+ 0.0194 |
+ 0.0245 |
+ 0.0141 |
+ 0.0170 |
+ 0.0170 |
+ 0.0224 |
+
+
+ (0.3, 0.7) |
+ 0.0230 |
+ 0.0287 |
+ 0.0137 |
+ 0.0164 |
+ 0.0164 |
+ 0.0222 |
+
+
+ (0.35, 0.65) |
+ 0.0309 |
+ 0.0338 |
+ 0.0132 |
+ 0.0168 |
+ 0.0168 |
+ 0.0210 |
+
+
+ (0.4, 0.6) |
+ 0.0350 |
+ 0.0371 |
+ 0.0097 |
+ 0.0144 |
+ 0.0144 |
+ 0.0164 |
+
+
+ (0.45, 0.55) |
+ 0.0358 |
+ 0.0390 |
+ 0.0086 |
+ 0.0125 |
+ 0.0125 |
+ 0.0150 |
+
+
+ (0.5, 0.5) |
+ 0.0369 |
+ 0.0386 |
+ 0.0073 |
+ 0.0122 |
+ 0.0122 |
+ 0.0138 |
+
+
+ (0.55, 0.45) |
+ 0.0373 |
+ 0.0398 |
+ 0.0071 |
+ 0.0110 |
+ 0.0110 |
+ 0.0128 |
+
+
+ (0.6, 0.4) |
+ 0.0368 |
+ 0.0398 |
+ 0.0064 |
+ 0.0085 |
+ 0.0085 |
+ 0.0103 |
+
+
+ (0.65, 0.35) |
+ 0.0357 |
+ 0.0385 |
+ 0.0074 |
+ 0.0103 |
+ 0.0103 |
+ 0.0105 |
+
+
+ (0.7, 0.3) |
+ 0.0319 |
+ 0.0370 |
+ 0.0067 |
+ 0.0082 |
+ 0.0082 |
+ 0.0086 |
+
+
+ (0.75, 0.25) |
+ 0.0298 |
+ 0.0358 |
+ 0.0079 |
+ 0.0066 |
+ 0.0066 |
+ 0.0070 |
+
+
+ (0.8, 0.2) |
+ 0.0235 |
+ 0.0302 |
+ 0.0073 |
+ 0.0083 |
+ 0.0083 |
+ 0.0069 |
+
+
+ (0.85, 0.15) |
+ 0.0154 |
+ 0.0244 |
+ 0.0097 |
+ 0.0077 |
+ 0.0077 |
+ 0.0066 |
+
+
+ (0.9, 0.1) |
+ 0.0083 |
+ 0.0157 |
+ 0.0108 |
+ 0.0082 |
+ 0.0082 |
+ 0.0069 |
+
+
+ (0.95, 0.05) |
+ 0.0055 |
+ 0.0098 |
+ 0.0131 |
+ 0.0080 |
+ 0.0080 |
+ 0.0066 |
+
+
+ (1.0, 0.0) |
+ 0.0007 |
+ 0.0046 |
+ 0.0145 |
+ 0.0088 |
+ 0.0088 |
+ 0.0082 |
+
+
+
+
+
+
+> train: [0.59996662 0.40003338]
+> validation: [0.59996662 0.40003338]
+> evaluate_bin_sld: 194.960s
+> evaluate_mul_sld: 53.330s
+> kfcv: 40.320s
+> atc_mc: 41.904s
+> atc_ne: 41.423s
+> doc_feat: 35.289s
+> tot: 196.151s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0003 |
+ 0.0055 |
+ 0.0815 |
+ 0.0285 |
+ 0.0285 |
+ 0.0825 |
+
+
+ (0.05, 0.95) |
+ 0.0065 |
+ 0.0127 |
+ 0.0747 |
+ 0.0278 |
+ 0.0278 |
+ 0.0758 |
+
+
+ (0.1, 0.9) |
+ 0.0072 |
+ 0.0172 |
+ 0.0677 |
+ 0.0224 |
+ 0.0224 |
+ 0.0688 |
+
+
+ (0.15, 0.85) |
+ 0.0100 |
+ 0.0257 |
+ 0.0627 |
+ 0.0218 |
+ 0.0218 |
+ 0.0638 |
+
+
+ (0.2, 0.8) |
+ 0.0135 |
+ 0.0308 |
+ 0.0548 |
+ 0.0180 |
+ 0.0180 |
+ 0.0560 |
+
+
+ (0.25, 0.75) |
+ 0.0165 |
+ 0.0338 |
+ 0.0491 |
+ 0.0160 |
+ 0.0160 |
+ 0.0503 |
+
+
+ (0.3, 0.7) |
+ 0.0205 |
+ 0.0409 |
+ 0.0438 |
+ 0.0168 |
+ 0.0168 |
+ 0.0450 |
+
+
+ (0.35, 0.65) |
+ 0.0248 |
+ 0.0459 |
+ 0.0374 |
+ 0.0156 |
+ 0.0156 |
+ 0.0386 |
+
+
+ (0.4, 0.6) |
+ 0.0284 |
+ 0.0491 |
+ 0.0277 |
+ 0.0112 |
+ 0.0112 |
+ 0.0290 |
+
+
+ (0.45, 0.55) |
+ 0.0318 |
+ 0.0515 |
+ 0.0224 |
+ 0.0099 |
+ 0.0099 |
+ 0.0237 |
+
+
+ (0.5, 0.5) |
+ 0.0342 |
+ 0.0516 |
+ 0.0159 |
+ 0.0081 |
+ 0.0081 |
+ 0.0170 |
+
+
+ (0.55, 0.45) |
+ 0.0374 |
+ 0.0519 |
+ 0.0111 |
+ 0.0073 |
+ 0.0073 |
+ 0.0121 |
+
+
+ (0.6, 0.4) |
+ 0.0410 |
+ 0.0537 |
+ 0.0069 |
+ 0.0079 |
+ 0.0079 |
+ 0.0075 |
+
+
+ (0.65, 0.35) |
+ 0.0444 |
+ 0.0517 |
+ 0.0064 |
+ 0.0076 |
+ 0.0076 |
+ 0.0064 |
+
+
+ (0.7, 0.3) |
+ 0.0438 |
+ 0.0502 |
+ 0.0100 |
+ 0.0085 |
+ 0.0085 |
+ 0.0090 |
+
+
+ (0.75, 0.25) |
+ 0.0458 |
+ 0.0483 |
+ 0.0171 |
+ 0.0089 |
+ 0.0089 |
+ 0.0157 |
+
+
+ (0.8, 0.2) |
+ 0.0412 |
+ 0.0419 |
+ 0.0218 |
+ 0.0105 |
+ 0.0105 |
+ 0.0204 |
+
+
+ (0.85, 0.15) |
+ 0.0319 |
+ 0.0348 |
+ 0.0291 |
+ 0.0117 |
+ 0.0117 |
+ 0.0276 |
+
+
+ (0.9, 0.1) |
+ 0.0192 |
+ 0.0254 |
+ 0.0358 |
+ 0.0147 |
+ 0.0147 |
+ 0.0343 |
+
+
+ (0.95, 0.05) |
+ 0.0079 |
+ 0.0154 |
+ 0.0427 |
+ 0.0166 |
+ 0.0166 |
+ 0.0412 |
+
+
+ (1.0, 0.0) |
+ 0.0005 |
+ 0.0034 |
+ 0.0490 |
+ 0.0190 |
+ 0.0190 |
+ 0.0474 |
+
+
+
+
+
+
+> train: [0.69993324 0.30006676]
+> validation: [0.70010013 0.29989987]
+> evaluate_bin_sld: 196.856s
+> evaluate_mul_sld: 54.245s
+> kfcv: 41.167s
+> atc_mc: 42.203s
+> atc_ne: 41.565s
+> doc_feat: 34.998s
+> tot: 198.332s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0003 |
+ 0.0071 |
+ 0.1570 |
+ 0.0625 |
+ 0.0625 |
+ 0.1677 |
+
+
+ (0.05, 0.95) |
+ 0.0089 |
+ 0.0102 |
+ 0.1428 |
+ 0.0548 |
+ 0.0548 |
+ 0.1536 |
+
+
+ (0.1, 0.9) |
+ 0.0078 |
+ 0.0121 |
+ 0.1327 |
+ 0.0521 |
+ 0.0521 |
+ 0.1435 |
+
+
+ (0.15, 0.85) |
+ 0.0073 |
+ 0.0155 |
+ 0.1227 |
+ 0.0517 |
+ 0.0517 |
+ 0.1336 |
+
+
+ (0.2, 0.8) |
+ 0.0081 |
+ 0.0196 |
+ 0.1094 |
+ 0.0464 |
+ 0.0464 |
+ 0.1203 |
+
+
+ (0.25, 0.75) |
+ 0.0095 |
+ 0.0225 |
+ 0.1001 |
+ 0.0427 |
+ 0.0427 |
+ 0.1111 |
+
+
+ (0.3, 0.7) |
+ 0.0117 |
+ 0.0272 |
+ 0.0885 |
+ 0.0400 |
+ 0.0400 |
+ 0.0995 |
+
+
+ (0.35, 0.65) |
+ 0.0131 |
+ 0.0309 |
+ 0.0774 |
+ 0.0368 |
+ 0.0368 |
+ 0.0885 |
+
+
+ (0.4, 0.6) |
+ 0.0144 |
+ 0.0333 |
+ 0.0626 |
+ 0.0307 |
+ 0.0307 |
+ 0.0737 |
+
+
+ (0.45, 0.55) |
+ 0.0179 |
+ 0.0365 |
+ 0.0528 |
+ 0.0297 |
+ 0.0297 |
+ 0.0640 |
+
+
+ (0.5, 0.5) |
+ 0.0183 |
+ 0.0359 |
+ 0.0418 |
+ 0.0259 |
+ 0.0259 |
+ 0.0531 |
+
+
+ (0.55, 0.45) |
+ 0.0189 |
+ 0.0369 |
+ 0.0313 |
+ 0.0222 |
+ 0.0222 |
+ 0.0426 |
+
+
+ (0.6, 0.4) |
+ 0.0220 |
+ 0.0379 |
+ 0.0201 |
+ 0.0190 |
+ 0.0190 |
+ 0.0314 |
+
+
+ (0.65, 0.35) |
+ 0.0218 |
+ 0.0364 |
+ 0.0104 |
+ 0.0160 |
+ 0.0160 |
+ 0.0208 |
+
+
+ (0.7, 0.3) |
+ 0.0229 |
+ 0.0371 |
+ 0.0067 |
+ 0.0119 |
+ 0.0119 |
+ 0.0096 |
+
+
+ (0.75, 0.25) |
+ 0.0250 |
+ 0.0378 |
+ 0.0161 |
+ 0.0101 |
+ 0.0101 |
+ 0.0067 |
+
+
+ (0.8, 0.2) |
+ 0.0237 |
+ 0.0333 |
+ 0.0259 |
+ 0.0082 |
+ 0.0082 |
+ 0.0143 |
+
+
+ (0.85, 0.15) |
+ 0.0227 |
+ 0.0282 |
+ 0.0381 |
+ 0.0060 |
+ 0.0060 |
+ 0.0265 |
+
+
+ (0.9, 0.1) |
+ 0.0180 |
+ 0.0202 |
+ 0.0499 |
+ 0.0049 |
+ 0.0049 |
+ 0.0382 |
+
+
+ (0.95, 0.05) |
+ 0.0097 |
+ 0.0117 |
+ 0.0607 |
+ 0.0072 |
+ 0.0072 |
+ 0.0489 |
+
+
+ (1.0, 0.0) |
+ 0.0014 |
+ 0.0024 |
+ 0.0724 |
+ 0.0103 |
+ 0.0103 |
+ 0.0606 |
+
+
+
+
+
+
+> train: [0.79989987 0.20010013]
+> validation: [0.80006676 0.19993324]
+> evaluate_bin_sld: 197.725s
+> evaluate_mul_sld: 53.526s
+> kfcv: 40.971s
+> atc_mc: 41.975s
+> atc_ne: 41.358s
+> doc_feat: 35.091s
+> tot: 199.051s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0009 |
+ 0.0082 |
+ 0.3148 |
+ 0.0571 |
+ 0.0571 |
+ 0.3213 |
+
+
+ (0.05, 0.95) |
+ 0.0297 |
+ 0.0223 |
+ 0.2925 |
+ 0.0492 |
+ 0.0492 |
+ 0.2991 |
+
+
+ (0.1, 0.9) |
+ 0.0283 |
+ 0.0209 |
+ 0.2733 |
+ 0.0493 |
+ 0.0493 |
+ 0.2800 |
+
+
+ (0.15, 0.85) |
+ 0.0247 |
+ 0.0182 |
+ 0.2528 |
+ 0.0447 |
+ 0.0447 |
+ 0.2596 |
+
+
+ (0.2, 0.8) |
+ 0.0216 |
+ 0.0156 |
+ 0.2328 |
+ 0.0407 |
+ 0.0407 |
+ 0.2397 |
+
+
+ (0.25, 0.75) |
+ 0.0170 |
+ 0.0136 |
+ 0.2136 |
+ 0.0425 |
+ 0.0425 |
+ 0.2205 |
+
+
+ (0.3, 0.7) |
+ 0.0146 |
+ 0.0126 |
+ 0.1941 |
+ 0.0384 |
+ 0.0384 |
+ 0.2012 |
+
+
+ (0.35, 0.65) |
+ 0.0125 |
+ 0.0113 |
+ 0.1734 |
+ 0.0331 |
+ 0.0331 |
+ 0.1806 |
+
+
+ (0.4, 0.6) |
+ 0.0113 |
+ 0.0110 |
+ 0.1510 |
+ 0.0272 |
+ 0.0272 |
+ 0.1583 |
+
+
+ (0.45, 0.55) |
+ 0.0093 |
+ 0.0135 |
+ 0.1328 |
+ 0.0247 |
+ 0.0247 |
+ 0.1402 |
+
+
+ (0.5, 0.5) |
+ 0.0088 |
+ 0.0135 |
+ 0.1131 |
+ 0.0222 |
+ 0.0222 |
+ 0.1206 |
+
+
+ (0.55, 0.45) |
+ 0.0092 |
+ 0.0155 |
+ 0.0919 |
+ 0.0207 |
+ 0.0207 |
+ 0.0995 |
+
+
+ (0.6, 0.4) |
+ 0.0092 |
+ 0.0173 |
+ 0.0742 |
+ 0.0190 |
+ 0.0190 |
+ 0.0819 |
+
+
+ (0.65, 0.35) |
+ 0.0087 |
+ 0.0178 |
+ 0.0544 |
+ 0.0161 |
+ 0.0161 |
+ 0.0621 |
+
+
+ (0.7, 0.3) |
+ 0.0093 |
+ 0.0197 |
+ 0.0323 |
+ 0.0124 |
+ 0.0124 |
+ 0.0401 |
+
+
+ (0.75, 0.25) |
+ 0.0101 |
+ 0.0218 |
+ 0.0114 |
+ 0.0093 |
+ 0.0093 |
+ 0.0187 |
+
+
+ (0.8, 0.2) |
+ 0.0117 |
+ 0.0208 |
+ 0.0098 |
+ 0.0088 |
+ 0.0088 |
+ 0.0063 |
+
+
+ (0.85, 0.15) |
+ 0.0103 |
+ 0.0178 |
+ 0.0285 |
+ 0.0064 |
+ 0.0064 |
+ 0.0204 |
+
+
+ (0.9, 0.1) |
+ 0.0103 |
+ 0.0164 |
+ 0.0480 |
+ 0.0062 |
+ 0.0062 |
+ 0.0398 |
+
+
+ (0.95, 0.05) |
+ 0.0092 |
+ 0.0117 |
+ 0.0684 |
+ 0.0071 |
+ 0.0071 |
+ 0.0601 |
+
+
+ (1.0, 0.0) |
+ 0.0011 |
+ 0.0019 |
+ 0.0887 |
+ 0.0097 |
+ 0.0097 |
+ 0.0803 |
+
+
+
+
+
+
+> train: [0.90003338 0.09996662]
+> validation: [0.90003338 0.09996662]
+> evaluate_bin_sld: 201.315s
+> evaluate_mul_sld: 50.974s
+> kfcv: 40.175s
+> atc_mc: 41.663s
+> atc_ne: 41.058s
+> doc_feat: 35.055s
+> tot: 202.573s
+
+
+
+
+ |
+ bin |
+ mul |
+ kfcv |
+ atc_mc |
+ atc_ne |
+ doc_feat |
+
+
+
+
+ (0.0, 1.0) |
+ 0.0321 |
+ 0.0184 |
+ 0.6421 |
+ 0.1336 |
+ 0.1336 |
+ 0.6454 |
+
+
+ (0.05, 0.95) |
+ 0.0835 |
+ 0.0729 |
+ 0.6056 |
+ 0.1244 |
+ 0.1244 |
+ 0.6090 |
+
+
+ (0.1, 0.9) |
+ 0.1080 |
+ 0.0976 |
+ 0.5703 |
+ 0.1204 |
+ 0.1204 |
+ 0.5739 |
+
+
+ (0.15, 0.85) |
+ 0.1154 |
+ 0.0971 |
+ 0.5354 |
+ 0.1147 |
+ 0.1147 |
+ 0.5390 |
+
+
+ (0.2, 0.8) |
+ 0.1081 |
+ 0.0916 |
+ 0.5007 |
+ 0.1064 |
+ 0.1064 |
+ 0.5045 |
+
+
+ (0.25, 0.75) |
+ 0.1032 |
+ 0.0830 |
+ 0.4632 |
+ 0.1005 |
+ 0.1005 |
+ 0.4671 |
+
+
+ (0.3, 0.7) |
+ 0.0945 |
+ 0.0775 |
+ 0.4274 |
+ 0.0916 |
+ 0.0916 |
+ 0.4313 |
+
+
+ (0.35, 0.65) |
+ 0.0966 |
+ 0.0709 |
+ 0.3914 |
+ 0.0843 |
+ 0.0843 |
+ 0.3954 |
+
+
+ (0.4, 0.6) |
+ 0.0795 |
+ 0.0639 |
+ 0.3543 |
+ 0.0748 |
+ 0.0748 |
+ 0.3584 |
+
+
+ (0.45, 0.55) |
+ 0.0735 |
+ 0.0533 |
+ 0.3210 |
+ 0.0728 |
+ 0.0728 |
+ 0.3253 |
+
+
+ (0.5, 0.5) |
+ 0.0716 |
+ 0.0473 |
+ 0.2829 |
+ 0.0633 |
+ 0.0633 |
+ 0.2873 |
+
+
+ (0.55, 0.45) |
+ 0.0550 |
+ 0.0393 |
+ 0.2465 |
+ 0.0568 |
+ 0.0568 |
+ 0.2509 |
+
+
+ (0.6, 0.4) |
+ 0.0505 |
+ 0.0317 |
+ 0.2117 |
+ 0.0509 |
+ 0.0509 |
+ 0.2162 |
+
+
+ (0.65, 0.35) |
+ 0.0403 |
+ 0.0226 |
+ 0.1741 |
+ 0.0438 |
+ 0.0438 |
+ 0.1788 |
+
+
+ (0.7, 0.3) |
+ 0.0372 |
+ 0.0178 |
+ 0.1387 |
+ 0.0348 |
+ 0.0348 |
+ 0.1434 |
+
+
+ (0.75, 0.25) |
+ 0.0262 |
+ 0.0122 |
+ 0.1009 |
+ 0.0256 |
+ 0.0256 |
+ 0.1057 |
+
+
+ (0.8, 0.2) |
+ 0.0248 |
+ 0.0110 |
+ 0.0651 |
+ 0.0194 |
+ 0.0194 |
+ 0.0701 |
+
+
+ (0.85, 0.15) |
+ 0.0181 |
+ 0.0075 |
+ 0.0298 |
+ 0.0128 |
+ 0.0128 |
+ 0.0348 |
+
+
+ (0.9, 0.1) |
+ 0.0129 |
+ 0.0093 |
+ 0.0069 |
+ 0.0080 |
+ 0.0080 |
+ 0.0037 |
+
+
+ (0.95, 0.05) |
+ 0.0077 |
+ 0.0085 |
+ 0.0426 |
+ 0.0046 |
+ 0.0046 |
+ 0.0373 |
+
+
+ (1.0, 0.0) |
+ 0.0010 |
+ 0.0010 |
+ 0.0789 |
+ 0.0088 |
+ 0.0088 |
+ 0.0735 |
+
+
+
+
+
+
diff --git a/quacc/evaluation/report.py b/quacc/evaluation/report.py
index 0236b98..3949d6b 100644
--- a/quacc/evaluation/report.py
+++ b/quacc/evaluation/report.py
@@ -69,6 +69,12 @@ class EvaluationReport:
columns=g_dict.keys(),
)
+ def get_plot(self, mode="delta", metric="acc"):
+ g_prevs, g_dict = self.groupby_prevs(metric=metric)
+ t_prev = int(round(self.train_prevs["train"][0] * 100))
+ title = f"{self.name}_{t_prev}_{metric}"
+ plot.plot_delta(g_prevs, g_dict, metric, title)
+
def to_md(self, *metrics):
res = ""
for k, v in self.train_prevs.items():
@@ -78,6 +84,7 @@ class EvaluationReport:
res += "\n"
for m in metrics:
res += self.get_dataframe(metric=m).to_html() + "\n\n"
+ self.get_plot(metric=m)
return res
diff --git a/quacc/plot.py b/quacc/plot.py
new file mode 100644
index 0000000..79977d7
--- /dev/null
+++ b/quacc/plot.py
@@ -0,0 +1,26 @@
+import matplotlib.pyplot as plt
+
+from quacc.environ import env
+
+
+def plot_delta(base_prevs, dict_vals, metric, title):
+ fig, ax = plt.subplots()
+
+ base_prevs = [f for f, p in base_prevs]
+ for method, deltas in dict_vals.items():
+ ax.plot(
+ base_prevs,
+ deltas,
+ label=method,
+ linestyle="-",
+ marker="o",
+ markersize=3,
+ zorder=2,
+ )
+
+ ax.set(xlabel="test prevalence", ylabel=metric, title=title)
+ # ax.set_ylim(0, 1)
+ # ax.set_xlim(0, 1)
+ ax.legend()
+ output_path = env.PLOT_OUT_DIR / f"{title}.png"
+ plt.savefig(output_path)