diff --git a/.gitignore b/.gitignore index f450568..1ae9719 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,4 @@ lipton_bbse/__pycache__/* elsahar19_rca/__pycache__/* *.coverage .coverage -scp_sync.py -out/* \ No newline at end of file +scp_sync.py \ No newline at end of file diff --git a/out/plot/rcv1_CCAT_10_acc.png b/out/plot/rcv1_CCAT_10_acc.png new file mode 100644 index 0000000..2994b60 Binary files /dev/null and b/out/plot/rcv1_CCAT_10_acc.png differ diff --git a/out/plot/rcv1_CCAT_20_acc.png b/out/plot/rcv1_CCAT_20_acc.png new file mode 100644 index 0000000..83a7991 Binary files /dev/null and b/out/plot/rcv1_CCAT_20_acc.png differ diff --git a/out/plot/rcv1_CCAT_30_acc.png b/out/plot/rcv1_CCAT_30_acc.png new file mode 100644 index 0000000..2e34308 Binary files /dev/null and b/out/plot/rcv1_CCAT_30_acc.png differ diff --git a/out/plot/rcv1_CCAT_40_acc.png b/out/plot/rcv1_CCAT_40_acc.png new file mode 100644 index 0000000..031feda Binary files /dev/null and b/out/plot/rcv1_CCAT_40_acc.png differ diff --git a/out/plot/rcv1_CCAT_50_acc.png b/out/plot/rcv1_CCAT_50_acc.png new file mode 100644 index 0000000..86d23e7 Binary files /dev/null and b/out/plot/rcv1_CCAT_50_acc.png differ diff --git a/out/plot/rcv1_CCAT_60_acc.png b/out/plot/rcv1_CCAT_60_acc.png new file mode 100644 index 0000000..374cb70 Binary files /dev/null and b/out/plot/rcv1_CCAT_60_acc.png differ diff --git a/out/plot/rcv1_CCAT_70_acc.png b/out/plot/rcv1_CCAT_70_acc.png new file mode 100644 index 0000000..3af314e Binary files /dev/null and b/out/plot/rcv1_CCAT_70_acc.png differ diff --git a/out/plot/rcv1_CCAT_80_acc.png b/out/plot/rcv1_CCAT_80_acc.png new file mode 100644 index 0000000..2f525d0 Binary files /dev/null and b/out/plot/rcv1_CCAT_80_acc.png differ diff --git a/out/plot/rcv1_CCAT_90_acc.png b/out/plot/rcv1_CCAT_90_acc.png new file mode 100644 index 0000000..0150b84 Binary files /dev/null and b/out/plot/rcv1_CCAT_90_acc.png differ diff --git a/out/rcv1_CCAT.md b/out/rcv1_CCAT.md new file mode 100644 index 0000000..1eff4e7 --- /dev/null +++ b/out/rcv1_CCAT.md @@ -0,0 +1,1955 @@ +rcv1_CCAT + +> train: [0.09996662 0.90003338] +> validation: [0.09996662 0.90003338] +> evaluate_bin_sld: 198.301s +> evaluate_mul_sld: 53.156s +> kfcv: 41.095s +> atc_mc: 42.167s +> atc_ne: 41.909s +> doc_feat: 35.796s +> tot: 202.108s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00480.00400.08660.02430.02430.0830
(0.05, 0.95)0.00600.00720.04410.01340.01340.0407
(0.1, 0.9)0.00840.01030.00320.00700.00700.0036
(0.15, 0.85)0.01270.01720.04180.00900.00900.0450
(0.2, 0.8)0.01840.02460.08410.01680.01680.0872
(0.25, 0.75)0.02310.03180.12460.02390.02390.1276
(0.3, 0.7)0.03130.04260.16780.03340.03340.1706
(0.35, 0.65)0.03920.05360.21100.04220.04220.2137
(0.4, 0.6)0.04180.05630.25280.05410.05410.2555
(0.45, 0.55)0.05270.07150.29660.06220.06220.2991
(0.5, 0.5)0.05690.07710.33830.07490.07490.3407
(0.55, 0.45)0.06370.08670.38170.08470.08470.3840
(0.6, 0.4)0.07270.09990.42500.09430.09430.4272
(0.65, 0.35)0.07780.10620.46620.10400.10400.4683
(0.7, 0.3)0.08250.11180.50990.11310.11310.5119
(0.75, 0.25)0.08790.11970.55190.12170.12170.5537
(0.8, 0.2)0.08870.11920.59450.13340.13340.5963
(0.85, 0.15)0.09260.12690.63680.14260.14260.6384
(0.9, 0.1)0.08870.12500.67910.15280.15280.6806
(0.95, 0.05)0.05010.09610.72270.16140.16140.7241
(1.0, 0.0)0.00040.03580.76310.17040.17040.7643
+ + + +> train: [0.19993324 0.80006676] +> validation: [0.20010013 0.79989987] +> evaluate_bin_sld: 199.250s +> evaluate_mul_sld: 55.414s +> kfcv: 41.131s +> atc_mc: 42.125s +> atc_ne: 41.892s +> doc_feat: 35.279s +> tot: 202.707s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00550.00580.09150.01470.01470.0775
(0.05, 0.95)0.01570.00840.07190.01300.01300.0581
(0.1, 0.9)0.01540.00990.05030.01080.01080.0365
(0.15, 0.85)0.01410.01110.02920.01040.01040.0158
(0.2, 0.8)0.01200.01160.01030.01000.01000.0068
(0.25, 0.75)0.00980.01240.01150.00910.00910.0243
(0.3, 0.7)0.00790.01310.03120.01060.01060.0445
(0.35, 0.65)0.00870.01540.05290.00970.00970.0660
(0.4, 0.6)0.00740.01430.07290.01100.01100.0859
(0.45, 0.55)0.00820.01480.09330.01110.01110.1062
(0.5, 0.5)0.00810.01520.11520.01360.01360.1280
(0.55, 0.45)0.01040.01640.13840.01470.01470.1511
(0.6, 0.4)0.01080.01930.15670.01680.01680.1692
(0.65, 0.35)0.01290.02120.18060.01960.01960.1930
(0.7, 0.3)0.01340.02420.20050.01780.01780.2128
(0.75, 0.25)0.01620.02380.21960.02010.02010.2318
(0.8, 0.2)0.01610.02480.24250.02140.02140.2546
(0.85, 0.15)0.02070.03200.26200.02270.02270.2740
(0.9, 0.1)0.02330.03400.28410.02670.02670.2960
(0.95, 0.05)0.02610.03930.30340.02740.02740.3151
(1.0, 0.0)0.00190.01620.32170.03110.03110.3333
+ + + +> train: [0.29989987 0.70010013] +> validation: [0.30006676 0.69993324] +> evaluate_bin_sld: 197.848s +> evaluate_mul_sld: 55.610s +> kfcv: 40.783s +> atc_mc: 42.124s +> atc_ne: 41.370s +> doc_feat: 35.340s +> tot: 199.287s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00510.00590.05300.00590.00590.0422
(0.05, 0.95)0.01080.00820.04550.00630.00630.0347
(0.1, 0.9)0.01270.01100.03560.00620.00620.0250
(0.15, 0.85)0.01470.01450.02650.00760.00760.0160
(0.2, 0.8)0.01580.01620.01730.00710.00710.0086
(0.25, 0.75)0.01470.01580.00910.00700.00700.0075
(0.3, 0.7)0.01340.01620.00730.00800.00800.0127
(0.35, 0.65)0.01380.01780.01320.01000.01000.0230
(0.4, 0.6)0.01300.01800.02040.00960.00960.0306
(0.45, 0.55)0.01020.01490.02970.01020.01020.0397
(0.5, 0.5)0.00940.01600.04050.01110.01110.0504
(0.55, 0.45)0.00950.01350.05160.01230.01230.0615
(0.6, 0.4)0.00860.01320.05960.01220.01220.0693
(0.65, 0.35)0.01020.01230.07170.01490.01490.0814
(0.7, 0.3)0.00980.01150.07970.01600.01600.0894
(0.75, 0.25)0.01110.01080.08800.01600.01600.0975
(0.8, 0.2)0.01120.00930.09960.02060.02060.1091
(0.85, 0.15)0.01490.01190.10940.01970.01970.1187
(0.9, 0.1)0.01670.01370.11780.02160.02160.1271
(0.95, 0.05)0.01840.01450.12750.02220.02220.1367
(1.0, 0.0)0.00070.00990.13710.02380.02380.1462
+ + + +> train: [0.40003338 0.59996662] +> validation: [0.40003338 0.59996662] +> evaluate_bin_sld: 197.597s +> evaluate_mul_sld: 55.556s +> kfcv: 40.650s +> atc_mc: 41.687s +> atc_ne: 41.375s +> doc_feat: 34.998s +> tot: 198.892s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00130.00480.01940.00710.00710.0126
(0.05, 0.95)0.00760.00840.01840.00710.00710.0111
(0.1, 0.9)0.00920.01070.01610.00780.00780.0093
(0.15, 0.85)0.01270.01490.01340.00700.00700.0077
(0.2, 0.8)0.01830.02000.01100.00660.00660.0075
(0.25, 0.75)0.02080.02300.00900.00750.00750.0069
(0.3, 0.7)0.02350.02600.00800.00760.00760.0073
(0.35, 0.65)0.02730.03060.00650.00790.00790.0095
(0.4, 0.6)0.02960.03350.00740.00720.00720.0099
(0.45, 0.55)0.02830.03130.00800.00850.00850.0116
(0.5, 0.5)0.02670.03170.00870.00850.00850.0147
(0.55, 0.45)0.02730.03310.01310.00860.00860.0196
(0.6, 0.4)0.02390.03200.01360.00820.00820.0202
(0.65, 0.35)0.02080.02900.01710.00840.00840.0241
(0.7, 0.3)0.01860.02880.02130.00840.00840.0281
(0.75, 0.25)0.01580.02610.02190.00900.00900.0288
(0.8, 0.2)0.01300.02350.02690.00890.00890.0338
(0.85, 0.15)0.00840.01800.02840.00830.00830.0352
(0.9, 0.1)0.00570.01340.03220.00920.00920.0390
(0.95, 0.05)0.00500.00910.03390.01010.01010.0406
(1.0, 0.0)0.00070.00640.03790.01060.01060.0447
+ + + +> train: [0.5 0.5] +> validation: [0.5 0.5] +> evaluate_bin_sld: 197.283s +> evaluate_mul_sld: 54.736s +> kfcv: 40.375s +> atc_mc: 41.898s +> atc_ne: 41.366s +> doc_feat: 35.145s +> tot: 198.630s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00040.00350.02570.02890.02890.0344
(0.05, 0.95)0.00750.00850.02240.02530.02530.0310
(0.1, 0.9)0.00810.01220.02050.02390.02390.0292
(0.15, 0.85)0.01020.01480.01800.02050.02050.0267
(0.2, 0.8)0.01390.01980.01650.02110.02110.0248
(0.25, 0.75)0.01940.02450.01410.01700.01700.0224
(0.3, 0.7)0.02300.02870.01370.01640.01640.0222
(0.35, 0.65)0.03090.03380.01320.01680.01680.0210
(0.4, 0.6)0.03500.03710.00970.01440.01440.0164
(0.45, 0.55)0.03580.03900.00860.01250.01250.0150
(0.5, 0.5)0.03690.03860.00730.01220.01220.0138
(0.55, 0.45)0.03730.03980.00710.01100.01100.0128
(0.6, 0.4)0.03680.03980.00640.00850.00850.0103
(0.65, 0.35)0.03570.03850.00740.01030.01030.0105
(0.7, 0.3)0.03190.03700.00670.00820.00820.0086
(0.75, 0.25)0.02980.03580.00790.00660.00660.0070
(0.8, 0.2)0.02350.03020.00730.00830.00830.0069
(0.85, 0.15)0.01540.02440.00970.00770.00770.0066
(0.9, 0.1)0.00830.01570.01080.00820.00820.0069
(0.95, 0.05)0.00550.00980.01310.00800.00800.0066
(1.0, 0.0)0.00070.00460.01450.00880.00880.0082
+ + + +> train: [0.59996662 0.40003338] +> validation: [0.59996662 0.40003338] +> evaluate_bin_sld: 194.960s +> evaluate_mul_sld: 53.330s +> kfcv: 40.320s +> atc_mc: 41.904s +> atc_ne: 41.423s +> doc_feat: 35.289s +> tot: 196.151s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00030.00550.08150.02850.02850.0825
(0.05, 0.95)0.00650.01270.07470.02780.02780.0758
(0.1, 0.9)0.00720.01720.06770.02240.02240.0688
(0.15, 0.85)0.01000.02570.06270.02180.02180.0638
(0.2, 0.8)0.01350.03080.05480.01800.01800.0560
(0.25, 0.75)0.01650.03380.04910.01600.01600.0503
(0.3, 0.7)0.02050.04090.04380.01680.01680.0450
(0.35, 0.65)0.02480.04590.03740.01560.01560.0386
(0.4, 0.6)0.02840.04910.02770.01120.01120.0290
(0.45, 0.55)0.03180.05150.02240.00990.00990.0237
(0.5, 0.5)0.03420.05160.01590.00810.00810.0170
(0.55, 0.45)0.03740.05190.01110.00730.00730.0121
(0.6, 0.4)0.04100.05370.00690.00790.00790.0075
(0.65, 0.35)0.04440.05170.00640.00760.00760.0064
(0.7, 0.3)0.04380.05020.01000.00850.00850.0090
(0.75, 0.25)0.04580.04830.01710.00890.00890.0157
(0.8, 0.2)0.04120.04190.02180.01050.01050.0204
(0.85, 0.15)0.03190.03480.02910.01170.01170.0276
(0.9, 0.1)0.01920.02540.03580.01470.01470.0343
(0.95, 0.05)0.00790.01540.04270.01660.01660.0412
(1.0, 0.0)0.00050.00340.04900.01900.01900.0474
+ + + +> train: [0.69993324 0.30006676] +> validation: [0.70010013 0.29989987] +> evaluate_bin_sld: 196.856s +> evaluate_mul_sld: 54.245s +> kfcv: 41.167s +> atc_mc: 42.203s +> atc_ne: 41.565s +> doc_feat: 34.998s +> tot: 198.332s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00030.00710.15700.06250.06250.1677
(0.05, 0.95)0.00890.01020.14280.05480.05480.1536
(0.1, 0.9)0.00780.01210.13270.05210.05210.1435
(0.15, 0.85)0.00730.01550.12270.05170.05170.1336
(0.2, 0.8)0.00810.01960.10940.04640.04640.1203
(0.25, 0.75)0.00950.02250.10010.04270.04270.1111
(0.3, 0.7)0.01170.02720.08850.04000.04000.0995
(0.35, 0.65)0.01310.03090.07740.03680.03680.0885
(0.4, 0.6)0.01440.03330.06260.03070.03070.0737
(0.45, 0.55)0.01790.03650.05280.02970.02970.0640
(0.5, 0.5)0.01830.03590.04180.02590.02590.0531
(0.55, 0.45)0.01890.03690.03130.02220.02220.0426
(0.6, 0.4)0.02200.03790.02010.01900.01900.0314
(0.65, 0.35)0.02180.03640.01040.01600.01600.0208
(0.7, 0.3)0.02290.03710.00670.01190.01190.0096
(0.75, 0.25)0.02500.03780.01610.01010.01010.0067
(0.8, 0.2)0.02370.03330.02590.00820.00820.0143
(0.85, 0.15)0.02270.02820.03810.00600.00600.0265
(0.9, 0.1)0.01800.02020.04990.00490.00490.0382
(0.95, 0.05)0.00970.01170.06070.00720.00720.0489
(1.0, 0.0)0.00140.00240.07240.01030.01030.0606
+ + + +> train: [0.79989987 0.20010013] +> validation: [0.80006676 0.19993324] +> evaluate_bin_sld: 197.725s +> evaluate_mul_sld: 53.526s +> kfcv: 40.971s +> atc_mc: 41.975s +> atc_ne: 41.358s +> doc_feat: 35.091s +> tot: 199.051s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.00090.00820.31480.05710.05710.3213
(0.05, 0.95)0.02970.02230.29250.04920.04920.2991
(0.1, 0.9)0.02830.02090.27330.04930.04930.2800
(0.15, 0.85)0.02470.01820.25280.04470.04470.2596
(0.2, 0.8)0.02160.01560.23280.04070.04070.2397
(0.25, 0.75)0.01700.01360.21360.04250.04250.2205
(0.3, 0.7)0.01460.01260.19410.03840.03840.2012
(0.35, 0.65)0.01250.01130.17340.03310.03310.1806
(0.4, 0.6)0.01130.01100.15100.02720.02720.1583
(0.45, 0.55)0.00930.01350.13280.02470.02470.1402
(0.5, 0.5)0.00880.01350.11310.02220.02220.1206
(0.55, 0.45)0.00920.01550.09190.02070.02070.0995
(0.6, 0.4)0.00920.01730.07420.01900.01900.0819
(0.65, 0.35)0.00870.01780.05440.01610.01610.0621
(0.7, 0.3)0.00930.01970.03230.01240.01240.0401
(0.75, 0.25)0.01010.02180.01140.00930.00930.0187
(0.8, 0.2)0.01170.02080.00980.00880.00880.0063
(0.85, 0.15)0.01030.01780.02850.00640.00640.0204
(0.9, 0.1)0.01030.01640.04800.00620.00620.0398
(0.95, 0.05)0.00920.01170.06840.00710.00710.0601
(1.0, 0.0)0.00110.00190.08870.00970.00970.0803
+ + + +> train: [0.90003338 0.09996662] +> validation: [0.90003338 0.09996662] +> evaluate_bin_sld: 201.315s +> evaluate_mul_sld: 50.974s +> kfcv: 40.175s +> atc_mc: 41.663s +> atc_ne: 41.058s +> doc_feat: 35.055s +> tot: 202.573s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
binmulkfcvatc_mcatc_nedoc_feat
(0.0, 1.0)0.03210.01840.64210.13360.13360.6454
(0.05, 0.95)0.08350.07290.60560.12440.12440.6090
(0.1, 0.9)0.10800.09760.57030.12040.12040.5739
(0.15, 0.85)0.11540.09710.53540.11470.11470.5390
(0.2, 0.8)0.10810.09160.50070.10640.10640.5045
(0.25, 0.75)0.10320.08300.46320.10050.10050.4671
(0.3, 0.7)0.09450.07750.42740.09160.09160.4313
(0.35, 0.65)0.09660.07090.39140.08430.08430.3954
(0.4, 0.6)0.07950.06390.35430.07480.07480.3584
(0.45, 0.55)0.07350.05330.32100.07280.07280.3253
(0.5, 0.5)0.07160.04730.28290.06330.06330.2873
(0.55, 0.45)0.05500.03930.24650.05680.05680.2509
(0.6, 0.4)0.05050.03170.21170.05090.05090.2162
(0.65, 0.35)0.04030.02260.17410.04380.04380.1788
(0.7, 0.3)0.03720.01780.13870.03480.03480.1434
(0.75, 0.25)0.02620.01220.10090.02560.02560.1057
(0.8, 0.2)0.02480.01100.06510.01940.01940.0701
(0.85, 0.15)0.01810.00750.02980.01280.01280.0348
(0.9, 0.1)0.01290.00930.00690.00800.00800.0037
(0.95, 0.05)0.00770.00850.04260.00460.00460.0373
(1.0, 0.0)0.00100.00100.07890.00880.00880.0735
+ + + diff --git a/quacc/evaluation/report.py b/quacc/evaluation/report.py index 0236b98..3949d6b 100644 --- a/quacc/evaluation/report.py +++ b/quacc/evaluation/report.py @@ -69,6 +69,12 @@ class EvaluationReport: columns=g_dict.keys(), ) + def get_plot(self, mode="delta", metric="acc"): + g_prevs, g_dict = self.groupby_prevs(metric=metric) + t_prev = int(round(self.train_prevs["train"][0] * 100)) + title = f"{self.name}_{t_prev}_{metric}" + plot.plot_delta(g_prevs, g_dict, metric, title) + def to_md(self, *metrics): res = "" for k, v in self.train_prevs.items(): @@ -78,6 +84,7 @@ class EvaluationReport: res += "\n" for m in metrics: res += self.get_dataframe(metric=m).to_html() + "\n\n" + self.get_plot(metric=m) return res diff --git a/quacc/plot.py b/quacc/plot.py new file mode 100644 index 0000000..79977d7 --- /dev/null +++ b/quacc/plot.py @@ -0,0 +1,26 @@ +import matplotlib.pyplot as plt + +from quacc.environ import env + + +def plot_delta(base_prevs, dict_vals, metric, title): + fig, ax = plt.subplots() + + base_prevs = [f for f, p in base_prevs] + for method, deltas in dict_vals.items(): + ax.plot( + base_prevs, + deltas, + label=method, + linestyle="-", + marker="o", + markersize=3, + zorder=2, + ) + + ax.set(xlabel="test prevalence", ylabel=metric, title=title) + # ax.set_ylim(0, 1) + # ax.set_xlim(0, 1) + ax.legend() + output_path = env.PLOT_OUT_DIR / f"{title}.png" + plt.savefig(output_path)