lorenzo.volpi
/
QuAcc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mul3w implemented

This commit is contained in:
Lorenzo Volpi 2023-11-11 18:46:27 +01:00
parent 64e546b8fd
commit 9f6333efb5
17 changed files with 1973 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,6 +5,7 @@ quavenv/*
__pycache__/* __pycache__/*
baselines/__pycache__/* baselines/__pycache__/*
baselines/densratio/__pycache__/* baselines/densratio/__pycache__/*
qcpanel/__pycache__/*
quacc/__pycache__/* quacc/__pycache__/*
quacc/evaluation/__pycache__/* quacc/evaluation/__pycache__/*
quacc/method/__pycache__/* quacc/method/__pycache__/*

2
.vscode/launch.json vendored
View File

@ -9,7 +9,7 @@
"name": "main", "name": "main",
"type": "python", "type": "python",
"request": "launch", "request": "launch",
"program": "C:\\Users\\Lorenzo Volpi\\source\\tesi\\quacc\\main.py", "program": "~/source/tesi/quacc/main.py",
"console": "integratedTerminal", "console": "integratedTerminal",
"justMyCode": true "justMyCode": true
}, },

4
TODO.md
View File

@ -42,7 +42,7 @@
- [x] testare anche su imbd - [x] testare anche su imbd
- [x] aggiungere esecuzione remota via ssh - [x] aggiungere esecuzione remota via ssh
- [ ] testare confidence con sia max_conf che exntropy - [x] testare confidence con sia max_conf che exntropy
- [x] implementare mul3
- [ ] rivedere nuove baselines - [ ] rivedere nuove baselines
- [ ] importare nuovi dataset - [ ] importare nuovi dataset
- [ ] implementare mul3

13
conf.yaml
View File

@ -5,16 +5,13 @@ debug_conf: &debug_conf
DATASET_N_PREVS: 9 DATASET_N_PREVS: 9
confs: confs:
- DATASET_NAME: rcv1 - DATASET_NAME: imdb
DATASET_TARGET: CCAT
plot_confs: plot_confs:
debug: debug_gs:
PLOT_ESTIMATORS: PLOT_ESTIMATORS:
- binc_sld - mul_sld
- binmc_sld - mul3w_sld
- binne_sld
- bin_sld
- atc_mc - atc_mc
PLOT_STDEV: true PLOT_STDEV: true
@ -224,4 +221,4 @@ main_conf: &main_conf
- atc_ne - atc_ne
- doc_feat - doc_feat
exec: *mc_conf exec: *debug_conf

1084
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

6
pyproject.toml
View File

@ -16,6 +16,8 @@ logging = "^0.4.9.6"
[tool.poetry.scripts] [tool.poetry.scripts]
main = "quacc.main:main" main = "quacc.main:main"
run = "run:run" run = "run:run"
panel = "qcpanel.run:run"
sync_up = "remote:sync_code"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
@ -25,6 +27,10 @@ pytest-mock = "^3.11.1"
pytest-cov = "^4.1.0" pytest-cov = "^4.1.0"
tabulate = "^0.9.0" tabulate = "^0.9.0"
paramiko = "^3.3.1" paramiko = "^3.3.1"
panel = "^1.3.1"
ipympl = "^0.9.3"
ipykernel = "^6.26.0"
ipywidgets-bokeh = "^1.5.0"
[tool.pytest.ini_options] [tool.pytest.ini_options]
addopts = "--cov=quacc --capture=tee-sys" addopts = "--cov=quacc --capture=tee-sys"

BIN
qcpanel/__pycache__/__init__.cpython-310.pyc
View File

Binary file not shown.

BIN
qcpanel/__pycache__/run.cpython-310.pyc
View File

Binary file not shown.

598
quacc.log
View File

@ -5035,3 +5035,601 @@
09/11/23 21:08:08| DEBUG [BinaryQuantifierAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00688) [took 1027.9305s] 09/11/23 21:08:08| DEBUG [BinaryQuantifierAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00688) [took 1027.9305s]
09/11/23 21:15:36| INFO bin_sld_gs finished [took 1476.4334s] 09/11/23 21:15:36| INFO bin_sld_gs finished [took 1476.4334s]
09/11/23 21:15:36| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 1479.7993s] 09/11/23 21:15:36| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 1479.7993s]
----------------------------------------------------------------------------------------------------
10/11/23 13:06:41| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:06:47| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:06:52| WARNING Method mul_sld failed. Exception: ExtendedCollection.from_lc() got an unexpected keyword argument 'pred_proba'
10/11/23 13:06:56| WARNING Method mul3w_sld failed. Exception: ExtendedCollection.from_lc() got an unexpected keyword argument 'pred_proba'
----------------------------------------------------------------------------------------------------
10/11/23 13:07:50| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:07:55| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:09:28| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:09:29| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:09:46| INFO ref finished [took 88.3729s]
10/11/23 13:09:54| INFO atc_mc finished [took 90.4167s]
10/11/23 13:09:54| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 118.7000s]
10/11/23 13:09:54| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 13:11:25| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:11:26| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
----------------------------------------------------------------------------------------------------
10/11/23 13:19:00| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:19:05| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:19:10| WARNING Method mul_sld failed. Exception: 'int' object is not iterable
10/11/23 13:19:13| WARNING Method mul3w_sld failed. Exception: 'int' object is not iterable
----------------------------------------------------------------------------------------------------
10/11/23 13:20:12| INFO dataset rcv1_CCAT_9prevs
10/11/23 13:20:17| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 13:21:36| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:21:39| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 13:21:54| INFO ref finished [took 77.8358s]
10/11/23 13:22:02| INFO atc_mc finished [took 81.6210s]
10/11/23 13:22:02| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 104.5107s]
10/11/23 13:22:02| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 13:23:31| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 13:23:31| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:23:47| INFO ref finished [took 83.8010s]
10/11/23 13:23:55| INFO atc_mc finished [took 82.0180s]
10/11/23 13:23:55| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 113.0759s]
10/11/23 13:23:55| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 13:25:35| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 13:25:35| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 13:25:52| INFO ref finished [took 91.9242s]
10/11/23 13:26:01| INFO atc_mc finished [took 91.0173s]
10/11/23 13:26:01| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 125.8152s]
10/11/23 13:26:01| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
----------------------------------------------------------------------------------------------------
10/11/23 14:35:06| INFO dataset rcv1_CCAT_1prevs
10/11/23 14:35:16| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 14:36:54| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 14:37:32| INFO ref finished [took 114.8732s]
----------------------------------------------------------------------------------------------------
10/11/23 14:45:14| INFO dataset rcv1_CCAT_1prevs
10/11/23 14:45:20| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 14:46:56| INFO ref finished [took 77.5985s]
10/11/23 14:47:09| INFO atc_mc finished [took 83.9760s]
10/11/23 14:56:11| WARNING Method mul_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0})
10/11/23 14:56:13| WARNING Method mul3w_sld failed. Exception: labels ({1, 2}) contain values not included in classes_ ({0, 1})
10/11/23 14:56:13| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 652.8237s]
----------------------------------------------------------------------------------------------------
10/11/23 14:56:25| INFO dataset rcv1_CCAT_1prevs
----------------------------------------------------------------------------------------------------
10/11/23 14:56:31| INFO dataset rcv1_CCAT_1prevs
10/11/23 14:56:37| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 14:58:44| INFO ref finished [took 106.4738s]
10/11/23 14:59:09| INFO atc_mc finished [took 117.9679s]
10/11/23 15:16:17| INFO mul3w_sld finished [took 1166.3690s]
----------------------------------------------------------------------------------------------------
10/11/23 15:49:18| INFO dataset rcv1_CCAT_1prevs
10/11/23 15:49:23| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 15:50:15| INFO ref finished [took 44.7880s]
----------------------------------------------------------------------------------------------------
10/11/23 16:24:50| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:24:57| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 16:26:54| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:27:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 16:28:15| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:28:22| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 16:29:17| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:29:25| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 16:30:30| INFO ref finished [took 52.9566s]
----------------------------------------------------------------------------------------------------
10/11/23 16:33:04| INFO dataset rcv1_CCAT_1prevs
10/11/23 16:33:11| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 16:34:10| INFO ref finished [took 50.4274s]
10/11/23 16:34:22| INFO mul_sld finished [took 66.6494s]
10/11/23 16:34:22| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 70.8885s]
----------------------------------------------------------------------------------------------------
10/11/23 16:36:29| INFO dataset rcv1_CCAT_9prevs
10/11/23 16:36:34| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 16:37:26| WARNING Method mul3w_sld failed. Exception: index 3 is out of bounds for axis 0 with size 3
10/11/23 16:37:40| INFO ref finished [took 50.7235s]
10/11/23 16:37:47| INFO atc_mc finished [took 54.6993s]
----------------------------------------------------------------------------------------------------
10/11/23 16:45:15| INFO dataset rcv1_CCAT_9prevs
10/11/23 16:45:20| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 16:46:11| WARNING Method mul_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
10/11/23 16:46:12| WARNING Method mul3w_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
10/11/23 16:46:25| INFO ref finished [took 50.7336s]
10/11/23 16:46:33| INFO atc_mc finished [took 54.1424s]
10/11/23 16:46:33| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 72.4632s]
10/11/23 16:46:33| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 16:47:28| WARNING Method mul_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
10/11/23 16:47:29| WARNING Method mul3w_sld failed. Exception: unsupported operand type(s) for /: 'NoneType' and 'int'
----------------------------------------------------------------------------------------------------
10/11/23 16:51:39| INFO dataset rcv1_CCAT_9prevs
10/11/23 16:51:44| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 16:52:50| INFO ref finished [took 51.5277s]
10/11/23 16:52:55| INFO mul3w_sld finished [took 62.7316s]
10/11/23 16:52:58| INFO atc_mc finished [took 55.6057s]
10/11/23 16:53:25| INFO mul_sld finished [took 97.7123s]
10/11/23 16:53:25| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 100.9808s]
10/11/23 16:53:25| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 16:54:36| INFO ref finished [took 55.7445s]
10/11/23 16:54:43| INFO atc_mc finished [took 56.7203s]
10/11/23 16:54:45| INFO mul3w_sld finished [took 71.0450s]
10/11/23 16:54:53| INFO mul_sld finished [took 84.6525s]
10/11/23 16:54:53| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 88.0027s]
10/11/23 16:54:53| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 16:56:06| INFO ref finished [took 54.1395s]
10/11/23 16:56:11| INFO mul3w_sld finished [took 70.1307s]
10/11/23 16:56:13| INFO atc_mc finished [took 57.6850s]
10/11/23 16:56:16| INFO mul_sld finished [took 79.3229s]
10/11/23 16:56:16| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 83.1293s]
10/11/23 16:56:16| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 16:57:25| INFO ref finished [took 53.7706s]
10/11/23 16:57:31| INFO mul3w_sld finished [took 67.0318s]
10/11/23 16:57:32| INFO atc_mc finished [took 57.7642s]
10/11/23 16:57:35| INFO mul_sld finished [took 76.4066s]
10/11/23 16:57:35| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 79.0392s]
10/11/23 16:57:35| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 16:58:46| INFO ref finished [took 53.7141s]
10/11/23 16:58:51| INFO mul3w_sld finished [took 68.1234s]
10/11/23 16:58:53| INFO atc_mc finished [took 57.5702s]
10/11/23 16:58:59| INFO mul_sld finished [took 80.1266s]
10/11/23 16:58:59| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 83.5102s]
10/11/23 16:58:59| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 17:00:08| INFO ref finished [took 54.7955s]
10/11/23 17:00:12| INFO mul3w_sld finished [took 65.3780s]
10/11/23 17:00:16| INFO atc_mc finished [took 58.0304s]
10/11/23 17:00:35| INFO mul_sld finished [took 92.9140s]
10/11/23 17:00:35| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 95.8577s]
10/11/23 17:00:35| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 17:01:48| INFO ref finished [took 56.4922s]
10/11/23 17:01:53| INFO atc_mc finished [took 57.1146s]
10/11/23 17:01:53| INFO mul3w_sld finished [took 70.4300s]
10/11/23 17:02:03| INFO mul_sld finished [took 84.4971s]
10/11/23 17:02:03| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 88.5686s]
10/11/23 17:02:03| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 17:03:17| INFO ref finished [took 55.9443s]
10/11/23 17:03:19| INFO mul3w_sld finished [took 68.1298s]
10/11/23 17:03:21| INFO atc_mc finished [took 55.8708s]
10/11/23 17:03:29| INFO mul_sld finished [took 81.8552s]
10/11/23 17:03:29| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 85.5029s]
10/11/23 17:03:29| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 17:04:36| INFO ref finished [took 52.4327s]
10/11/23 17:04:42| INFO mul3w_sld finished [took 65.1205s]
10/11/23 17:04:44| INFO atc_mc finished [took 56.4258s]
10/11/23 17:05:07| INFO mul_sld finished [took 95.0175s]
10/11/23 17:05:07| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 98.2742s]
----------------------------------------------------------------------------------------------------
10/11/23 17:11:56| INFO dataset rcv1_CCAT_9prevs
10/11/23 17:12:01| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 17:13:39| INFO ref finished [took 70.2028s]
10/11/23 17:13:41| INFO mul3w_sld finished [took 92.9234s]
10/11/23 17:13:43| INFO mul3wc_sld finished [took 81.4985s]
10/11/23 17:13:47| INFO atc_mc finished [took 69.8737s]
10/11/23 17:13:49| INFO mulc_sld finished [took 91.9609s]
10/11/23 17:14:09| INFO mul_sld finished [took 125.3021s]
10/11/23 17:14:09| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 128.4443s]
10/11/23 17:14:09| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 17:15:49| INFO ref finished [took 69.9186s]
10/11/23 17:15:52| INFO mul3w_sld finished [took 94.1565s]
10/11/23 17:15:53| INFO mul3wc_sld finished [took 81.7286s]
10/11/23 17:15:55| INFO atc_mc finished [took 68.0996s]
10/11/23 17:15:59| INFO mulc_sld finished [took 93.4749s]
10/11/23 17:16:02| INFO mul_sld finished [took 109.9866s]
10/11/23 17:16:02| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 113.2243s]
10/11/23 17:16:02| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 17:17:46| INFO ref finished [took 69.7954s]
10/11/23 17:17:49| INFO mul3w_sld finished [took 97.4538s]
10/11/23 17:17:52| INFO atc_mc finished [took 68.3256s]
10/11/23 17:17:53| INFO mul3wc_sld finished [took 84.1359s]
10/11/23 17:17:54| INFO mul_sld finished [took 107.4652s]
10/11/23 17:17:55| INFO mulc_sld finished [took 95.2790s]
10/11/23 17:17:55| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 112.5479s]
10/11/23 17:17:55| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 17:19:33| INFO ref finished [took 69.0242s]
10/11/23 17:19:37| INFO mul3w_sld finished [took 94.6726s]
10/11/23 17:19:39| INFO mul3wc_sld finished [took 83.5757s]
10/11/23 17:19:40| INFO atc_mc finished [took 70.0895s]
10/11/23 17:19:42| INFO mul_sld finished [took 104.3169s]
10/11/23 17:19:45| INFO mulc_sld finished [took 96.4947s]
10/11/23 17:19:45| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 109.8588s]
10/11/23 17:19:45| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 17:21:25| INFO ref finished [took 68.6827s]
10/11/23 17:21:28| INFO mul3w_sld finished [took 95.7660s]
10/11/23 17:21:33| INFO mul3wc_sld finished [took 84.6669s]
10/11/23 17:21:34| INFO atc_mc finished [took 68.2059s]
10/11/23 17:21:38| INFO mul_sld finished [took 109.7998s]
10/11/23 17:21:41| INFO mulc_sld finished [took 100.2322s]
10/11/23 17:21:41| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 116.3329s]
10/11/23 17:21:41| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 17:23:20| INFO ref finished [took 68.4213s]
10/11/23 17:23:21| INFO mul3w_sld finished [took 92.4175s]
10/11/23 17:23:25| INFO mul3wc_sld finished [took 84.2143s]
10/11/23 17:23:28| INFO atc_mc finished [took 70.9748s]
10/11/23 17:23:33| INFO mul_sld finished [took 108.9404s]
10/11/23 17:23:35| INFO mulc_sld finished [took 98.6542s]
10/11/23 17:23:35| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 114.0658s]
10/11/23 17:23:35| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 17:25:15| INFO ref finished [took 67.6212s]
10/11/23 17:25:19| INFO mul3w_sld finished [took 94.4720s]
10/11/23 17:25:23| INFO mul3wc_sld finished [took 84.2353s]
10/11/23 17:25:24| INFO atc_mc finished [took 68.8926s]
10/11/23 17:25:29| INFO mul_sld finished [took 110.0608s]
10/11/23 17:25:31| INFO mulc_sld finished [took 98.8512s]
10/11/23 17:25:31| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 116.1487s]
10/11/23 17:25:31| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 17:27:13| INFO ref finished [took 67.6916s]
10/11/23 17:27:14| INFO mul3wc_sld finished [took 77.9027s]
10/11/23 17:27:16| INFO mul3w_sld finished [took 95.6551s]
10/11/23 17:27:21| INFO atc_mc finished [took 69.1522s]
10/11/23 17:27:25| INFO mul_sld finished [took 110.5946s]
10/11/23 17:27:26| INFO mulc_sld finished [took 97.1366s]
10/11/23 17:27:26| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 115.0205s]
10/11/23 17:27:26| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 17:28:57| INFO ref finished [took 60.3214s]
10/11/23 17:29:05| INFO mul3w_sld finished [took 89.8969s]
10/11/23 17:29:08| INFO mul3wc_sld finished [took 79.4281s]
10/11/23 17:29:09| INFO mulc_sld finished [took 89.2493s]
10/11/23 17:29:10| INFO atc_mc finished [took 69.0056s]
10/11/23 17:29:29| INFO mul_sld finished [took 119.2894s]
10/11/23 17:29:29| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 122.1553s]
----------------------------------------------------------------------------------------------------
10/11/23 17:53:55| INFO dataset rcv1_CCAT_9prevs
10/11/23 17:54:01| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 17:54:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:54:06| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:54:53| INFO ref finished [took 44.4485s]
10/11/23 17:55:01| INFO atc_mc finished [took 49.8926s]
10/11/23 17:55:01| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 60.0268s]
10/11/23 17:55:01| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 17:55:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:55:07| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:55:54| INFO ref finished [took 44.7491s]
10/11/23 17:56:02| INFO atc_mc finished [took 49.9022s]
10/11/23 17:56:02| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 60.8386s]
10/11/23 17:56:02| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 17:56:05| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:56:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:56:56| INFO ref finished [took 45.2058s]
10/11/23 17:57:03| INFO atc_mc finished [took 49.0981s]
10/11/23 17:57:03| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 61.1000s]
10/11/23 17:57:03| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 17:57:05| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:57:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:57:54| INFO ref finished [took 44.3211s]
10/11/23 17:58:01| INFO atc_mc finished [took 48.9814s]
10/11/23 17:58:01| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 58.0909s]
10/11/23 17:58:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 17:58:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:58:06| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:58:53| INFO ref finished [took 43.7895s]
10/11/23 17:59:01| INFO atc_mc finished [took 49.8161s]
10/11/23 17:59:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 60.3531s]
10/11/23 17:59:01| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 17:59:04| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:59:07| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 17:59:53| INFO ref finished [took 43.8473s]
10/11/23 18:00:02| INFO atc_mc finished [took 50.1003s]
10/11/23 18:00:02| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 60.5473s]
10/11/23 18:00:02| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 18:00:05| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:00:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:00:55| INFO ref finished [took 44.2168s]
10/11/23 18:01:02| INFO atc_mc finished [took 49.0013s]
10/11/23 18:01:02| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 60.5261s]
10/11/23 18:01:02| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 18:01:06| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:01:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:01:55| INFO ref finished [took 44.0374s]
10/11/23 18:02:02| INFO atc_mc finished [took 49.0107s]
10/11/23 18:02:02| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 60.0171s]
10/11/23 18:02:02| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 18:02:06| WARNING Method mul_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:02:08| WARNING Method mul3w_sld_gs failed. Exception: no combination of hyperparameters seem to work
10/11/23 18:02:55| INFO ref finished [took 44.8341s]
10/11/23 18:03:02| INFO atc_mc finished [took 49.3663s]
10/11/23 18:03:02| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 59.9146s]
----------------------------------------------------------------------------------------------------
10/11/23 18:25:20| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:25:26| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 18:25:33| WARNING Method mul_sld_gs failed. Exception: '>=' not supported between instances of 'AttributeError' and 'int'
----------------------------------------------------------------------------------------------------
10/11/23 18:27:03| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:27:08| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
----------------------------------------------------------------------------------------------------
10/11/23 18:32:33| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:32:38| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 18:32:56| WARNING Method mul_sld_gs failed. Exception: '>=' not supported between instances of 'AssertionError' and 'int'
----------------------------------------------------------------------------------------------------
10/11/23 18:34:35| INFO dataset rcv1_CCAT_1prevs
10/11/23 18:34:40| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 18:36:25| INFO ref finished [took 96.5839s]
10/11/23 18:44:44| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00784) [took 599.7376s]
10/11/23 18:45:34| INFO mul_sld_gs finished [took 649.9561s]
10/11/23 18:45:34| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 654.0138s]
----------------------------------------------------------------------------------------------------
10/11/23 19:09:37| INFO dataset rcv1_CCAT_9prevs
10/11/23 19:09:42| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
10/11/23 19:12:34| INFO ref finished [took 161.2859s]
10/11/23 19:13:10| INFO atc_mc finished [took 195.4184s]
10/11/23 19:23:09| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00980) [took 799.2257s]
10/11/23 19:25:06| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01058) [took 920.3112s]
10/11/23 19:25:20| INFO mul3w_sld_gs finished [took 930.7628s]
10/11/23 19:25:54| INFO mul_sld_gs finished [took 968.7880s]
10/11/23 19:25:54| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 972.0481s]
10/11/23 19:25:54| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
10/11/23 19:29:20| INFO ref finished [took 194.4623s]
10/11/23 19:29:41| INFO atc_mc finished [took 212.3133s]
10/11/23 19:39:18| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01783) [took 796.9196s]
10/11/23 19:41:39| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01419) [took 940.9938s]
10/11/23 19:41:44| INFO mul3w_sld_gs finished [took 942.8448s]
10/11/23 19:42:29| INFO mul_sld_gs finished [took 991.5975s]
10/11/23 19:42:29| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 995.0756s]
10/11/23 19:42:29| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
10/11/23 19:45:53| INFO ref finished [took 192.2438s]
10/11/23 19:46:27| INFO atc_mc finished [took 222.4684s]
10/11/23 19:56:09| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1000.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00855) [took 811.9404s]
10/11/23 19:58:17| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00987) [took 944.2852s]
10/11/23 19:58:26| INFO mul3w_sld_gs finished [took 949.1296s]
10/11/23 19:59:09| INFO mul_sld_gs finished [took 995.5734s]
10/11/23 19:59:09| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 999.2987s]
10/11/23 19:59:09| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
10/11/23 20:02:30| INFO ref finished [took 192.8661s]
10/11/23 20:02:59| INFO atc_mc finished [took 217.1095s]
10/11/23 20:12:26| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.01004) [took 792.0336s]
10/11/23 20:14:45| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.01023) [took 933.3629s]
10/11/23 20:14:49| INFO mul3w_sld_gs finished [took 935.2472s]
10/11/23 20:15:33| INFO mul_sld_gs finished [took 981.2718s]
10/11/23 20:15:33| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 983.8857s]
10/11/23 20:15:33| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
10/11/23 20:18:54| INFO ref finished [took 190.6586s]
10/11/23 20:19:31| INFO atc_mc finished [took 222.0446s]
10/11/23 20:28:53| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00628) [took 794.2257s]
10/11/23 20:31:06| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00601) [took 930.8288s]
10/11/23 20:31:12| INFO mul3w_sld_gs finished [took 932.7508s]
10/11/23 20:31:58| INFO mul_sld_gs finished [took 982.4600s]
10/11/23 20:31:58| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 985.3991s]
10/11/23 20:31:58| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
10/11/23 20:35:16| INFO ref finished [took 187.4145s]
10/11/23 20:35:52| INFO atc_mc finished [took 219.1775s]
10/11/23 20:45:24| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00683) [took 799.7247s]
10/11/23 20:47:38| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00635) [took 936.5460s]
10/11/23 20:47:46| INFO mul3w_sld_gs finished [took 941.6249s]
10/11/23 20:48:28| INFO mul_sld_gs finished [took 987.3566s]
10/11/23 20:48:28| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 990.3392s]
10/11/23 20:48:28| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
10/11/23 20:52:00| INFO ref finished [took 199.2882s]
10/11/23 20:52:27| INFO atc_mc finished [took 221.3995s]
10/11/23 21:01:50| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01112) [took 794.5357s]
10/11/23 21:03:48| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01010) [took 916.6784s]
10/11/23 21:04:04| INFO mul3w_sld_gs finished [took 928.2783s]
10/11/23 21:04:45| INFO mul_sld_gs finished [took 972.9080s]
10/11/23 21:04:45| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 976.3012s]
10/11/23 21:04:45| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
10/11/23 21:07:55| INFO ref finished [took 179.6820s]
10/11/23 21:08:25| INFO atc_mc finished [took 205.4880s]
10/11/23 21:17:51| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01448) [took 778.9722s]
10/11/23 21:20:04| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01845) [took 915.8827s]
10/11/23 21:20:10| INFO mul3w_sld_gs finished [took 917.5008s]
10/11/23 21:20:53| INFO mul_sld_gs finished [took 965.1015s]
10/11/23 21:20:53| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 968.7572s]
10/11/23 21:20:53| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
10/11/23 21:23:59| INFO ref finished [took 175.7336s]
10/11/23 21:24:36| INFO atc_mc finished [took 208.4952s]
10/11/23 21:34:01| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00971) [took 780.9362s]
10/11/23 21:35:59| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00915) [took 902.5701s]
10/11/23 21:36:12| INFO mul3w_sld_gs finished [took 911.8556s]
10/11/23 21:36:47| INFO mul_sld_gs finished [took 950.1102s]
10/11/23 21:36:47| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 953.3643s]
----------------------------------------------------------------------------------------------------
10/11/23 23:35:14| INFO dataset rcv1_CCAT_1prevs
10/11/23 23:35:19| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 23:36:36| INFO ref finished [took 57.4602s]
10/11/23 23:36:41| INFO mul3w_sld finished [took 72.3696s]
10/11/23 23:36:43| INFO atc_mc finished [took 57.1442s]
10/11/23 23:36:48| INFO mul_sld finished [took 84.9653s]
10/11/23 23:36:48| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 88.9861s]
10/11/23 23:36:48| ERROR Evaluation over rcv1_CCAT_1prevs failed. Exception: file must have a 'write' attribute
----------------------------------------------------------------------------------------------------
10/11/23 23:40:17| INFO dataset rcv1_CCAT_1prevs
10/11/23 23:40:23| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 23:41:49| INFO ref finished [took 68.3161s]
10/11/23 23:41:53| INFO mul3w_sld finished [took 80.2113s]
10/11/23 23:41:56| INFO atc_mc finished [took 67.4302s]
10/11/23 23:42:01| INFO mul_sld finished [took 94.1596s]
10/11/23 23:42:01| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 98.1999s]
10/11/23 23:42:01| ERROR Evaluation over rcv1_CCAT_1prevs failed. Exception: write() argument must be str, not bytes
----------------------------------------------------------------------------------------------------
10/11/23 23:46:43| INFO dataset rcv1_CCAT_1prevs
10/11/23 23:46:48| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs started
10/11/23 23:48:04| INFO ref finished [took 56.6494s]
10/11/23 23:48:10| INFO mul3w_sld finished [took 71.5272s]
10/11/23 23:48:12| INFO atc_mc finished [took 56.8870s]
10/11/23 23:48:15| INFO mul_sld finished [took 83.1487s]
10/11/23 23:48:15| INFO Dataset sample 0.50 of dataset rcv1_CCAT_1prevs finished [took 86.6542s]
----------------------------------------------------------------------------------------------------
11/11/23 02:18:21| ERROR Evaluation over rcv1_CCAT_3prevs failed. Exception: 'Invalid estimator: estimator mul3w_slds does not exist'
11/11/23 02:18:21| ERROR Failed while saving configuration rcv1_CCAT_debug_gs of rcv1_CCAT_3prevs. Exception: local variable 'dr' referenced before assignment
----------------------------------------------------------------------------------------------------
11/11/23 02:20:09| INFO dataset rcv1_CCAT_3prevs
11/11/23 02:20:15| INFO Dataset sample 0.20 of dataset rcv1_CCAT_3prevs started
11/11/23 02:21:26| INFO ref finished [took 54.1901s]
11/11/23 02:21:33| INFO mul3w_sld finished [took 69.3537s]
11/11/23 02:21:34| INFO atc_mc finished [took 56.0851s]
11/11/23 02:21:37| INFO mul_sld finished [took 78.6476s]
11/11/23 02:21:37| INFO Dataset sample 0.20 of dataset rcv1_CCAT_3prevs finished [took 82.3064s]
11/11/23 02:21:37| INFO Dataset sample 0.50 of dataset rcv1_CCAT_3prevs started
11/11/23 02:22:46| INFO ref finished [took 53.7198s]
11/11/23 02:22:52| INFO mul3w_sld finished [took 67.7595s]
11/11/23 02:22:54| INFO atc_mc finished [took 57.2473s]
11/11/23 02:23:00| INFO mul_sld finished [took 80.1637s]
11/11/23 02:23:00| INFO Dataset sample 0.50 of dataset rcv1_CCAT_3prevs finished [took 83.2895s]
11/11/23 02:23:00| INFO Dataset sample 0.80 of dataset rcv1_CCAT_3prevs started
11/11/23 02:24:11| INFO ref finished [took 53.3185s]
11/11/23 02:24:16| INFO mul3w_sld finished [took 67.0687s]
11/11/23 02:24:19| INFO atc_mc finished [took 56.5244s]
11/11/23 02:24:26| INFO mul_sld finished [took 82.6075s]
11/11/23 02:24:26| INFO Dataset sample 0.80 of dataset rcv1_CCAT_3prevs finished [took 85.5541s]
----------------------------------------------------------------------------------------------------
11/11/23 03:03:37| INFO dataset rcv1_CCAT_9prevs
11/11/23 03:03:42| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
11/11/23 03:04:49| INFO ref finished [took 51.5735s]
11/11/23 03:04:53| INFO mul3w_sld finished [took 62.2816s]
11/11/23 03:04:57| INFO atc_mc finished [took 56.1819s]
11/11/23 03:05:22| INFO mul_sld finished [took 96.8027s]
11/11/23 03:05:22| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 100.0654s]
11/11/23 03:05:22| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
11/11/23 03:06:32| INFO ref finished [took 53.7892s]
11/11/23 03:06:37| INFO mul3w_sld finished [took 66.3243s]
11/11/23 03:06:40| INFO atc_mc finished [took 57.0767s]
11/11/23 03:06:48| INFO mul_sld finished [took 82.1666s]
11/11/23 03:06:48| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 85.6326s]
11/11/23 03:06:48| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
11/11/23 03:07:59| INFO ref finished [took 53.4098s]
11/11/23 03:08:05| INFO mul3w_sld finished [took 67.3941s]
11/11/23 03:08:06| INFO atc_mc finished [took 56.8497s]
11/11/23 03:08:09| INFO mul_sld finished [took 77.4972s]
11/11/23 03:08:09| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 81.1647s]
11/11/23 03:08:09| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
11/11/23 03:09:18| INFO ref finished [took 54.4416s]
11/11/23 03:09:23| INFO mul3w_sld finished [took 67.3565s]
11/11/23 03:09:25| INFO atc_mc finished [took 57.5687s]
11/11/23 03:09:29| INFO mul_sld finished [took 77.1110s]
11/11/23 03:09:29| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 79.8726s]
11/11/23 03:09:29| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
11/11/23 03:10:38| INFO ref finished [took 54.2066s]
11/11/23 03:10:44| INFO mul3w_sld finished [took 66.9062s]
11/11/23 03:10:46| INFO atc_mc finished [took 57.1371s]
11/11/23 03:10:52| INFO mul_sld finished [took 79.7498s]
11/11/23 03:10:52| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 83.1506s]
11/11/23 03:10:52| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
11/11/23 03:11:59| INFO ref finished [took 51.1698s]
11/11/23 03:12:03| INFO mul3w_sld finished [took 63.7260s]
11/11/23 03:12:07| INFO atc_mc finished [took 56.1393s]
11/11/23 03:12:15| INFO mul_sld finished [took 79.3176s]
11/11/23 03:12:15| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 82.4614s]
11/11/23 03:12:15| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
11/11/23 03:13:23| INFO ref finished [took 51.5505s]
11/11/23 03:13:28| INFO mul3w_sld finished [took 64.8809s]
11/11/23 03:13:32| INFO atc_mc finished [took 56.2766s]
11/11/23 03:13:39| INFO mul_sld finished [took 80.9284s]
11/11/23 03:13:39| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 84.2727s]
11/11/23 03:13:39| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
11/11/23 03:14:50| INFO ref finished [took 54.6314s]
11/11/23 03:14:55| INFO mul3w_sld finished [took 67.8281s]
11/11/23 03:14:58| INFO atc_mc finished [took 57.4433s]
11/11/23 03:15:04| INFO mul_sld finished [took 82.1945s]
11/11/23 03:15:04| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 85.5484s]
11/11/23 03:15:04| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
11/11/23 03:16:12| INFO ref finished [took 54.0583s]
11/11/23 03:16:19| INFO mul3w_sld finished [took 65.9654s]
11/11/23 03:16:20| INFO atc_mc finished [took 56.7869s]
11/11/23 03:16:42| INFO mul_sld finished [took 94.5979s]
11/11/23 03:16:42| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 97.8705s]
----------------------------------------------------------------------------------------------------
11/11/23 14:55:30| INFO dataset rcv1_CCAT_9prevs
11/11/23 14:55:35| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs started
11/11/23 14:56:42| INFO ref finished [took 51.5446s]
11/11/23 14:56:45| INFO mul3w_sld finished [took 61.1724s]
11/11/23 14:56:49| INFO atc_mc finished [took 55.8947s]
11/11/23 14:57:16| INFO mul_sld finished [took 97.5932s]
11/11/23 14:57:16| INFO Dataset sample 0.10 of dataset rcv1_CCAT_9prevs finished [took 100.9322s]
11/11/23 14:57:16| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs started
11/11/23 14:58:25| INFO ref finished [took 52.3894s]
11/11/23 14:58:32| INFO mul3w_sld finished [took 66.8082s]
11/11/23 14:58:34| INFO atc_mc finished [took 57.1608s]
11/11/23 14:58:43| INFO mul_sld finished [took 82.8613s]
11/11/23 14:58:43| INFO Dataset sample 0.20 of dataset rcv1_CCAT_9prevs finished [took 86.2458s]
11/11/23 14:58:43| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs started
11/11/23 14:59:55| INFO ref finished [took 53.4860s]
11/11/23 15:00:00| INFO mul3w_sld finished [took 68.6692s]
11/11/23 15:00:02| INFO atc_mc finished [took 56.8460s]
11/11/23 15:00:04| INFO mul_sld finished [took 77.8829s]
11/11/23 15:00:04| INFO Dataset sample 0.30 of dataset rcv1_CCAT_9prevs finished [took 81.6584s]
11/11/23 15:00:04| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs started
11/11/23 15:01:12| INFO ref finished [took 52.5890s]
11/11/23 15:01:18| INFO mul3w_sld finished [took 66.3191s]
11/11/23 15:01:18| INFO atc_mc finished [took 55.8094s]
11/11/23 15:01:23| INFO mul_sld finished [took 76.2650s]
11/11/23 15:01:23| INFO Dataset sample 0.40 of dataset rcv1_CCAT_9prevs finished [took 78.8604s]
11/11/23 15:01:23| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs started
11/11/23 15:02:30| INFO ref finished [took 52.1273s]
11/11/23 15:02:39| INFO mul3w_sld finished [took 67.7736s]
11/11/23 15:02:41| INFO atc_mc finished [took 58.1392s]
11/11/23 15:02:47| INFO mul_sld finished [took 80.6840s]
11/11/23 15:02:47| INFO Dataset sample 0.50 of dataset rcv1_CCAT_9prevs finished [took 84.0856s]
11/11/23 15:02:47| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs started
11/11/23 15:03:54| INFO ref finished [took 51.3613s]
11/11/23 15:03:59| INFO mul3w_sld finished [took 64.3562s]
11/11/23 15:04:03| INFO atc_mc finished [took 57.0352s]
11/11/23 15:04:10| INFO mul_sld finished [took 79.8549s]
11/11/23 15:04:10| INFO Dataset sample 0.60 of dataset rcv1_CCAT_9prevs finished [took 82.9290s]
11/11/23 15:04:10| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs started
11/11/23 15:05:20| INFO ref finished [took 52.6105s]
11/11/23 15:05:25| INFO mul3w_sld finished [took 66.7591s]
11/11/23 15:05:27| INFO atc_mc finished [took 55.8870s]
11/11/23 15:05:34| INFO mul_sld finished [took 80.9824s]
11/11/23 15:05:34| INFO Dataset sample 0.70 of dataset rcv1_CCAT_9prevs finished [took 84.2673s]
11/11/23 15:05:34| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs started
11/11/23 15:06:45| INFO ref finished [took 53.6830s]
11/11/23 15:06:50| INFO mul3w_sld finished [took 67.4974s]
11/11/23 15:06:53| INFO atc_mc finished [took 57.2897s]
11/11/23 15:06:59| INFO mul_sld finished [took 81.3697s]
11/11/23 15:06:59| INFO Dataset sample 0.80 of dataset rcv1_CCAT_9prevs finished [took 84.7980s]
11/11/23 15:06:59| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs started
11/11/23 15:08:08| INFO ref finished [took 54.0902s]
11/11/23 15:08:14| INFO mul3w_sld finished [took 66.9136s]
11/11/23 15:08:16| INFO atc_mc finished [took 57.8269s]
11/11/23 15:08:39| INFO mul_sld finished [took 96.4137s]
11/11/23 15:08:39| INFO Dataset sample 0.90 of dataset rcv1_CCAT_9prevs finished [took 99.5541s]
----------------------------------------------------------------------------------------------------
11/11/23 18:04:49| INFO dataset imdb_9prevs
11/11/23 18:05:00| INFO Dataset sample 0.10 of dataset imdb_9prevs started
11/11/23 18:05:30| INFO ref finished [took 26.1862s]
11/11/23 18:05:37| INFO atc_mc finished [took 32.4255s]
11/11/23 18:05:42| INFO mul3w_sld finished [took 38.9162s]
11/11/23 18:06:19| INFO mul_sld finished [took 76.8053s]
11/11/23 18:06:19| INFO Dataset sample 0.10 of dataset imdb_9prevs finished [took 79.1732s]
11/11/23 18:06:19| INFO Dataset sample 0.20 of dataset imdb_9prevs started
11/11/23 18:06:51| INFO ref finished [took 27.6356s]
11/11/23 18:06:58| INFO atc_mc finished [took 33.5521s]
11/11/23 18:07:01| INFO mul3w_sld finished [took 38.2698s]
11/11/23 18:07:26| INFO mul_sld finished [took 63.9553s]
11/11/23 18:07:26| INFO Dataset sample 0.20 of dataset imdb_9prevs finished [took 66.8514s]
11/11/23 18:07:26| INFO Dataset sample 0.30 of dataset imdb_9prevs started
11/11/23 18:08:02| INFO ref finished [took 27.3880s]
11/11/23 18:08:09| INFO atc_mc finished [took 34.5060s]
11/11/23 18:08:12| INFO mul3w_sld finished [took 38.3408s]
11/11/23 18:08:23| INFO mul_sld finished [took 50.5943s]
11/11/23 18:08:23| INFO Dataset sample 0.30 of dataset imdb_9prevs finished [took 57.8801s]
11/11/23 18:08:23| INFO Dataset sample 0.40 of dataset imdb_9prevs started
11/11/23 18:08:57| INFO ref finished [took 27.9013s]
11/11/23 18:09:03| INFO atc_mc finished [took 33.7677s]
11/11/23 18:09:06| INFO mul3w_sld finished [took 37.6649s]
11/11/23 18:09:18| INFO mul_sld finished [took 51.5975s]
11/11/23 18:09:18| INFO Dataset sample 0.40 of dataset imdb_9prevs finished [took 54.8989s]
11/11/23 18:09:18| INFO Dataset sample 0.50 of dataset imdb_9prevs started
11/11/23 18:09:53| INFO ref finished [took 28.6798s]
11/11/23 18:10:00| INFO atc_mc finished [took 35.3206s]
11/11/23 18:10:03| INFO mul3w_sld finished [took 38.9082s]
11/11/23 18:10:15| INFO mul_sld finished [took 52.0709s]
11/11/23 18:10:15| INFO Dataset sample 0.50 of dataset imdb_9prevs finished [took 56.2785s]
11/11/23 18:10:15| INFO Dataset sample 0.60 of dataset imdb_9prevs started
11/11/23 18:10:48| INFO ref finished [took 28.5387s]
11/11/23 18:10:55| INFO atc_mc finished [took 34.9152s]
11/11/23 18:10:58| INFO mul3w_sld finished [took 39.0343s]
11/11/23 18:11:07| INFO mul_sld finished [took 49.9013s]
11/11/23 18:11:07| INFO Dataset sample 0.60 of dataset imdb_9prevs finished [took 52.8028s]
11/11/23 18:11:07| INFO Dataset sample 0.70 of dataset imdb_9prevs started
11/11/23 18:11:43| INFO ref finished [took 29.2208s]
11/11/23 18:11:50| INFO atc_mc finished [took 36.2494s]
11/11/23 18:11:54| INFO mul3w_sld finished [took 39.4380s]
11/11/23 18:12:02| INFO mul_sld finished [took 50.3782s]
11/11/23 18:12:02| INFO Dataset sample 0.70 of dataset imdb_9prevs finished [took 54.9711s]
11/11/23 18:12:02| INFO Dataset sample 0.80 of dataset imdb_9prevs started
11/11/23 18:12:37| INFO ref finished [took 28.2535s]
11/11/23 18:12:45| INFO atc_mc finished [took 34.9431s]
11/11/23 18:12:47| INFO mul3w_sld finished [took 38.4049s]
11/11/23 18:12:56| INFO mul_sld finished [took 48.3663s]
11/11/23 18:12:56| INFO Dataset sample 0.80 of dataset imdb_9prevs finished [took 53.7749s]
11/11/23 18:12:56| INFO Dataset sample 0.90 of dataset imdb_9prevs started
11/11/23 18:13:28| INFO ref finished [took 27.5002s]
11/11/23 18:13:35| INFO atc_mc finished [took 34.2726s]
11/11/23 18:13:41| INFO mul3w_sld finished [took 41.9968s]
11/11/23 18:13:54| INFO mul_sld finished [took 56.7537s]
11/11/23 18:13:54| INFO Dataset sample 0.90 of dataset imdb_9prevs finished [took 58.1750s]

57
quacc/data.py
View File

@ -20,13 +20,20 @@ from quapy.data import LabelledCollection
# #
class ExtensionPolicy:
def __init__(self, collapse_false=False):
self.collapse_false = collapse_false
class ExtendedData: class ExtendedData:
def __init__( def __init__(
self, self,
instances: np.ndarray | sp.csr_matrix, instances: np.ndarray | sp.csr_matrix,
pred_proba: np.ndarray, pred_proba: np.ndarray,
ext: np.ndarray = None, ext: np.ndarray = None,
extpol=None,
): ):
self.extpol = ExtensionPolicy() if extpol is None else extpol
self.b_instances_ = instances self.b_instances_ = instances
self.pred_proba_ = pred_proba self.pred_proba_ = pred_proba
self.ext_ = ext self.ext_ = ext
@ -89,15 +96,32 @@ class ExtendedData:
class ExtendedLabels: class ExtendedLabels:
def __init__(self, true: np.ndarray, pred: np.ndarray, ncl: np.ndarray): def __init__(
self,
true: np.ndarray,
pred: np.ndarray,
ncl: np.ndarray,
extpol: ExtensionPolicy = None,
):
self.extpol = ExtensionPolicy() if extpol is None else extpol
self.true = true self.true = true
self.pred = pred self.pred = pred
self.ncl = ncl self.ncl = ncl
@property @property
def y(self): def y(self):
if self.extpol.collapse_false:
return self.true + self.pred
else:
return self.true * self.ncl + self.pred return self.true * self.ncl + self.pred
@property
def classes(self):
if self.extpol.collapse_false:
return np.arange(self.ncl + 1)
else:
return np.arange(self.ncl**2)
def __getitem__(self, idx): def __getitem__(self, idx):
return ExtendedLabels(self.true[idx], self.pred[idx], self.ncl) return ExtendedLabels(self.true[idx], self.pred[idx], self.ncl)
@ -109,8 +133,10 @@ class ExtendedCollection(LabelledCollection):
labels: np.ndarray, labels: np.ndarray,
pred_proba: np.ndarray = None, pred_proba: np.ndarray = None,
ext: np.ndarray = None, ext: np.ndarray = None,
extpol=None,
): ):
e_data, e_labels, _classes = self.__extend_collection( self.extpol = ExtensionPolicy() if extpol is None else extpol
e_data, e_labels = self.__extend_collection(
instances=instances, instances=instances,
labels=labels, labels=labels,
pred_proba=pred_proba, pred_proba=pred_proba,
@ -118,16 +144,19 @@ class ExtendedCollection(LabelledCollection):
) )
self.e_data_ = e_data self.e_data_ = e_data
self.e_labels_ = e_labels self.e_labels_ = e_labels
super().__init__(e_data.X, e_labels.y, classes=_classes) super().__init__(e_data.X, e_labels.y, classes=e_labels.classes)
@classmethod @classmethod
def from_lc( def from_lc(
cls, cls,
lc: LabelledCollection, lc: LabelledCollection,
predict_proba: np.ndarray, pred_proba: np.ndarray,
ext: np.ndarray = None, ext: np.ndarray = None,
extpol=None,
): ):
return ExtendedCollection(lc.X, lc.y, pred_proba=predict_proba, ext=ext) return ExtendedCollection(
lc.X, lc.y, pred_proba=pred_proba, ext=ext, extpol=extpol
)
@property @property
def pred_proba(self): def pred_proba(self):
@ -145,6 +174,13 @@ class ExtendedCollection(LabelledCollection):
def ey(self): def ey(self):
return self.e_labels_ return self.e_labels_
def counts(self):
_counts = super().counts()
if self.extpol.collapse_false:
_counts = np.insert(_counts, 2, 0)
return _counts
def split_by_pred(self): def split_by_pred(self):
_ncl = len(self.pred_proba) _ncl = len(self.pred_proba)
_instances, _indexes = self.e_data_.split_by_pred(return_indexes=True) _instances, _indexes = self.e_data_.split_by_pred(return_indexes=True)
@ -160,13 +196,14 @@ class ExtendedCollection(LabelledCollection):
labels: np.ndarray, labels: np.ndarray,
pred_proba: np.ndarray, pred_proba: np.ndarray,
ext: np.ndarray = None, ext: np.ndarray = None,
) -> Tuple[ExtendedData, ExtendedLabels, np.ndarray]: extpol=None,
n_classes = np.unique(labels).shape[0] ) -> Tuple[ExtendedData, ExtendedLabels]:
n_classes = pred_proba.shape[1]
# n_X = [ X | predicted probs. ] # n_X = [ X | predicted probs. ]
e_instances = ExtendedData(instances, pred_proba, ext=ext) e_instances = ExtendedData(instances, pred_proba, ext=ext, extpol=self.extpol)
# n_y = (exptected y, predicted y) # n_y = (exptected y, predicted y)
preds = np.argmax(pred_proba, axis=-1) preds = np.argmax(pred_proba, axis=-1)
e_labels = ExtendedLabels(labels, preds, n_classes) e_labels = ExtendedLabels(labels, preds, n_classes, extpol=self.extpol)
return e_instances, e_labels, np.arange(n_classes**2) return e_instances, e_labels

40
quacc/evaluation/method.py
View File

@ -76,6 +76,15 @@ def mul_sld(c_model, validation, protocol) -> EvaluationReport:
) )
@method
def mul3w_sld(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(c_model, SLD(LogisticRegression()), collapse_false=True).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method @method
def binc_sld(c_model, validation, protocol) -> EvaluationReport: def binc_sld(c_model, validation, protocol) -> EvaluationReport:
est = BQAE( est = BQAE(
@ -102,6 +111,20 @@ def mulc_sld(c_model, validation, protocol) -> EvaluationReport:
) )
@method
def mul3wc_sld(c_model, validation, protocol) -> EvaluationReport:
est = MCAE(
c_model,
SLD(LogisticRegression()),
confidence=["max_conf", "entropy"],
collapse_false=True,
).fit(validation)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method @method
def binmc_sld(c_model, validation, protocol) -> EvaluationReport: def binmc_sld(c_model, validation, protocol) -> EvaluationReport:
est = BQAE( est = BQAE(
@ -188,6 +211,23 @@ def mul_sld_gs(c_model, validation, protocol) -> EvaluationReport:
) )
@method
def mul3w_sld_gs(c_model, validation, protocol) -> EvaluationReport:
v_train, v_val = validation.split_stratified(0.6, random_state=0)
model = MCAE(c_model, SLD(LogisticRegression()), collapse_false=True)
est = GridSearchAE(
model=model,
param_grid=_sld_param_grid,
refit=False,
protocol=UPP(v_val, repeats=100),
verbose=True,
).fit(v_train)
return evaluation_report(
estimator=est,
protocol=protocol,
)
@method @method
def bin_sld_gsq(c_model, validation, protocol) -> EvaluationReport: def bin_sld_gsq(c_model, validation, protocol) -> EvaluationReport:
est = BQAEgsq( est = BQAEgsq(

187
quacc/evaluation/report.py
View File

@ -1,3 +1,4 @@
import pickle
from pathlib import Path from pathlib import Path
from typing import List, Tuple from typing import List, Tuple
@ -145,7 +146,12 @@ class CompReport:
return avg_p return avg_p
def get_plots( def get_plots(
self, mode="delta", metric="acc", estimators=None, conf="default", stdev=False self,
mode="delta",
metric="acc",
estimators=None,
conf="default",
return_fig=False,
) -> List[Tuple[str, Path]]: ) -> List[Tuple[str, Path]]:
if mode == "delta": if mode == "delta":
avg_data = self.avg_by_prevs(metric=metric, estimators=estimators) avg_data = self.avg_by_prevs(metric=metric, estimators=estimators)
@ -156,6 +162,7 @@ class CompReport:
metric=metric, metric=metric,
name=conf, name=conf,
train_prev=self.train_prev, train_prev=self.train_prev,
return_fig=return_fig,
) )
elif mode == "delta_stdev": elif mode == "delta_stdev":
avg_data = self.avg_by_prevs(metric=metric, estimators=estimators) avg_data = self.avg_by_prevs(metric=metric, estimators=estimators)
@ -168,6 +175,7 @@ class CompReport:
name=conf, name=conf,
train_prev=self.train_prev, train_prev=self.train_prev,
stdevs=st_data.T.to_numpy(), stdevs=st_data.T.to_numpy(),
return_fig=return_fig,
) )
elif mode == "diagonal": elif mode == "diagonal":
f_data = self.data(metric=metric + "_score", estimators=estimators) f_data = self.data(metric=metric + "_score", estimators=estimators)
@ -180,6 +188,7 @@ class CompReport:
metric=metric, metric=metric,
name=conf, name=conf,
train_prev=self.train_prev, train_prev=self.train_prev,
return_fig=return_fig,
) )
elif mode == "shift": elif mode == "shift":
_shift_data = self.shift_data(metric=metric, estimators=estimators) _shift_data = self.shift_data(metric=metric, estimators=estimators)
@ -197,6 +206,7 @@ class CompReport:
name=conf, name=conf,
train_prev=self.train_prev, train_prev=self.train_prev,
counts=shift_counts.T.to_numpy(), counts=shift_counts.T.to_numpy(),
return_fig=return_fig,
) )
def to_md(self, conf="default", metric="acc", estimators=None, stdev=False) -> str: def to_md(self, conf="default", metric="acc", estimators=None, stdev=False) -> str:
@ -219,7 +229,6 @@ class CompReport:
metric=metric, metric=metric,
estimators=estimators, estimators=estimators,
conf=conf, conf=conf,
stdev=stdev,
) )
res += f"![plot_{mode}]({op.relative_to(env.OUT_DIR).as_posix()})\n" res += f"![plot_{mode}]({op.relative_to(env.OUT_DIR).as_posix()})\n"
@ -287,6 +296,95 @@ class DatasetReport:
self.add(cr) self.add(cr)
return self return self
def get_plots(
self,
data=None,
mode="delta_train",
metric="acc",
estimators=None,
conf="default",
return_fig=False,
):
if mode == "delta_train":
_data = self.data(metric, estimators) if data is None else data
avg_on_train = _data.groupby(level=1).mean()
prevs_on_train = np.sort(avg_on_train.index.unique(0))
return plot.plot_delta(
base_prevs=np.around(
[(1.0 - p, p) for p in prevs_on_train], decimals=2
),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
avg="train",
return_fig=return_fig,
)
elif mode == "stdev_train":
_data = self.data(metric, estimators) if data is None else data
avg_on_train = _data.groupby(level=1).mean()
prevs_on_train = np.sort(avg_on_train.index.unique(0))
stdev_on_train = _data.groupby(level=1).std()
return plot.plot_delta(
base_prevs=np.around(
[(1.0 - p, p) for p in prevs_on_train], decimals=2
),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
stdevs=stdev_on_train.T.to_numpy(),
avg="train",
return_fig=return_fig,
)
elif mode == "delta_test":
_data = self.data(metric, estimators) if data is None else data
avg_on_test = _data.groupby(level=0).mean()
prevs_on_test = np.sort(avg_on_test.index.unique(0))
return plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2),
columns=avg_on_test.columns.to_numpy(),
data=avg_on_test.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
avg="test",
return_fig=return_fig,
)
elif mode == "stdev_test":
_data = self.data(metric, estimators) if data is None else data
avg_on_test = _data.groupby(level=0).mean()
prevs_on_test = np.sort(avg_on_test.index.unique(0))
stdev_on_test = _data.groupby(level=0).std()
return plot.plot_delta(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2),
columns=avg_on_test.columns.to_numpy(),
data=avg_on_test.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
stdevs=stdev_on_test.T.to_numpy(),
avg="test",
return_fig=return_fig,
)
elif mode == "shift":
_shift_data = self.shift_data(metric, estimators) if data is None else data
avg_shift = _shift_data.groupby(level=0).mean()
count_shift = _shift_data.groupby(level=0).count()
prevs_shift = np.sort(avg_shift.index.unique(0))
return plot.plot_shift(
shift_prevs=np.around([(1.0 - p, p) for p in prevs_shift], decimals=2),
columns=avg_shift.columns.to_numpy(),
data=avg_shift.T.to_numpy(),
metric=metric,
name=conf,
train_prev=None,
counts=count_shift.T.to_numpy(),
return_fig=return_fig,
)
def to_md(self, conf="default", metric="acc", estimators=[], stdev=False): def to_md(self, conf="default", metric="acc", estimators=[], stdev=False):
res = f"# {self.name}\n\n" res = f"# {self.name}\n\n"
for cr in self.crs: for cr in self.crs:
@ -300,95 +398,82 @@ class DatasetReport:
######################## avg on train ######################## ######################## avg on train ########################
res += "### avg on train\n" res += "### avg on train\n"
avg_on_train = _data.groupby(level=1).mean()
prevs_on_train = np.sort(avg_on_train.index.unique(0))
stdev_on_train = _data.groupby(level=1).std() if stdev else None
avg_on_train_tbl = _data.groupby(level=1).mean() avg_on_train_tbl = _data.groupby(level=1).mean()
avg_on_train_tbl.loc["avg", :] = _data.mean() avg_on_train_tbl.loc["avg", :] = _data.mean()
res += avg_on_train_tbl.to_html() + "\n\n" res += avg_on_train_tbl.to_html() + "\n\n"
delta_op = plot.plot_delta( delta_op = self.get_plots(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_train], decimals=2), data=_data,
columns=avg_on_train.columns.to_numpy(), mode="delta_train",
data=avg_on_train.T.to_numpy(),
metric=metric, metric=metric,
name=conf, estimators=estimators,
train_prev=None, conf=conf,
avg="train",
) )
res += f"![plot_delta]({delta_op.relative_to(env.OUT_DIR).as_posix()})\n" res += f"![plot_delta]({delta_op.relative_to(env.OUT_DIR).as_posix()})\n"
if stdev: if stdev:
delta_stdev_op = plot.plot_delta( delta_stdev_op = self.get_plots(
base_prevs=np.around( data=_data,
[(1.0 - p, p) for p in prevs_on_train], decimals=2 mode="stdev_train",
),
columns=avg_on_train.columns.to_numpy(),
data=avg_on_train.T.to_numpy(),
metric=metric, metric=metric,
name=conf, estimators=estimators,
train_prev=None, conf=conf,
stdevs=stdev_on_train.T.to_numpy(),
avg="train",
) )
res += f"![plot_delta_stdev]({delta_stdev_op.relative_to(env.OUT_DIR).as_posix()})\n" res += f"![plot_delta_stdev]({delta_stdev_op.relative_to(env.OUT_DIR).as_posix()})\n"
######################## avg on test ######################## ######################## avg on test ########################
res += "### avg on test\n" res += "### avg on test\n"
avg_on_test = _data.groupby(level=0).mean()
prevs_on_test = np.sort(avg_on_test.index.unique(0))
stdev_on_test = _data.groupby(level=0).std() if stdev else None
avg_on_test_tbl = _data.groupby(level=0).mean() avg_on_test_tbl = _data.groupby(level=0).mean()
avg_on_test_tbl.loc["avg", :] = _data.mean() avg_on_test_tbl.loc["avg", :] = _data.mean()
res += avg_on_test_tbl.to_html() + "\n\n" res += avg_on_test_tbl.to_html() + "\n\n"
delta_op = plot.plot_delta( delta_op = self.get_plots(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2), data=_data,
columns=avg_on_test.columns.to_numpy(), mode="delta_test",
data=avg_on_test.T.to_numpy(),
metric=metric, metric=metric,
name=conf, estimators=estimators,
train_prev=None, conf=conf,
avg="test",
) )
res += f"![plot_delta]({delta_op.relative_to(env.OUT_DIR).as_posix()})\n" res += f"![plot_delta]({delta_op.relative_to(env.OUT_DIR).as_posix()})\n"
if stdev: if stdev:
delta_stdev_op = plot.plot_delta( delta_stdev_op = self.get_plots(
base_prevs=np.around([(1.0 - p, p) for p in prevs_on_test], decimals=2), data=_data,
columns=avg_on_test.columns.to_numpy(), mode="stdev_test",
data=avg_on_test.T.to_numpy(),
metric=metric, metric=metric,
name=conf, estimators=estimators,
train_prev=None, conf=conf,
stdevs=stdev_on_test.T.to_numpy(),
avg="test",
) )
res += f"![plot_delta_stdev]({delta_stdev_op.relative_to(env.OUT_DIR).as_posix()})\n" res += f"![plot_delta_stdev]({delta_stdev_op.relative_to(env.OUT_DIR).as_posix()})\n"
######################## avg shift ######################## ######################## avg shift ########################
res += "### avg dataset shift\n" res += "### avg dataset shift\n"
avg_shift = _shift_data.groupby(level=0).mean() shift_op = self.get_plots(
count_shift = _shift_data.groupby(level=0).count() data=_shift_data,
prevs_shift = np.sort(avg_shift.index.unique(0)) mode="shift",
shift_op = plot.plot_shift(
shift_prevs=np.around([(1.0 - p, p) for p in prevs_shift], decimals=2),
columns=avg_shift.columns.to_numpy(),
data=avg_shift.T.to_numpy(),
metric=metric, metric=metric,
name=conf, estimators=estimators,
train_prev=None, conf=conf,
counts=count_shift.T.to_numpy(),
) )
res += f"![plot_shift]({shift_op.relative_to(env.OUT_DIR).as_posix()})\n" res += f"![plot_shift]({shift_op.relative_to(env.OUT_DIR).as_posix()})\n"
return res return res
def pickle(self, pickle_path: Path):
with open(pickle_path, "wb") as f:
pickle.dump(self, f)
@classmethod
def unpickle(cls, pickle_path: Path):
with open(pickle_path, "rb") as f:
dr = pickle.load(f)
return dr
def __iter__(self): def __iter__(self):
return (cr for cr in self.crs) return (cr for cr in self.crs)

1
quacc/main.py
View File

@ -25,6 +25,7 @@ def estimate_comparison():
dataset, dataset,
estimators=CE.name[env.COMP_ESTIMATORS], estimators=CE.name[env.COMP_ESTIMATORS],
) )
dr.pickle(env.OUT_DIR / f"{dataset.name}.pickle")
except Exception as e: except Exception as e:
log.error(f"Evaluation over {dataset.name} failed. Exception: {e}") log.error(f"Evaluation over {dataset.name} failed. Exception: {e}")
traceback(e) traceback(e)

31
quacc/method/base.py
View File

@ -8,7 +8,7 @@ from quapy.data import LabelledCollection
from quapy.method.aggregative import BaseQuantifier from quapy.method.aggregative import BaseQuantifier
from sklearn.base import BaseEstimator from sklearn.base import BaseEstimator
from quacc.data import ExtendedCollection, ExtendedData from quacc.data import ExtendedCollection, ExtendedData, ExtensionPolicy
class BaseAccuracyEstimator(BaseQuantifier): class BaseAccuracyEstimator(BaseQuantifier):
@ -19,6 +19,7 @@ class BaseAccuracyEstimator(BaseQuantifier):
): ):
self.__check_classifier(classifier) self.__check_classifier(classifier)
self.quantifier = quantifier self.quantifier = quantifier
self.extpol = ExtensionPolicy()
def __check_classifier(self, classifier): def __check_classifier(self, classifier):
if not hasattr(classifier, "predict_proba"): if not hasattr(classifier, "predict_proba"):
@ -31,13 +32,15 @@ class BaseAccuracyEstimator(BaseQuantifier):
if pred_proba is None: if pred_proba is None:
pred_proba = self.classifier.predict_proba(coll.X) pred_proba = self.classifier.predict_proba(coll.X)
return ExtendedCollection.from_lc(coll, pred_proba=pred_proba) return ExtendedCollection.from_lc(
coll, pred_proba=pred_proba, extpol=self.extpol
)
def _extend_instances(self, instances: np.ndarray | sp.csr_matrix, pred_proba=None): def _extend_instances(self, instances: np.ndarray | sp.csr_matrix, pred_proba=None):
if pred_proba is None: if pred_proba is None:
pred_proba = self.classifier.predict_proba(instances) pred_proba = self.classifier.predict_proba(instances)
return ExtendedData(instances, pred_proba=pred_proba) return ExtendedData(instances, pred_proba=pred_proba, extpol=self.extpol)
@abstractmethod @abstractmethod
def fit(self, train: LabelledCollection | ExtendedCollection): def fit(self, train: LabelledCollection | ExtendedCollection):
@ -106,7 +109,9 @@ class ConfidenceBasedAccuracyEstimator(BaseAccuracyEstimator):
pred_proba = self.classifier.predict_proba(coll.X) pred_proba = self.classifier.predict_proba(coll.X)
_ext = self.__get_ext(pred_proba) _ext = self.__get_ext(pred_proba)
return ExtendedCollection.from_lc(coll, pred_proba=pred_proba, ext=_ext) return ExtendedCollection.from_lc(
coll, pred_proba=pred_proba, ext=_ext, extpol=self.extpol
)
def _extend_instances( def _extend_instances(
self, self,
@ -117,7 +122,9 @@ class ConfidenceBasedAccuracyEstimator(BaseAccuracyEstimator):
pred_proba = self.classifier.predict_proba(instances) pred_proba = self.classifier.predict_proba(instances)
_ext = self.__get_ext(pred_proba) _ext = self.__get_ext(pred_proba)
return ExtendedData(instances, pred_proba=pred_proba, ext=_ext) return ExtendedData(
instances, pred_proba=pred_proba, ext=_ext, extpol=self.extpol
)
class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator): class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
@ -126,6 +133,7 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
classifier: BaseEstimator, classifier: BaseEstimator,
quantifier: BaseQuantifier, quantifier: BaseQuantifier,
confidence: str = None, confidence: str = None,
collapse_false=False,
): ):
super().__init__( super().__init__(
classifier=classifier, classifier=classifier,
@ -133,6 +141,7 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
confidence=confidence, confidence=confidence,
) )
self.e_train = None self.e_train = None
self.extpol = ExtensionPolicy(collapse_false=collapse_false)
def fit(self, train: LabelledCollection): def fit(self, train: LabelledCollection):
self.e_train = self.extend(train) self.e_train = self.extend(train)
@ -149,7 +158,13 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
e_inst = self._extend_instances(instances) e_inst = self._extend_instances(instances)
estim_prev = self.quantifier.quantify(e_inst.X) estim_prev = self.quantifier.quantify(e_inst.X)
return self._check_prevalence_classes(estim_prev, self.quantifier.classes_) estim_prev = self._check_prevalence_classes(
estim_prev, self.quantifier.classes_
)
if self.extpol.collapse_false:
estim_prev = np.insert(estim_prev, 2, 0.0)
return estim_prev
def _check_prevalence_classes(self, estim_prev, estim_classes) -> np.ndarray: def _check_prevalence_classes(self, estim_prev, estim_classes) -> np.ndarray:
true_classes = self.e_train.classes_ true_classes = self.e_train.classes_
@ -158,6 +173,10 @@ class MultiClassAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
estim_prev = np.insert(estim_prev, _cls, [0.0], axis=0) estim_prev = np.insert(estim_prev, _cls, [0.0], axis=0)
return estim_prev return estim_prev
@property
def collapse_false(self):
return self.extpol.collapse_false
class BinaryQuantifierAccuracyEstimator(ConfidenceBasedAccuracyEstimator): class BinaryQuantifierAccuracyEstimator(ConfidenceBasedAccuracyEstimator):
def __init__( def __init__(

1
quacc/method/model_selection.py
View File

@ -174,6 +174,7 @@ class GridSearchAE(BaseAccuracyEstimator):
except Exception as e: except Exception as e:
self._sout(f"something went wrong for config {params}; skipping:") self._sout(f"something went wrong for config {params}; skipping:")
self._sout(f"\tException: {e}") self._sout(f"\tException: {e}")
score = None score = None
return params, score, model return params, score, model

16
quacc/plot.py
View File

@ -29,6 +29,7 @@ def plot_delta(
train_prev=None, train_prev=None,
legend=True, legend=True,
avg=None, avg=None,
return_fig=False,
) -> Path: ) -> Path:
_base_title = "delta_stdev" if stdevs is not None else "delta" _base_title = "delta_stdev" if stdevs is not None else "delta"
if train_prev is not None: if train_prev is not None:
@ -84,9 +85,12 @@ def plot_delta(
if legend: if legend:
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5)) ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
if return_fig:
return fig
output_path = env.PLOT_OUT_DIR / f"{title}.png" output_path = env.PLOT_OUT_DIR / f"{title}.png"
fig.savefig(output_path, bbox_inches="tight") fig.savefig(output_path, bbox_inches="tight")
return output_path return output_path
@ -100,6 +104,7 @@ def plot_diagonal(
name="default", name="default",
train_prev=None, train_prev=None,
legend=True, legend=True,
return_fig=False,
): ):
if train_prev is not None: if train_prev is not None:
t_prev_pos = int(round(train_prev[pos_class] * 100)) t_prev_pos = int(round(train_prev[pos_class] * 100))
@ -169,6 +174,10 @@ def plot_diagonal(
if legend: if legend:
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5)) ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
if return_fig:
return fig
output_path = env.PLOT_OUT_DIR / f"{title}.png" output_path = env.PLOT_OUT_DIR / f"{title}.png"
fig.savefig(output_path, bbox_inches="tight") fig.savefig(output_path, bbox_inches="tight")
return output_path return output_path
@ -185,6 +194,7 @@ def plot_shift(
name="default", name="default",
train_prev=None, train_prev=None,
legend=True, legend=True,
return_fig=False,
) -> Path: ) -> Path:
if train_prev is not None: if train_prev is not None:
t_prev_pos = int(round(train_prev[pos_class] * 100)) t_prev_pos = int(round(train_prev[pos_class] * 100))
@ -233,6 +243,10 @@ def plot_shift(
if legend: if legend:
ax.legend(loc="center left", bbox_to_anchor=(1, 0.5)) ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
if return_fig:
return fig
output_path = env.PLOT_OUT_DIR / f"{title}.png" output_path = env.PLOT_OUT_DIR / f"{title}.png"
fig.savefig(output_path, bbox_inches="tight") fig.savefig(output_path, bbox_inches="tight")

9
remote.py
View File

@ -21,10 +21,12 @@ __to_sync_up = {
"dir": [ "dir": [
"quacc", "quacc",
"baselines", "baselines",
"qcpanel",
], ],
"file": [ "file": [
"conf.yaml", "conf.yaml",
"run.py", "run.py",
"pyproject.toml",
], ],
} }
__to_sync_down = { __to_sync_down = {
@ -60,10 +62,13 @@ def put_dir(sftp: paramiko.SFTPClient, from_: Path, to_: Path):
elif (from_ / f).is_dir(): elif (from_ / f).is_dir():
_ex_list += put_dir(sftp, from_ / f, to_ / f) _ex_list += put_dir(sftp, from_ / f, to_ / f)
try:
to_list = sftp.listdir(str(to_)) to_list = sftp.listdir(str(to_))
for f in to_list: for f in to_list:
if f not in from_list: if f not in from_list:
_ex_list += prune_remote(sftp, to_ / f) _ex_list += prune_remote(sftp, to_ / f)
except FileNotFoundError:
pass
return _ex_list return _ex_list
@ -78,10 +83,10 @@ def get_dir(sftp: paramiko.SFTPClient, from_: Path, to_: Path):
mode = sftp.stat(str(from_ / f)).st_mode mode = sftp.stat(str(from_ / f)).st_mode
if stat.S_ISDIR(mode): if stat.S_ISDIR(mode):
_ex_list += get_dir(sftp, from_ / f, to_ / f) _ex_list += get_dir(sftp, from_ / f, to_ / f)
_ex_list.append([sftp.rmdir, str(from_ / f)]) # _ex_list.append([sftp.rmdir, str(from_ / f)])
elif stat.S_ISREG(mode): elif stat.S_ISREG(mode):
_ex_list.append([sftp.get, str(from_ / f), str(to_ / f)]) _ex_list.append([sftp.get, str(from_ / f), str(to_ / f)])
_ex_list.append([sftp.remove, str(from_ / f)]) # _ex_list.append([sftp.remove, str(from_ / f)])
return _ex_list return _ex_list