diff --git a/conf.yaml b/conf.yaml index 7dcb010..5f6cd33 100644 --- a/conf.yaml +++ b/conf.yaml @@ -137,10 +137,10 @@ add_conf: &add_conf DATASET_N_PREVS: 9 confs: + - DATASET_NAME: imdb + confs_next: - DATASET_NAME: rcv1 DATASET_TARGET: CCAT - confs_next: - - DATASET_NAME: imdb - DATASET_NAME: rcv1 DATASET_TARGET: GCAT - DATASET_NAME: rcv1 diff --git a/quacc.log b/quacc.log index dfbd382..c7b22ba 100644 --- a/quacc.log +++ b/quacc.log @@ -10765,3 +10765,95 @@ 22/11/23 14:46:10| INFO m3wne_sld finished [took 228.2859s] 22/11/23 14:46:11| INFO m3wc_sld finished [took 231.3908s] 22/11/23 14:46:13| INFO m3wis_sld finished [took 228.9895s] +---------------------------------------------------------------------------------------------------- +22/11/23 22:39:25| INFO dataset imdb [pool size: 10] +22/11/23 22:39:31| INFO Dataset sample 0.10 of dataset imdb started +22/11/23 22:40:05| INFO ref finished [took 26.7125s] +22/11/23 22:40:15| INFO m3wne_sld finished [took 39.9900s] +22/11/23 22:40:16| INFO m3wmc_sld finished [took 41.6852s] +22/11/23 22:40:16| INFO m3wc_sld finished [took 42.1895s] +22/11/23 22:40:17| INFO m3wis_sld finished [took 40.8057s] +22/11/23 22:40:17| INFO m3w_sld finished [took 45.2795s] +22/11/23 22:45:50| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00789) [took 372.6118s] +22/11/23 22:46:00| INFO m3w_sld_gs finished [took 383.2764s] +22/11/23 22:46:00| INFO Dataset sample 0.10 of dataset imdb finished [took 388.8456s] +22/11/23 22:46:00| INFO Dataset sample 0.20 of dataset imdb started +22/11/23 22:46:36| INFO ref finished [took 26.0360s] +22/11/23 22:46:42| INFO m3wne_sld finished [took 35.7751s] +22/11/23 22:46:43| INFO m3wc_sld finished [took 41.0112s] +22/11/23 22:46:44| INFO m3w_sld finished [took 42.4419s] +22/11/23 22:46:45| INFO m3wis_sld finished [took 37.4503s] +22/11/23 22:46:45| INFO m3wmc_sld finished [took 40.8884s] +22/11/23 22:53:05| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1000.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00786) [took 417.0861s] +22/11/23 22:53:17| INFO m3w_sld_gs finished [took 428.8639s] +22/11/23 22:53:17| INFO Dataset sample 0.20 of dataset imdb finished [took 436.4691s] +22/11/23 22:53:17| INFO Dataset sample 0.30 of dataset imdb started +22/11/23 22:54:04| INFO ref finished [took 32.2664s] +22/11/23 22:54:15| INFO m3w_sld finished [took 51.8639s] +22/11/23 22:54:15| INFO m3wis_sld finished [took 44.8139s] +22/11/23 22:54:15| INFO m3wmc_sld finished [took 46.5044s] +22/11/23 22:54:15| INFO m3wc_sld finished [took 50.4231s] +22/11/23 22:54:16| INFO m3wne_sld finished [took 46.3459s] +22/11/23 23:00:21| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00853) [took 411.8293s] +22/11/23 23:00:31| INFO m3w_sld_gs finished [took 422.5298s] +22/11/23 23:00:31| INFO Dataset sample 0.30 of dataset imdb finished [took 434.6217s] +22/11/23 23:00:31| INFO Dataset sample 0.40 of dataset imdb started +22/11/23 23:01:13| INFO ref finished [took 30.7259s] +22/11/23 23:01:21| INFO m3w_sld finished [took 48.4534s] +22/11/23 23:01:22| INFO m3wmc_sld finished [took 45.3415s] +22/11/23 23:01:22| INFO m3wc_sld finished [took 47.5734s] +22/11/23 23:01:24| INFO m3wis_sld finished [took 44.8038s] +22/11/23 23:01:25| INFO m3wne_sld finished [took 46.9841s] +22/11/23 23:06:59| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.01874) [took 380.7642s] +22/11/23 23:07:10| INFO m3w_sld_gs finished [took 391.7746s] +22/11/23 23:07:10| INFO Dataset sample 0.40 of dataset imdb finished [took 398.3842s] +22/11/23 23:07:10| INFO Dataset sample 0.50 of dataset imdb started +22/11/23 23:07:53| INFO ref finished [took 33.9676s] +22/11/23 23:08:01| INFO m3w_sld finished [took 49.9842s] +22/11/23 23:08:04| INFO m3wis_sld finished [took 47.8397s] +22/11/23 23:08:05| INFO m3wmc_sld finished [took 48.4244s] +22/11/23 23:08:05| INFO m3wc_sld finished [took 50.3583s] +22/11/23 23:08:06| INFO m3wne_sld finished [took 49.9964s] +22/11/23 23:13:46| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.01851) [took 387.4653s] +22/11/23 23:13:57| INFO m3w_sld_gs finished [took 398.5473s] +22/11/23 23:13:57| INFO Dataset sample 0.50 of dataset imdb finished [took 407.1600s] +22/11/23 23:13:57| INFO Dataset sample 0.60 of dataset imdb started +22/11/23 23:14:36| INFO ref finished [took 31.8305s] +22/11/23 23:14:46| INFO m3w_sld finished [took 47.2148s] +22/11/23 23:14:48| INFO m3wmc_sld finished [took 45.0544s] +22/11/23 23:14:49| INFO m3wis_sld finished [took 45.5847s] +22/11/23 23:14:49| INFO m3wne_sld finished [took 46.6914s] +22/11/23 23:14:49| INFO m3wc_sld finished [took 48.5027s] +22/11/23 23:20:29| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.02000) [took 385.6402s] +22/11/23 23:20:39| INFO m3w_sld_gs finished [took 396.1211s] +22/11/23 23:20:39| INFO Dataset sample 0.60 of dataset imdb finished [took 402.5207s] +22/11/23 23:20:39| INFO Dataset sample 0.70 of dataset imdb started +22/11/23 23:21:15| INFO ref finished [took 28.4956s] +22/11/23 23:21:26| INFO m3wis_sld finished [took 39.6972s] +22/11/23 23:21:27| INFO m3wne_sld finished [took 40.9912s] +22/11/23 23:21:27| INFO m3w_sld finished [took 45.7233s] +22/11/23 23:21:28| INFO m3wmc_sld finished [took 42.7001s] +22/11/23 23:21:29| INFO m3wc_sld finished [took 45.8509s] +22/11/23 23:27:06| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.001, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00875) [took 379.2411s] +22/11/23 23:27:16| INFO m3w_sld_gs finished [took 389.5301s] +22/11/23 23:27:16| INFO Dataset sample 0.70 of dataset imdb finished [took 397.0096s] +22/11/23 23:27:16| INFO Dataset sample 0.80 of dataset imdb started +22/11/23 23:27:51| INFO ref finished [took 28.1018s] +22/11/23 23:28:00| INFO m3wis_sld finished [took 37.8200s] +22/11/23 23:28:00| INFO m3wc_sld finished [took 40.2210s] +22/11/23 23:28:00| INFO m3wne_sld finished [took 38.4939s] +22/11/23 23:28:01| INFO m3w_sld finished [took 42.3041s] +22/11/23 23:28:01| INFO m3wmc_sld finished [took 39.0396s] +22/11/23 23:33:09| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00789) [took 344.2546s] +22/11/23 23:33:19| INFO m3w_sld_gs finished [took 354.4437s] +22/11/23 23:33:19| INFO Dataset sample 0.80 of dataset imdb finished [took 362.5184s] +22/11/23 23:33:19| INFO Dataset sample 0.90 of dataset imdb started +22/11/23 23:33:52| INFO ref finished [took 26.5296s] +22/11/23 23:34:00| INFO m3wc_sld finished [took 39.1261s] +22/11/23 23:34:00| INFO m3wis_sld finished [took 37.7493s] +22/11/23 23:34:01| INFO m3wne_sld finished [took 38.9486s] +22/11/23 23:34:03| INFO m3wmc_sld finished [took 39.5300s] +22/11/23 23:34:03| INFO m3w_sld finished [took 43.5324s] +22/11/23 23:39:04| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00807) [took 339.8089s] +22/11/23 23:39:14| INFO m3w_sld_gs finished [took 349.9083s] +22/11/23 23:39:14| INFO Dataset sample 0.90 of dataset imdb finished [took 355.6212s] diff --git a/remote.log b/remote.log index 6510979..de2af92 100644 --- a/remote.log +++ b/remote.log @@ -6341,3 +6341,118 @@ 22/11/23 19:12:36| INFO m3wne_sld finished [took 151.7698s] 22/11/23 19:12:41| INFO m3wc_sld finished [took 170.6292s] 22/11/23 19:12:41| INFO m3wis_sld finished [took 148.7262s] +22/11/23 19:23:47| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.01201) [took 806.8229s] +22/11/23 19:24:35| INFO m3w_sld_gs finished [took 855.0132s] +22/11/23 19:24:35| INFO Dataset sample 0.50 of dataset rcv1_CCAT finished [took 893.3306s] +22/11/23 19:24:35| INFO Dataset sample 0.60 of dataset rcv1_CCAT started +22/11/23 19:27:22| INFO m3w_sld finished [took 164.2611s] +22/11/23 19:27:24| INFO ref finished [took 128.6429s] +22/11/23 19:27:28| INFO m3wne_sld finished [took 152.0422s] +22/11/23 19:27:30| INFO m3wis_sld finished [took 146.4067s] +22/11/23 19:27:30| INFO m3wmc_sld finished [took 163.4932s] +22/11/23 19:27:36| INFO m3wc_sld finished [took 174.6631s] +22/11/23 19:38:10| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1000.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01344) [took 779.9355s] +22/11/23 19:38:58| INFO m3w_sld_gs finished [took 827.8476s] +22/11/23 19:38:58| INFO Dataset sample 0.60 of dataset rcv1_CCAT finished [took 862.8897s] +22/11/23 19:38:58| INFO Dataset sample 0.70 of dataset rcv1_CCAT started +22/11/23 19:41:40| INFO m3wc_sld finished [took 154.0147s] +22/11/23 19:41:48| INFO m3w_sld finished [took 167.3630s] +22/11/23 19:41:51| INFO m3wmc_sld finished [took 157.6084s] +22/11/23 19:41:53| INFO ref finished [took 131.6654s] +22/11/23 19:41:56| INFO m3wne_sld finished [took 156.5850s] +22/11/23 19:42:00| INFO m3wis_sld finished [took 152.7024s] +22/11/23 19:52:59| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.01, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.01149) [took 806.0229s] +22/11/23 19:53:47| INFO m3w_sld_gs finished [took 853.9109s] +22/11/23 19:53:47| INFO Dataset sample 0.70 of dataset rcv1_CCAT finished [took 889.2442s] +22/11/23 19:53:47| INFO Dataset sample 0.80 of dataset rcv1_CCAT started +22/11/23 19:56:28| INFO m3w_sld finished [took 158.0458s] +22/11/23 19:56:32| INFO m3wmc_sld finished [took 150.1753s] +22/11/23 19:56:34| INFO m3wne_sld finished [took 143.2132s] +22/11/23 19:56:34| INFO m3wc_sld finished [took 158.6395s] +22/11/23 19:56:41| INFO m3wis_sld finished [took 143.7058s] +22/11/23 19:56:43| INFO ref finished [took 131.2530s] +22/11/23 20:07:14| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00667) [took 767.0383s] +22/11/23 20:08:02| INFO m3w_sld_gs finished [took 815.4612s] +22/11/23 20:08:02| INFO Dataset sample 0.80 of dataset rcv1_CCAT finished [took 855.2950s] +22/11/23 20:08:02| INFO Dataset sample 0.90 of dataset rcv1_CCAT started +22/11/23 20:10:39| INFO m3wc_sld finished [took 148.6246s] +22/11/23 20:10:43| INFO m3wne_sld finished [took 139.3615s] +22/11/23 20:10:48| INFO m3wmc_sld finished [took 150.8955s] +22/11/23 20:10:48| INFO m3wis_sld finished [took 137.8562s] +22/11/23 20:10:50| INFO ref finished [took 126.2941s] +22/11/23 20:10:50| INFO m3w_sld finished [took 164.5047s] +22/11/23 20:21:26| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': None, 'confidence': ['isoft']} (score=0.00779) [took 767.6325s] +22/11/23 20:22:13| INFO m3w_sld_gs finished [took 815.3539s] +22/11/23 20:22:13| INFO Dataset sample 0.90 of dataset rcv1_CCAT finished [took 851.1516s] +---------------------------------------------------------------------------------------------------- +22/11/23 21:34:29| INFO dataset rcv1_GCAT [pool size: 10] +22/11/23 21:34:34| INFO Dataset sample 0.10 of dataset rcv1_GCAT started +22/11/23 21:36:48| INFO mulc_sld finished [took 112.8084s] +22/11/23 21:37:02| INFO binne_sld finished [took 96.0149s] +22/11/23 21:37:21| INFO bin_sld finished [took 164.0656s] +22/11/23 21:37:37| INFO binc_sld finished [took 166.3021s] +22/11/23 21:37:42| INFO m3w_sld finished [took 178.1518s] +22/11/23 21:37:57| INFO m3wc_sld finished [took 176.7652s] +22/11/23 21:38:03| INFO binmc_sld finished [took 174.2003s] +22/11/23 21:38:07| INFO m3wmc_sld finished [took 167.4303s] +22/11/23 21:38:17| INFO mulmc_sld finished [took 183.7776s] +22/11/23 21:38:33| WARNING Method bin_sld_gs failed. Exception: This solver needs samples of at least 2 classes in the data, but the data contains only one class: 1 +22/11/23 21:39:15| INFO mul_sld finished [took 275.2884s] +22/11/23 21:40:36| INFO mulne_sld finished [took 223.4173s] +22/11/23 21:40:41| INFO m3wne_sld finished [took 214.6766s] +22/11/23 21:41:23| INFO binis_sld finished [took 236.6607s] +22/11/23 21:41:53| INFO m3wis_sld finished [took 244.2287s] +22/11/23 21:42:01| INFO mulis_sld finished [took 258.6646s] +22/11/23 21:42:20| INFO ref finished [took 237.6574s] +22/11/23 21:42:51| INFO atc_mc finished [took 252.8951s] +22/11/23 21:43:19| INFO doc finished [took 233.1199s] +22/11/23 21:46:18| INFO gde finished [took 288.0360s] +22/11/23 22:21:31| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 0.1, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00589) [took 2596.9217s] +22/11/23 22:27:49| INFO m3w_sld_gs finished [took 2974.5469s] +22/11/23 22:29:48| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 100.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': None, 'confidence': ['max_conf', 'entropy']} (score=0.00754) [took 3099.6747s] +22/11/23 22:34:03| INFO mul_sld_gs finished [took 3354.7404s] +22/11/23 22:43:23| INFO rca finished [took 3760.5542s] +22/11/23 22:43:27| INFO rca_star finished [took 3758.0984s] +22/11/23 22:52:44| INFO mandoline finished [took 4243.1679s] +22/11/23 22:52:44| INFO Dataset sample 0.10 of dataset rcv1_GCAT finished [took 4689.9850s] +22/11/23 22:52:44| INFO Dataset sample 0.20 of dataset rcv1_GCAT started +22/11/23 22:54:59| INFO m3w_sld finished [took 123.9110s] +22/11/23 22:55:23| INFO m3wmc_sld finished [took 112.3163s] +22/11/23 22:55:54| INFO mulc_sld finished [took 168.5599s] +22/11/23 22:55:56| INFO bin_sld finished [took 189.1257s] +22/11/23 22:56:11| INFO m3wc_sld finished [took 180.5087s] +22/11/23 22:56:13| INFO mulmc_sld finished [took 167.7909s] +22/11/23 22:56:35| INFO mul_sld finished [took 224.8312s] +22/11/23 22:56:50| INFO binmc_sld finished [took 211.3909s] +22/11/23 22:57:02| INFO binc_sld finished [took 242.1542s] +22/11/23 22:57:28| INFO binne_sld finished [took 228.1177s] +22/11/23 22:59:01| INFO mulne_sld finished [took 236.1188s] +22/11/23 22:59:33| INFO m3wne_sld finished [took 243.0177s] +22/11/23 23:00:15| INFO mulis_sld finished [took 250.2579s] +22/11/23 23:00:59| INFO m3wis_sld finished [took 283.9276s] +22/11/23 23:02:10| INFO ref finished [took 301.8968s] +22/11/23 23:02:43| INFO binis_sld finished [took 403.9164s] +22/11/23 23:02:55| INFO atc_mc finished [took 317.1908s] +22/11/23 23:04:48| INFO doc finished [took 336.0960s] +22/11/23 23:08:01| INFO gde finished [took 412.1785s] +22/11/23 23:26:28| DEBUG [BinaryQuantifierAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1.0, 'quantifier__classifier__class_weight': None, 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00815) [took 1805.6920s] +22/11/23 23:35:01| INFO bin_sld_gs finished [took 2318.8880s] +22/11/23 23:47:01| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 1.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['isoft']} (score=0.00601) [took 3003.0418s] +22/11/23 23:53:48| INFO m3w_sld_gs finished [took 3409.7680s] +22/11/23 23:58:24| DEBUG [MultiClassAccuracyEstimator] optimization finished: best params {'quantifier__classifier__C': 10.0, 'quantifier__classifier__class_weight': 'balanced', 'quantifier__recalib': 'bcts', 'confidence': ['max_conf', 'entropy']} (score=0.00664) [took 3704.1129s] +23/11/23 00:03:03| INFO mul_sld_gs finished [took 3983.0059s] +23/11/23 00:11:36| INFO rca finished [took 4313.4684s] +23/11/23 00:11:57| INFO rca_star finished [took 4293.5614s] +23/11/23 00:21:07| INFO mandoline finished [took 4726.9787s] +23/11/23 00:21:07| INFO Dataset sample 0.20 of dataset rcv1_GCAT finished [took 5303.1089s] +23/11/23 00:21:07| INFO Dataset sample 0.30 of dataset rcv1_GCAT started +23/11/23 00:23:36| INFO bin_sld finished [took 145.6711s] +23/11/23 00:23:50| INFO mul_sld finished [took 156.5291s] +23/11/23 00:24:06| INFO m3wc_sld finished [took 151.1949s] +23/11/23 00:24:20| INFO m3w_sld finished [took 182.1882s] +23/11/23 00:24:37| INFO mulmc_sld finished [took 169.5278s] +23/11/23 00:24:37| INFO mulc_sld finished [took 188.3250s] +23/11/23 00:24:45| INFO m3wmc_sld finished [took 170.5893s] +23/11/23 00:24:53| INFO binc_sld finished [took 208.7789s] +23/11/23 00:25:15| INFO binmc_sld finished [took 212.4442s] +23/11/23 00:25:36| INFO binne_sld finished [took 214.4027s] diff --git a/remote.py b/remote.py index a3c0d32..02fe2c9 100644 --- a/remote.py +++ b/remote.py @@ -27,6 +27,7 @@ __to_sync_up = { "conf.yaml", "run.py", "remote.py", + "merge_data.py", "pyproject.toml", ], }