1632 lines
186 KiB
HTML
1632 lines
186 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="../../../" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>quapy.data.datasets — QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
<!--
|
||
this give us a css class that will be invisible only if js is disabled
|
||
-->
|
||
<noscript>
|
||
<style>
|
||
.pst-js-only { display: none !important; }
|
||
|
||
</style>
|
||
</noscript>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="../../../_static/styles/theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
|
||
<link href="../../../_static/styles/pydata-sphinx-theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" />
|
||
<link rel="stylesheet" type="text/css" href="../../../_static/sphinx-design.min.css?v=95c83b7e" />
|
||
<link rel="stylesheet" type="text/css" href="../../../_static/custom.css?v=9f2a2228" />
|
||
|
||
<!-- So that users can add custom icons -->
|
||
<script defer src="../../../_static/scripts/fontawesome.js?digest=90905a2f556bf617f1a9"></script>
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="../../../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9" />
|
||
<link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9" />
|
||
|
||
<script src="../../../_static/documentation_options.js?v=37f418d5"></script>
|
||
<script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
|
||
<script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||
<script src="../../../_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = '_modules/quapy/data/datasets';</script>
|
||
<script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script>
|
||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||
<link rel="search" title="Search" href="../../../search.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
<meta name="docsearch:version" content="" />
|
||
|
||
|
||
<script src="../../../_static/searchtools.js"></script>
|
||
<script src="../../../_static/language_data.js"></script>
|
||
<script src="../../../searchindex.js"></script>
|
||
|
||
</head>
|
||
<body data-default-mode="">
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
|
||
|
||
<dialog id="pst-search-dialog">
|
||
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="../../../search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
placeholder="Search the docs ..."
|
||
aria-label="Search the docs ..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form>
|
||
</dialog>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
<div class="bd-header__inner bd-page-width">
|
||
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button>
|
||
|
||
|
||
<div class="col-lg-3 navbar-header-items__start">
|
||
|
||
<div class="navbar-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="../../../index.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="../../../_static/quapy_logo.png" class="logo__image only-light" alt="QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - Home"/>
|
||
<img src="../../../_static/quapy_logo_dark.png" class="logo__image only-dark pst-js-only" alt="QuaPy: A Python-based open-source framework for quantification 0.2.1 documentation - Home"/>
|
||
|
||
|
||
</a></div>
|
||
|
||
</div>
|
||
|
||
<div class="col-lg-9 navbar-header-items">
|
||
|
||
<div class="me-auto navbar-header-items__center">
|
||
|
||
<div class="navbar-item">
|
||
<nav>
|
||
<ul class="bd-navbar-elements navbar-nav">
|
||
|
||
<li class="nav-item ">
|
||
<a class="nav-link nav-internal" href="../../../index.html">
|
||
Home
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="nav-item ">
|
||
<a class="nav-link nav-internal" href="../../../manuals.html">
|
||
Manuals
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="nav-item ">
|
||
<a class="nav-link nav-internal" href="../../../quapy.html">
|
||
API
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="navbar-header-items__end">
|
||
|
||
<div class="navbar-item navbar-persistent--container">
|
||
|
||
|
||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
</div>
|
||
|
||
|
||
<div class="navbar-item">
|
||
|
||
<div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
|
||
</button>
|
||
<ul class="dropdown-menu dropdown-menu-end">
|
||
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
|
||
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
|
||
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
|
||
</ul>
|
||
</div></div>
|
||
|
||
<div class="navbar-item"><ul class="navbar-icon-links"
|
||
aria-label="Icon Links">
|
||
<li class="nav-item">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://github.com/HLT-ISTI/QuaPy" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
|
||
</li>
|
||
</ul></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="navbar-persistent--mobile">
|
||
|
||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
</div>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
|
||
|
||
<dialog id="pst-primary-sidebar-modal"></dialog>
|
||
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar hide-on-wide">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
<div class="sidebar-header-items__center">
|
||
|
||
|
||
|
||
<div class="navbar-item">
|
||
<nav>
|
||
<ul class="bd-navbar-elements navbar-nav">
|
||
|
||
<li class="nav-item ">
|
||
<a class="nav-link nav-internal" href="../../../index.html">
|
||
Home
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="nav-item ">
|
||
<a class="nav-link nav-internal" href="../../../manuals.html">
|
||
Manuals
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="nav-item ">
|
||
<a class="nav-link nav-internal" href="../../../quapy.html">
|
||
API
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
</nav></div>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="sidebar-header-items__end">
|
||
|
||
<div class="navbar-item">
|
||
|
||
<div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
|
||
</button>
|
||
<ul class="dropdown-menu dropdown-menu-end">
|
||
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
|
||
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
|
||
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
|
||
</ul>
|
||
</div></div>
|
||
|
||
<div class="navbar-item"><ul class="navbar-icon-links"
|
||
aria-label="Icon Links">
|
||
<li class="nav-item">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://github.com/HLT-ISTI/QuaPy" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
|
||
</li>
|
||
</ul></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
<div id="ethical-ad-placement"
|
||
class="flat"
|
||
data-ea-publisher="readthedocs"
|
||
data-ea-type="readthedocs-sidebar"
|
||
data-ea-manual="true">
|
||
</div></div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<nav aria-label="Breadcrumb" class="d-print-none">
|
||
<ul class="bd-breadcrumbs">
|
||
|
||
<li class="breadcrumb-item breadcrumb-home">
|
||
<a href="../../../index.html" class="nav-link" aria-label="Home">
|
||
<i class="fa-solid fa-home"></i>
|
||
</a>
|
||
</li>
|
||
|
||
<li class="breadcrumb-item"><a href="../../index.html" class="nav-link">Module code</a></li>
|
||
|
||
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">quapy.data.datasets</span></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<h1>Source code for quapy.data.datasets</h1><div class="highlight"><pre>
|
||
<span></span><span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">contextlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">contextmanager</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">zipfile</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">os.path</span><span class="w"> </span><span class="kn">import</span> <span class="n">join</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">pandas</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pd</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data.base</span><span class="w"> </span><span class="kn">import</span> <span class="n">Dataset</span><span class="p">,</span> <span class="n">LabelledCollection</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data.preprocessing</span><span class="w"> </span><span class="kn">import</span> <span class="n">text2tfidf</span><span class="p">,</span> <span class="n">reduce_columns</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data.preprocessing</span><span class="w"> </span><span class="kn">import</span> <span class="n">standardize</span> <span class="k">as</span> <span class="n">standardizer</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data.reader</span><span class="w"> </span><span class="kn">import</span> <span class="o">*</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.util</span><span class="w"> </span><span class="kn">import</span> <span class="n">download_file_if_not_exists</span><span class="p">,</span> <span class="n">download_file</span><span class="p">,</span> <span class="n">get_quapy_home</span><span class="p">,</span> <span class="n">pickled_resource</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">sklearn.preprocessing</span><span class="w"> </span><span class="kn">import</span> <span class="n">StandardScaler</span>
|
||
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">_fetch_ucirepo</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">ucimlrepo</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_ucirepo</span>
|
||
<span class="k">except</span> <span class="ne">ImportError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="ne">ImportError</span><span class="p">(</span>
|
||
<span class="s2">"UCI dataset fetching requires the optional 'ucimlrepo' package. "</span>
|
||
<span class="s2">"Install it to use fetch_UCIBinaryDataset or fetch_UCIMulticlassDataset."</span>
|
||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">exc</span>
|
||
<span class="k">return</span> <span class="n">fetch_ucirepo</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">REVIEWS_SENTIMENT_DATASETS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'hp'</span><span class="p">,</span> <span class="s1">'kindle'</span><span class="p">,</span> <span class="s1">'imdb'</span><span class="p">]</span>
|
||
|
||
<span class="n">TWITTER_SENTIMENT_DATASETS_TEST</span> <span class="o">=</span> <span class="p">[</span>
|
||
<span class="s1">'gasp'</span><span class="p">,</span> <span class="s1">'hcr'</span><span class="p">,</span> <span class="s1">'omd'</span><span class="p">,</span> <span class="s1">'sanders'</span><span class="p">,</span>
|
||
<span class="s1">'semeval13'</span><span class="p">,</span> <span class="s1">'semeval14'</span><span class="p">,</span> <span class="s1">'semeval15'</span><span class="p">,</span> <span class="s1">'semeval16'</span><span class="p">,</span>
|
||
<span class="s1">'sst'</span><span class="p">,</span> <span class="s1">'wa'</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">,</span>
|
||
<span class="p">]</span>
|
||
|
||
<span class="n">TWITTER_SENTIMENT_DATASETS_TRAIN</span> <span class="o">=</span> <span class="p">[</span>
|
||
<span class="s1">'gasp'</span><span class="p">,</span> <span class="s1">'hcr'</span><span class="p">,</span> <span class="s1">'omd'</span><span class="p">,</span> <span class="s1">'sanders'</span><span class="p">,</span>
|
||
<span class="s1">'semeval'</span><span class="p">,</span> <span class="s1">'semeval16'</span><span class="p">,</span>
|
||
<span class="s1">'sst'</span><span class="p">,</span> <span class="s1">'wa'</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">,</span>
|
||
<span class="p">]</span>
|
||
|
||
<span class="n">UCI_BINARY_DATASETS</span> <span class="o">=</span> <span class="p">[</span>
|
||
<span class="c1">#'acute.a', 'acute.b',</span>
|
||
<span class="s1">'balance.1'</span><span class="p">,</span>
|
||
<span class="c1">#'balance.2', </span>
|
||
<span class="s1">'balance.3'</span><span class="p">,</span>
|
||
<span class="s1">'breast-cancer'</span><span class="p">,</span>
|
||
<span class="s1">'cmc.1'</span><span class="p">,</span> <span class="s1">'cmc.2'</span><span class="p">,</span> <span class="s1">'cmc.3'</span><span class="p">,</span>
|
||
<span class="s1">'ctg.1'</span><span class="p">,</span> <span class="s1">'ctg.2'</span><span class="p">,</span> <span class="s1">'ctg.3'</span><span class="p">,</span>
|
||
<span class="c1">#'diabetes', # <-- I haven't found this one...</span>
|
||
<span class="s1">'german'</span><span class="p">,</span>
|
||
<span class="s1">'haberman'</span><span class="p">,</span>
|
||
<span class="s1">'ionosphere'</span><span class="p">,</span>
|
||
<span class="s1">'iris.1'</span><span class="p">,</span> <span class="s1">'iris.2'</span><span class="p">,</span> <span class="s1">'iris.3'</span><span class="p">,</span>
|
||
<span class="s1">'mammographic'</span><span class="p">,</span>
|
||
<span class="s1">'pageblocks.5'</span><span class="p">,</span>
|
||
<span class="c1">#'phoneme', # <-- I haven't found this one...</span>
|
||
<span class="s1">'semeion'</span><span class="p">,</span>
|
||
<span class="s1">'sonar'</span><span class="p">,</span>
|
||
<span class="s1">'spambase'</span><span class="p">,</span>
|
||
<span class="s1">'spectf'</span><span class="p">,</span>
|
||
<span class="s1">'tictactoe'</span><span class="p">,</span>
|
||
<span class="s1">'transfusion'</span><span class="p">,</span>
|
||
<span class="s1">'wdbc'</span><span class="p">,</span>
|
||
<span class="s1">'wine.1'</span><span class="p">,</span> <span class="s1">'wine.2'</span><span class="p">,</span> <span class="s1">'wine.3'</span><span class="p">,</span>
|
||
<span class="s1">'wine-q-red'</span><span class="p">,</span>
|
||
<span class="s1">'wine-q-white'</span><span class="p">,</span>
|
||
<span class="s1">'yeast'</span><span class="p">,</span>
|
||
<span class="p">]</span>
|
||
|
||
<span class="n">UCI_MULTICLASS_DATASETS</span> <span class="o">=</span> <span class="p">[</span>
|
||
<span class="s1">'dry-bean'</span><span class="p">,</span>
|
||
<span class="s1">'wine-quality'</span><span class="p">,</span>
|
||
<span class="s1">'academic-success'</span><span class="p">,</span>
|
||
<span class="s1">'digits'</span><span class="p">,</span>
|
||
<span class="s1">'letter'</span><span class="p">,</span>
|
||
<span class="s1">'abalone'</span><span class="p">,</span>
|
||
<span class="s1">'obesity'</span><span class="p">,</span>
|
||
<span class="s1">'nursery'</span><span class="p">,</span>
|
||
<span class="s1">'yeast'</span><span class="p">,</span>
|
||
<span class="s1">'hand_digits'</span><span class="p">,</span>
|
||
<span class="s1">'satellite'</span><span class="p">,</span>
|
||
<span class="s1">'shuttle'</span><span class="p">,</span>
|
||
<span class="s1">'cmc'</span><span class="p">,</span>
|
||
<span class="s1">'isolet'</span><span class="p">,</span>
|
||
<span class="s1">'waveform-v1'</span><span class="p">,</span>
|
||
<span class="s1">'molecular'</span><span class="p">,</span>
|
||
<span class="s1">'poker_hand'</span><span class="p">,</span>
|
||
<span class="s1">'connect-4'</span><span class="p">,</span>
|
||
<span class="s1">'mhr'</span><span class="p">,</span>
|
||
<span class="s1">'chess'</span><span class="p">,</span>
|
||
<span class="s1">'page_block'</span><span class="p">,</span>
|
||
<span class="s1">'phishing'</span><span class="p">,</span>
|
||
<span class="s1">'image_seg'</span><span class="p">,</span>
|
||
<span class="s1">'hcv'</span><span class="p">,</span>
|
||
<span class="p">]</span>
|
||
|
||
<span class="n">LEQUA2022_VECTOR_TASKS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'T1A'</span><span class="p">,</span> <span class="s1">'T1B'</span><span class="p">]</span>
|
||
<span class="n">LEQUA2022_TEXT_TASKS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'T2A'</span><span class="p">,</span> <span class="s1">'T2B'</span><span class="p">]</span>
|
||
<span class="n">LEQUA2022_TASKS</span> <span class="o">=</span> <span class="n">LEQUA2022_VECTOR_TASKS</span> <span class="o">+</span> <span class="n">LEQUA2022_TEXT_TASKS</span>
|
||
|
||
<span class="n">LEQUA2024_TASKS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'T1'</span><span class="p">,</span> <span class="s1">'T2'</span><span class="p">,</span> <span class="s1">'T3'</span><span class="p">,</span> <span class="s1">'T4'</span><span class="p">]</span>
|
||
|
||
<span class="n">_TXA_SAMPLE_SIZE</span> <span class="o">=</span> <span class="mi">250</span>
|
||
<span class="n">_TXB_SAMPLE_SIZE</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
|
||
<span class="n">LEQUA2022_SAMPLE_SIZE</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'TXA'</span><span class="p">:</span> <span class="n">_TXA_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'TXB'</span><span class="p">:</span> <span class="n">_TXB_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'T1A'</span><span class="p">:</span> <span class="n">_TXA_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'T1B'</span><span class="p">:</span> <span class="n">_TXB_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'T2A'</span><span class="p">:</span> <span class="n">_TXA_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'T2B'</span><span class="p">:</span> <span class="n">_TXB_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'binary'</span><span class="p">:</span> <span class="n">_TXA_SAMPLE_SIZE</span><span class="p">,</span>
|
||
<span class="s1">'multiclass'</span><span class="p">:</span> <span class="n">_TXB_SAMPLE_SIZE</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">LEQUA2024_SAMPLE_SIZE</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'T1'</span><span class="p">:</span> <span class="mi">250</span><span class="p">,</span>
|
||
<span class="s1">'T2'</span><span class="p">:</span> <span class="mi">1000</span><span class="p">,</span>
|
||
<span class="s1">'T3'</span><span class="p">:</span> <span class="mi">200</span><span class="p">,</span>
|
||
<span class="s1">'T4'</span><span class="p">:</span> <span class="mi">250</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">IMAGE_DATASETS</span><span class="o">=</span><span class="p">[</span><span class="s1">'cifar10'</span><span class="p">,</span> <span class="s1">'cifar100'</span><span class="p">,</span> <span class="s1">'cifar100coarse'</span><span class="p">,</span> <span class="s1">'svhn'</span><span class="p">,</span> <span class="s1">'fashionmnist'</span><span class="p">,</span> <span class="s1">'mnist'</span><span class="p">]</span>
|
||
<span class="n">IMAGE_EMBEDDINGS</span><span class="o">=</span><span class="p">[</span><span class="s1">'features'</span><span class="p">,</span> <span class="s1">'logits'</span><span class="p">,</span> <span class="s1">'predictions'</span><span class="p">]</span>
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_reviews">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_reviews">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_reviews</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">tfidf</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">min_df</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">pickle</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dataset</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a Reviews dataset as a Dataset instance, as used in</span>
|
||
<span class="sd"> `Esuli, A., Moreo, A., and Sebastiani, F. "A recurrent neural network for sentiment quantification."</span>
|
||
<span class="sd"> Proceedings of the 27th ACM International Conference on Information and Knowledge Management. 2018.</span>
|
||
<span class="sd"> <https://dl.acm.org/doi/abs/10.1145/3269206.3269287>`_.</span>
|
||
<span class="sd"> The list of valid dataset names can be accessed in `quapy.data.datasets.REVIEWS_SENTIMENT_DATASETS`</span>
|
||
|
||
<span class="sd"> :param dataset_name: the name of the dataset: valid ones are 'hp', 'kindle', 'imdb'</span>
|
||
<span class="sd"> :param tfidf: set to True to transform the raw documents into tfidf weighted matrices</span>
|
||
<span class="sd"> :param min_df: minimun number of documents that should contain a term in order for the term to be</span>
|
||
<span class="sd"> kept (ignored if tfidf==False)</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :param pickle: set to True to pickle the Dataset object the first time it is generated, in order to allow for</span>
|
||
<span class="sd"> faster subsequent invokations</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.Dataset` instance</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">assert</span> <span class="n">dataset_name</span> <span class="ow">in</span> <span class="n">REVIEWS_SENTIMENT_DATASETS</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Name </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1"> does not match any known dataset for sentiment reviews. '</span> \
|
||
<span class="sa">f</span><span class="s1">'Valid ones are </span><span class="si">{</span><span class="n">REVIEWS_SENTIMENT_DATASETS</span><span class="si">}</span><span class="s1">'</span>
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">URL_TRAIN</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'https://zenodo.org/record/4117827/files/</span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1">_train.txt'</span>
|
||
<span class="n">URL_TEST</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'https://zenodo.org/record/4117827/files/</span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1">_test.txt'</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'reviews'</span><span class="p">),</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">train_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'reviews'</span><span class="p">,</span> <span class="n">dataset_name</span><span class="p">,</span> <span class="s1">'train.txt'</span><span class="p">)</span>
|
||
<span class="n">test_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'reviews'</span><span class="p">,</span> <span class="n">dataset_name</span><span class="p">,</span> <span class="s1">'test.txt'</span><span class="p">)</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">URL_TRAIN</span><span class="p">,</span> <span class="n">train_path</span><span class="p">)</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">URL_TEST</span><span class="p">,</span> <span class="n">test_path</span><span class="p">)</span>
|
||
|
||
<span class="n">pickle_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="k">if</span> <span class="n">pickle</span><span class="p">:</span>
|
||
<span class="n">pickle_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'reviews'</span><span class="p">,</span> <span class="s1">'pickle'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1">.pkl'</span><span class="p">)</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickled_resource</span><span class="p">(</span><span class="n">pickle_path</span><span class="p">,</span> <span class="n">Dataset</span><span class="o">.</span><span class="n">load</span><span class="p">,</span> <span class="n">train_path</span><span class="p">,</span> <span class="n">test_path</span><span class="p">,</span> <span class="n">from_text</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">tfidf</span><span class="p">:</span>
|
||
<span class="n">text2tfidf</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">min_df</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">reduce_columns</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">min_df</span><span class="o">=</span><span class="n">min_df</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="n">data</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">dataset_name</span>
|
||
|
||
<span class="k">return</span> <span class="n">data</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_twitter">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_twitter">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_twitter</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">for_model_selection</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">min_df</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">pickle</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dataset</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a Twitter dataset as a :class:`quapy.data.base.Dataset` instance, as used in:</span>
|
||
<span class="sd"> `Gao, W., Sebastiani, F.: From classification to quantification in tweet sentiment analysis.</span>
|
||
<span class="sd"> Social Network Analysis and Mining6(19), 1–22 (2016) <https://link.springer.com/content/pdf/10.1007/s13278-016-0327-z.pdf>`_</span>
|
||
<span class="sd"> Note that the datasets 'semeval13', 'semeval14', 'semeval15' share the same training set.</span>
|
||
<span class="sd"> The list of valid dataset names corresponding to training sets can be accessed in</span>
|
||
<span class="sd"> `quapy.data.datasets.TWITTER_SENTIMENT_DATASETS_TRAIN`, while the test sets can be accessed in</span>
|
||
<span class="sd"> `quapy.data.datasets.TWITTER_SENTIMENT_DATASETS_TEST`</span>
|
||
|
||
<span class="sd"> :param dataset_name: the name of the dataset: valid ones are 'gasp', 'hcr', 'omd', 'sanders', 'semeval13',</span>
|
||
<span class="sd"> 'semeval14', 'semeval15', 'semeval16', 'sst', 'wa', 'wb'</span>
|
||
<span class="sd"> :param for_model_selection: if True, then returns the train split as the training set and the devel split</span>
|
||
<span class="sd"> as the test set; if False, then returns the train+devel split as the training set and the test set as the</span>
|
||
<span class="sd"> test set</span>
|
||
<span class="sd"> :param min_df: minimun number of documents that should contain a term in order for the term to be kept</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :param pickle: set to True to pickle the Dataset object the first time it is generated, in order to allow for</span>
|
||
<span class="sd"> faster subsequent invokations</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.Dataset` instance</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">assert</span> <span class="n">dataset_name</span> <span class="ow">in</span> <span class="n">TWITTER_SENTIMENT_DATASETS_TRAIN</span> <span class="o">+</span> <span class="n">TWITTER_SENTIMENT_DATASETS_TEST</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Name </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1"> does not match any known dataset for sentiment twitter. '</span> \
|
||
<span class="sa">f</span><span class="s1">'Valid ones are </span><span class="si">{</span><span class="n">TWITTER_SENTIMENT_DATASETS_TRAIN</span><span class="si">}</span><span class="s1"> for model selection and '</span> \
|
||
<span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">TWITTER_SENTIMENT_DATASETS_TEST</span><span class="si">}</span><span class="s1"> for test (datasets "semeval14", "semeval15", "semeval16" share '</span> \
|
||
<span class="sa">f</span><span class="s1">'a common training set "semeval")'</span>
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">URL</span> <span class="o">=</span> <span class="s1">'https://zenodo.org/record/4255764/files/tweet_sentiment_quantification_snam.zip'</span>
|
||
<span class="n">unzipped_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'tweet_sentiment_quantification_snam'</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">):</span>
|
||
<span class="n">downloaded_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'tweet_sentiment_quantification_snam.zip'</span><span class="p">)</span>
|
||
<span class="n">download_file</span><span class="p">(</span><span class="n">URL</span><span class="p">,</span> <span class="n">downloaded_path</span><span class="p">)</span>
|
||
<span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">downloaded_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">file</span><span class="p">:</span>
|
||
<span class="n">file</span><span class="o">.</span><span class="n">extractall</span><span class="p">(</span><span class="n">data_home</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">downloaded_path</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">dataset_name</span> <span class="ow">in</span> <span class="p">{</span><span class="s1">'semeval13'</span><span class="p">,</span> <span class="s1">'semeval14'</span><span class="p">,</span> <span class="s1">'semeval15'</span><span class="p">}:</span>
|
||
<span class="n">trainset_name</span> <span class="o">=</span> <span class="s1">'semeval'</span>
|
||
<span class="n">testset_name</span> <span class="o">=</span> <span class="s1">'semeval'</span> <span class="k">if</span> <span class="n">for_model_selection</span> <span class="k">else</span> <span class="n">dataset_name</span>
|
||
<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||
<span class="sa">f</span><span class="s2">"the training and development sets for datasets 'semeval13', 'semeval14', 'semeval15' are common "</span>
|
||
<span class="sa">f</span><span class="s2">"(called 'semeval'); returning trainin-set='</span><span class="si">{</span><span class="n">trainset_name</span><span class="si">}</span><span class="s2">' and test-set=</span><span class="si">{</span><span class="n">testset_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="n">dataset_name</span> <span class="o">==</span> <span class="s1">'semeval'</span> <span class="ow">and</span> <span class="n">for_model_selection</span><span class="o">==</span><span class="kc">False</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'dataset "semeval" can only be used for model selection. '</span>
|
||
<span class="s1">'Use "semeval13", "semeval14", or "semeval15" for model evaluation.'</span><span class="p">)</span>
|
||
<span class="n">trainset_name</span> <span class="o">=</span> <span class="n">testset_name</span> <span class="o">=</span> <span class="n">dataset_name</span>
|
||
|
||
<span class="k">if</span> <span class="n">for_model_selection</span><span class="p">:</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="s1">'train'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">trainset_name</span><span class="si">}</span><span class="s1">.train.feature.txt'</span><span class="p">)</span>
|
||
<span class="n">test</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="s1">'test'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">testset_name</span><span class="si">}</span><span class="s1">.dev.feature.txt'</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="s1">'train'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">trainset_name</span><span class="si">}</span><span class="s1">.train+dev.feature.txt'</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">dataset_name</span> <span class="o">==</span> <span class="s1">'semeval16'</span><span class="p">:</span> <span class="c1"># there is a different test name in the case of semeval16 only</span>
|
||
<span class="n">test</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="s1">'test'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">testset_name</span><span class="si">}</span><span class="s1">.dev-test.feature.txt'</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">test</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="s1">'test'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">testset_name</span><span class="si">}</span><span class="s1">.test.feature.txt'</span><span class="p">)</span>
|
||
|
||
<span class="n">pickle_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="k">if</span> <span class="n">pickle</span><span class="p">:</span>
|
||
<span class="n">mode</span> <span class="o">=</span> <span class="s2">"train-dev"</span> <span class="k">if</span> <span class="n">for_model_selection</span> <span class="k">else</span> <span class="s2">"train+dev-test"</span>
|
||
<span class="n">pickle_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="s1">'pickle'</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">testset_name</span><span class="si">}</span><span class="s1">.</span><span class="si">{</span><span class="n">mode</span><span class="si">}</span><span class="s1">.pkl'</span><span class="p">)</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickled_resource</span><span class="p">(</span><span class="n">pickle_path</span><span class="p">,</span> <span class="n">Dataset</span><span class="o">.</span><span class="n">load</span><span class="p">,</span> <span class="n">train</span><span class="p">,</span> <span class="n">test</span><span class="p">,</span> <span class="n">from_sparse</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">min_df</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">reduce_columns</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">min_df</span><span class="o">=</span><span class="n">min_df</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="n">data</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">dataset_name</span>
|
||
|
||
<span class="k">return</span> <span class="n">data</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_UCIBinaryDataset">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_UCIBinaryDataset">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_UCIBinaryDataset</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">test_split</span><span class="o">=</span><span class="mf">0.3</span><span class="p">,</span> <span class="n">standardize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dataset</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a UCI dataset as an instance of :class:`quapy.data.base.Dataset`, as used in</span>
|
||
<span class="sd"> `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017).</span>
|
||
<span class="sd"> Using ensembles for problems with characterizable changes in data distribution: A case study on quantification.</span>
|
||
<span class="sd"> Information Fusion, 34, 87-100. <https://www.sciencedirect.com/science/article/pii/S1566253516300628>`_</span>
|
||
<span class="sd"> and</span>
|
||
<span class="sd"> `Pérez-Gállego, P., Castano, A., Quevedo, J. R., & del Coz, J. J. (2019).</span>
|
||
<span class="sd"> Dynamic ensemble selection for quantification tasks.</span>
|
||
<span class="sd"> Information Fusion, 45, 1-15. <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.</span>
|
||
<span class="sd"> The datasets do not come with a predefined train-test split (see :meth:`fetch_UCILabelledCollection` for further</span>
|
||
<span class="sd"> information on how to use these collections), and so a train-test split is generated at desired proportion.</span>
|
||
<span class="sd"> The list of valid dataset names can be accessed in `quapy.data.datasets.UCI_DATASETS`</span>
|
||
|
||
<span class="sd"> :param dataset_name: a dataset name</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :param test_split: proportion of documents to be included in the test set. The rest conforms the training set</span>
|
||
<span class="sd"> :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested,</span>
|
||
<span class="sd"> standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the</span>
|
||
<span class="sd"> training portion of the data.</span>
|
||
<span class="sd"> :param verbose: set to True (default is False) to get information (from the UCI ML repository) about the datasets</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.Dataset` instance</span>
|
||
<span class="sd"> """</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">fetch_UCIBinaryLabelledCollection</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">data_home</span><span class="p">,</span> <span class="n">verbose</span><span class="p">)</span>
|
||
<span class="n">dataset</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="o">*</span><span class="n">data</span><span class="o">.</span><span class="n">split_stratified</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">test_split</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="n">dataset_name</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">standardize</span><span class="p">:</span>
|
||
<span class="n">dataset</span> <span class="o">=</span> <span class="n">standardizer</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">dataset</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_UCIBinaryLabelledCollection">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_UCIBinaryLabelledCollection">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_UCIBinaryLabelledCollection</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">standardize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">LabelledCollection</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a UCI collection as an instance of :class:`quapy.data.base.LabelledCollection`, as used in</span>
|
||
<span class="sd"> `Pérez-Gállego, P., Quevedo, J. R., & del Coz, J. J. (2017).</span>
|
||
<span class="sd"> Using ensembles for problems with characterizable changes in data distribution: A case study on quantification.</span>
|
||
<span class="sd"> Information Fusion, 34, 87-100. <https://www.sciencedirect.com/science/article/pii/S1566253516300628>`_</span>
|
||
<span class="sd"> and</span>
|
||
<span class="sd"> `Pérez-Gállego, P., Castano, A., Quevedo, J. R., & del Coz, J. J. (2019).</span>
|
||
<span class="sd"> Dynamic ensemble selection for quantification tasks.</span>
|
||
<span class="sd"> Information Fusion, 45, 1-15. <https://www.sciencedirect.com/science/article/pii/S1566253517303652>`_.</span>
|
||
<span class="sd"> The datasets do not come with a predefined train-test split, and so Pérez-Gállego et al. adopted a 5FCVx2 evaluation</span>
|
||
<span class="sd"> protocol, meaning that each collection was used to generate two rounds (hence the x2) of 5 fold cross validation.</span>
|
||
<span class="sd"> This can be reproduced by using :meth:`quapy.data.base.Dataset.kFCV`, e.g.:</span>
|
||
|
||
<span class="sd"> >>> import quapy as qp</span>
|
||
<span class="sd"> >>> collection = qp.datasets.fetch_UCIBinaryLabelledCollection("yeast")</span>
|
||
<span class="sd"> >>> for data in qp.datasets.Dataset.kFCV(collection, nfolds=5, nrepeats=2):</span>
|
||
<span class="sd"> >>> ...</span>
|
||
|
||
<span class="sd"> The list of valid dataset names can be accessed in `quapy.data.datasets.UCI_DATASETS`</span>
|
||
|
||
<span class="sd"> :param dataset_name: a dataset name</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :param standardize: indicates whether the covariates should be standardized or not (default is True). </span>
|
||
<span class="sd"> :param verbose: set to True (default is False) to get information (from the UCI ML repository) about the datasets</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.LabelledCollection` instance</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">assert</span> <span class="n">dataset_name</span> <span class="ow">in</span> <span class="n">UCI_BINARY_DATASETS</span><span class="p">,</span> <span class="p">(</span>
|
||
<span class="sa">f</span><span class="s2">"Name </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s2"> does not match any known dataset from the UCI Machine Learning datasets repository. "</span>
|
||
<span class="sa">f</span><span class="s2">"Valid ones are </span><span class="si">{</span><span class="n">UCI_BINARY_DATASETS</span><span class="si">}</span><span class="s2">"</span>
|
||
<span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="c1"># mapping bewteen dataset names and UCI api ids</span>
|
||
<span class="n">identifiers</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s2">"acute.a"</span><span class="p">:</span> <span class="mi">184</span><span class="p">,</span>
|
||
<span class="s2">"acute.b"</span><span class="p">:</span> <span class="mi">184</span><span class="p">,</span>
|
||
<span class="s2">"balance.1"</span><span class="p">:</span> <span class="mi">12</span><span class="p">,</span>
|
||
<span class="s2">"balance.2"</span><span class="p">:</span> <span class="mi">12</span><span class="p">,</span>
|
||
<span class="s2">"balance.3"</span><span class="p">:</span> <span class="mi">12</span><span class="p">,</span>
|
||
<span class="s2">"breast-cancer"</span><span class="p">:</span> <span class="mi">15</span><span class="p">,</span>
|
||
<span class="s2">"cmc.1"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
|
||
<span class="s2">"cmc.2"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
|
||
<span class="s2">"cmc.3"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
|
||
<span class="c1"># "ctg.1": , # not python importable</span>
|
||
<span class="c1"># "ctg.2": , # not python importable</span>
|
||
<span class="c1"># "ctg.3": , # not python importable</span>
|
||
<span class="c1"># "german": , # not python importable</span>
|
||
<span class="s2">"haberman"</span><span class="p">:</span> <span class="mi">43</span><span class="p">,</span>
|
||
<span class="s2">"ionosphere"</span><span class="p">:</span> <span class="mi">52</span><span class="p">,</span>
|
||
<span class="s2">"iris.1"</span><span class="p">:</span> <span class="mi">53</span><span class="p">,</span>
|
||
<span class="s2">"iris.2"</span><span class="p">:</span> <span class="mi">53</span><span class="p">,</span>
|
||
<span class="s2">"iris.3"</span><span class="p">:</span> <span class="mi">53</span><span class="p">,</span>
|
||
<span class="s2">"mammographic"</span><span class="p">:</span> <span class="mi">161</span><span class="p">,</span>
|
||
<span class="s2">"pageblocks.5"</span><span class="p">:</span> <span class="mi">78</span><span class="p">,</span>
|
||
<span class="c1"># "semeion": , # not python importable</span>
|
||
<span class="s2">"sonar"</span><span class="p">:</span> <span class="mi">151</span><span class="p">,</span>
|
||
<span class="s2">"spambase"</span><span class="p">:</span> <span class="mi">94</span><span class="p">,</span>
|
||
<span class="s2">"spectf"</span><span class="p">:</span> <span class="mi">96</span><span class="p">,</span>
|
||
<span class="s2">"tictactoe"</span><span class="p">:</span> <span class="mi">101</span><span class="p">,</span>
|
||
<span class="s2">"transfusion"</span><span class="p">:</span> <span class="mi">176</span><span class="p">,</span>
|
||
<span class="s2">"wdbc"</span><span class="p">:</span> <span class="mi">17</span><span class="p">,</span>
|
||
<span class="s2">"wine.1"</span><span class="p">:</span> <span class="mi">109</span><span class="p">,</span>
|
||
<span class="s2">"wine.2"</span><span class="p">:</span> <span class="mi">109</span><span class="p">,</span>
|
||
<span class="s2">"wine.3"</span><span class="p">:</span> <span class="mi">109</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-red"</span><span class="p">:</span> <span class="mi">186</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-white"</span><span class="p">:</span> <span class="mi">186</span><span class="p">,</span>
|
||
<span class="s2">"yeast"</span><span class="p">:</span> <span class="mi">110</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="c1"># mapping between dataset names and dataset groups</span>
|
||
<span class="n">groups</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s2">"acute.a"</span><span class="p">:</span> <span class="s2">"acute"</span><span class="p">,</span>
|
||
<span class="s2">"acute.b"</span><span class="p">:</span> <span class="s2">"acute"</span><span class="p">,</span>
|
||
<span class="s2">"balance.1"</span><span class="p">:</span> <span class="s2">"balance"</span><span class="p">,</span>
|
||
<span class="s2">"balance.2"</span><span class="p">:</span> <span class="s2">"balance"</span><span class="p">,</span>
|
||
<span class="s2">"balance.3"</span><span class="p">:</span> <span class="s2">"balance"</span><span class="p">,</span>
|
||
<span class="s2">"breast-cancer"</span><span class="p">:</span> <span class="s2">"breast-cancer"</span><span class="p">,</span>
|
||
<span class="s2">"cmc.1"</span><span class="p">:</span> <span class="s2">"cmc"</span><span class="p">,</span>
|
||
<span class="s2">"cmc.2"</span><span class="p">:</span> <span class="s2">"cmc"</span><span class="p">,</span>
|
||
<span class="s2">"cmc.3"</span><span class="p">:</span> <span class="s2">"cmc"</span><span class="p">,</span>
|
||
<span class="s2">"ctg.1"</span><span class="p">:</span> <span class="s2">"ctg"</span><span class="p">,</span>
|
||
<span class="s2">"ctg.2"</span><span class="p">:</span> <span class="s2">"ctg"</span><span class="p">,</span>
|
||
<span class="s2">"ctg.3"</span><span class="p">:</span> <span class="s2">"ctg"</span><span class="p">,</span>
|
||
<span class="s2">"german"</span><span class="p">:</span> <span class="s2">"german"</span><span class="p">,</span>
|
||
<span class="s2">"haberman"</span><span class="p">:</span> <span class="s2">"haberman"</span><span class="p">,</span>
|
||
<span class="s2">"ionosphere"</span><span class="p">:</span> <span class="s2">"ionosphere"</span><span class="p">,</span>
|
||
<span class="s2">"iris.1"</span><span class="p">:</span> <span class="s2">"iris"</span><span class="p">,</span>
|
||
<span class="s2">"iris.2"</span><span class="p">:</span> <span class="s2">"iris"</span><span class="p">,</span>
|
||
<span class="s2">"iris.3"</span><span class="p">:</span> <span class="s2">"iris"</span><span class="p">,</span>
|
||
<span class="s2">"mammographic"</span><span class="p">:</span> <span class="s2">"mammographic"</span><span class="p">,</span>
|
||
<span class="s2">"pageblocks.5"</span><span class="p">:</span> <span class="s2">"pageblocks"</span><span class="p">,</span>
|
||
<span class="s2">"semeion"</span><span class="p">:</span> <span class="s2">"semeion"</span><span class="p">,</span>
|
||
<span class="s2">"sonar"</span><span class="p">:</span> <span class="s2">"sonar"</span><span class="p">,</span>
|
||
<span class="s2">"spambase"</span><span class="p">:</span> <span class="s2">"spambase"</span><span class="p">,</span>
|
||
<span class="s2">"spectf"</span><span class="p">:</span> <span class="s2">"spectf"</span><span class="p">,</span>
|
||
<span class="s2">"tictactoe"</span><span class="p">:</span> <span class="s2">"tictactoe"</span><span class="p">,</span>
|
||
<span class="s2">"transfusion"</span><span class="p">:</span> <span class="s2">"transfusion"</span><span class="p">,</span>
|
||
<span class="s2">"wdbc"</span><span class="p">:</span> <span class="s2">"wdbc"</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-red"</span><span class="p">:</span> <span class="s2">"wine-quality"</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-white"</span><span class="p">:</span> <span class="s2">"wine-quality"</span><span class="p">,</span>
|
||
<span class="s2">"wine.1"</span><span class="p">:</span> <span class="s2">"wine"</span><span class="p">,</span>
|
||
<span class="s2">"wine.2"</span><span class="p">:</span> <span class="s2">"wine"</span><span class="p">,</span>
|
||
<span class="s2">"wine.3"</span><span class="p">:</span> <span class="s2">"wine"</span><span class="p">,</span>
|
||
<span class="s2">"yeast"</span><span class="p">:</span> <span class="s2">"yeast"</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="c1"># mapping between dataset short names and full names</span>
|
||
<span class="n">full_names</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s2">"acute.a"</span><span class="p">:</span> <span class="s2">"Acute Inflammations (urinary bladder)"</span><span class="p">,</span>
|
||
<span class="s2">"acute.b"</span><span class="p">:</span> <span class="s2">"Acute Inflammations (renal pelvis)"</span><span class="p">,</span>
|
||
<span class="s2">"balance.1"</span><span class="p">:</span> <span class="s2">"Balance Scale Weight & Distance Database (left)"</span><span class="p">,</span>
|
||
<span class="s2">"balance.2"</span><span class="p">:</span> <span class="s2">"Balance Scale Weight & Distance Database (balanced)"</span><span class="p">,</span>
|
||
<span class="s2">"balance.3"</span><span class="p">:</span> <span class="s2">"Balance Scale Weight & Distance Database (right)"</span><span class="p">,</span>
|
||
<span class="s2">"breast-cancer"</span><span class="p">:</span> <span class="s2">"Breast Cancer Wisconsin (Original)"</span><span class="p">,</span>
|
||
<span class="s2">"cmc.1"</span><span class="p">:</span> <span class="s2">"Contraceptive Method Choice (no use)"</span><span class="p">,</span>
|
||
<span class="s2">"cmc.2"</span><span class="p">:</span> <span class="s2">"Contraceptive Method Choice (long term)"</span><span class="p">,</span>
|
||
<span class="s2">"cmc.3"</span><span class="p">:</span> <span class="s2">"Contraceptive Method Choice (short term)"</span><span class="p">,</span>
|
||
<span class="s2">"ctg.1"</span><span class="p">:</span> <span class="s2">"Cardiotocography Data Set (normal)"</span><span class="p">,</span>
|
||
<span class="s2">"ctg.2"</span><span class="p">:</span> <span class="s2">"Cardiotocography Data Set (suspect)"</span><span class="p">,</span>
|
||
<span class="s2">"ctg.3"</span><span class="p">:</span> <span class="s2">"Cardiotocography Data Set (pathologic)"</span><span class="p">,</span>
|
||
<span class="s2">"german"</span><span class="p">:</span> <span class="s2">"Statlog German Credit Data"</span><span class="p">,</span>
|
||
<span class="s2">"haberman"</span><span class="p">:</span> <span class="s2">"Haberman's Survival Data"</span><span class="p">,</span>
|
||
<span class="s2">"ionosphere"</span><span class="p">:</span> <span class="s2">"Johns Hopkins University Ionosphere DB"</span><span class="p">,</span>
|
||
<span class="s2">"iris.1"</span><span class="p">:</span> <span class="s2">"Iris Plants Database(x)"</span><span class="p">,</span>
|
||
<span class="s2">"iris.2"</span><span class="p">:</span> <span class="s2">"Iris Plants Database(versicolour)"</span><span class="p">,</span>
|
||
<span class="s2">"iris.3"</span><span class="p">:</span> <span class="s2">"Iris Plants Database(virginica)"</span><span class="p">,</span>
|
||
<span class="s2">"mammographic"</span><span class="p">:</span> <span class="s2">"Mammographic Mass"</span><span class="p">,</span>
|
||
<span class="s2">"pageblocks.5"</span><span class="p">:</span> <span class="s2">"Page Blocks Classification (5)"</span><span class="p">,</span>
|
||
<span class="s2">"semeion"</span><span class="p">:</span> <span class="s2">"Semeion Handwritten Digit (8)"</span><span class="p">,</span>
|
||
<span class="s2">"sonar"</span><span class="p">:</span> <span class="s2">"Sonar, Mines vs. Rocks"</span><span class="p">,</span>
|
||
<span class="s2">"spambase"</span><span class="p">:</span> <span class="s2">"Spambase Data Set"</span><span class="p">,</span>
|
||
<span class="s2">"spectf"</span><span class="p">:</span> <span class="s2">"SPECTF Heart Data"</span><span class="p">,</span>
|
||
<span class="s2">"tictactoe"</span><span class="p">:</span> <span class="s2">"Tic-Tac-Toe Endgame Database"</span><span class="p">,</span>
|
||
<span class="s2">"transfusion"</span><span class="p">:</span> <span class="s2">"Blood Transfusion Service Center Data Set"</span><span class="p">,</span>
|
||
<span class="s2">"wdbc"</span><span class="p">:</span> <span class="s2">"Wisconsin Diagnostic Breast Cancer"</span><span class="p">,</span>
|
||
<span class="s2">"wine.1"</span><span class="p">:</span> <span class="s2">"Wine Recognition Data (1)"</span><span class="p">,</span>
|
||
<span class="s2">"wine.2"</span><span class="p">:</span> <span class="s2">"Wine Recognition Data (2)"</span><span class="p">,</span>
|
||
<span class="s2">"wine.3"</span><span class="p">:</span> <span class="s2">"Wine Recognition Data (3)"</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-red"</span><span class="p">:</span> <span class="s2">"Wine Quality Red (6-10)"</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-white"</span><span class="p">:</span> <span class="s2">"Wine Quality White (6-10)"</span><span class="p">,</span>
|
||
<span class="s2">"yeast"</span><span class="p">:</span> <span class="s2">"Yeast"</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="c1"># mapping between dataset names and values of positive class</span>
|
||
<span class="n">pos_class</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s2">"acute.a"</span><span class="p">:</span> <span class="s2">"yes"</span><span class="p">,</span>
|
||
<span class="s2">"acute.b"</span><span class="p">:</span> <span class="s2">"yes"</span><span class="p">,</span>
|
||
<span class="s2">"balance.1"</span><span class="p">:</span> <span class="s2">"L"</span><span class="p">,</span>
|
||
<span class="s2">"balance.2"</span><span class="p">:</span> <span class="s2">"B"</span><span class="p">,</span>
|
||
<span class="s2">"balance.3"</span><span class="p">:</span> <span class="s2">"R"</span><span class="p">,</span>
|
||
<span class="s2">"breast-cancer"</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
|
||
<span class="s2">"cmc.1"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"cmc.2"</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
|
||
<span class="s2">"cmc.3"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
|
||
<span class="s2">"ctg.1"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="c1"># 1==Normal</span>
|
||
<span class="s2">"ctg.2"</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="c1"># 2==Suspect</span>
|
||
<span class="s2">"ctg.3"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="c1"># 3==Pathologic</span>
|
||
<span class="s2">"german"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"haberman"</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
|
||
<span class="s2">"ionosphere"</span><span class="p">:</span> <span class="s2">"b"</span><span class="p">,</span>
|
||
<span class="s2">"iris.1"</span><span class="p">:</span> <span class="s2">"Iris-setosa"</span><span class="p">,</span> <span class="c1"># 1==Setosa</span>
|
||
<span class="s2">"iris.2"</span><span class="p">:</span> <span class="s2">"Iris-versicolor"</span><span class="p">,</span> <span class="c1"># 2==Versicolor</span>
|
||
<span class="s2">"iris.3"</span><span class="p">:</span> <span class="s2">"Iris-virginica"</span><span class="p">,</span> <span class="c1"># 3==Virginica</span>
|
||
<span class="s2">"mammographic"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"pageblocks.5"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="c1"># 5==block "graphic"</span>
|
||
<span class="s2">"semeion"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"sonar"</span><span class="p">:</span> <span class="s2">"R"</span><span class="p">,</span>
|
||
<span class="s2">"spambase"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"spectf"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
|
||
<span class="s2">"tictactoe"</span><span class="p">:</span> <span class="s2">"negative"</span><span class="p">,</span>
|
||
<span class="s2">"transfusion"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"wdbc"</span><span class="p">:</span> <span class="s2">"M"</span><span class="p">,</span>
|
||
<span class="s2">"wine.1"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"wine.2"</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
|
||
<span class="s2">"wine.3"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-red"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"wine-q-white"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s2">"yeast"</span><span class="p">:</span> <span class="s2">"NUC"</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">identifier</span> <span class="o">=</span> <span class="n">identifiers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||
<span class="n">dataset_group</span> <span class="o">=</span> <span class="n">groups</span><span class="p">[</span><span class="n">dataset_name</span><span class="p">]</span>
|
||
<span class="n">fullname</span> <span class="o">=</span> <span class="n">full_names</span><span class="p">[</span><span class="n">dataset_name</span><span class="p">]</span>
|
||
|
||
<span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Loading UCI Binary </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">fullname</span><span class="si">}</span><span class="s2">)"</span><span class="p">)</span>
|
||
|
||
<span class="n">file</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s2">"uci_datasets"</span><span class="p">,</span> <span class="n">dataset_group</span> <span class="o">+</span> <span class="s2">".pkl"</span><span class="p">)</span>
|
||
|
||
<span class="nd">@contextmanager</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download_tmp_file</span><span class="p">(</span><span class="n">url_group</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Download a data file for a group of datasets temporarely.</span>
|
||
<span class="sd"> When used as a context, the file is removed once the context exits.</span>
|
||
|
||
<span class="sd"> :param url_group: identifier of the dataset group in the URL</span>
|
||
<span class="sd"> :param filename: name of the file to be downloaded</span>
|
||
<span class="sd"> """</span>
|
||
<span class="n">data_dir</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s2">"uci_datasets"</span><span class="p">,</span> <span class="s2">"tmp"</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">data_dir</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">data_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_dir</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
|
||
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"http://archive.ics.uci.edu/ml/machine-learning-databases/</span><span class="si">{</span><span class="n">url_group</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">filename</span><span class="si">}</span><span class="s2">"</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">data_path</span><span class="p">)</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="k">yield</span> <span class="n">data_path</span>
|
||
<span class="k">finally</span><span class="p">:</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">data_path</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download</span><span class="p">(</span><span class="nb">id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">group</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Download the data to be pickled for a dataset group. Use the `fetch_ucirepo` api when possible.</span>
|
||
|
||
<span class="sd"> :param id: numeric identifier for the group; can be None</span>
|
||
<span class="sd"> :param group: group name</span>
|
||
<span class="sd"> :return: a dictionary with X and y as keys and, optionally, extra data.</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="c1"># use the fetch_ucirepo api, when possible, to download data</span>
|
||
<span class="c1"># fall back to direct download when needed</span>
|
||
<span class="k">if</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"german"</span><span class="p">:</span>
|
||
<span class="k">with</span> <span class="n">download_tmp_file</span><span class="p">(</span><span class="s2">"statlog/german"</span><span class="p">,</span> <span class="s2">"german.data-numeric"</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">tmp</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delim_whitespace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">:</span><span class="mi">24</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">float</span><span class="p">)</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="n">df</span><span class="p">[</span><span class="mi">24</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span><span class="o">.</span><span class="n">values</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"ctg"</span><span class="p">:</span>
|
||
<span class="k">with</span> <span class="n">download_tmp_file</span><span class="p">(</span><span class="s2">"00193"</span><span class="p">,</span> <span class="s2">"CTG.xls"</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_excel</span><span class="p">(</span><span class="n">tmp</span><span class="p">,</span> <span class="n">sheet_name</span><span class="o">=</span><span class="s2">"Data"</span><span class="p">,</span> <span class="n">skipfooter</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">24</span><span class="p">))]</span> <span class="c1"># select columns numbered (number 23 is the target label)</span>
|
||
<span class="c1"># replaces the header with the first row</span>
|
||
<span class="n">new_header</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># grab the first row for the header</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span> <span class="c1"># take the data less the header row</span>
|
||
<span class="n">df</span><span class="o">.</span><span class="n">columns</span> <span class="o">=</span> <span class="n">new_header</span> <span class="c1"># set the header row as the df header</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">:</span><span class="mi">21</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">float</span><span class="p">)</span><span class="o">.</span><span class="n">values</span> <span class="c1"># column 21 is skipped, it is a class column</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s2">"NSP"</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span><span class="o">.</span><span class="n">values</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"semeion"</span><span class="p">:</span>
|
||
<span class="k">with</span> <span class="n">download_tmp_file</span><span class="p">(</span><span class="s2">"semeion"</span><span class="p">,</span> <span class="s2">"semeion.data"</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">tmp</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'</span><span class="se">\\</span><span class="s1">s+'</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">:</span><span class="mi">256</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">float</span><span class="p">)</span><span class="o">.</span><span class="n">values</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="mi">263</span><span class="p">]</span><span class="o">.</span><span class="n">values</span> <span class="c1"># 263 stands for digit 8 (labels are one-hot vectors from col 256-266)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">_fetch_ucirepo</span><span class="p">(</span><span class="nb">id</span><span class="o">=</span><span class="nb">id</span><span class="p">)</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">features</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">(),</span> <span class="n">df</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">targets</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span>
|
||
|
||
<span class="c1"># transform data when needed before returning (returned data will be pickled)</span>
|
||
<span class="k">if</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"acute"</span><span class="p">:</span>
|
||
<span class="n">_array_replace</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">y</span><span class="p">}</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"balance"</span><span class="p">:</span>
|
||
<span class="c1"># features' order is reversed to match data retrieved via direct download</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">X</span><span class="p">[:,</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])[::</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">y</span><span class="p">}</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"breast-cancer"</span><span class="p">:</span>
|
||
<span class="c1"># remove rows with nan values</span>
|
||
<span class="n">Xy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">([</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">[:,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]])</span>
|
||
<span class="n">nan_rows</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">isnan</span><span class="p">(</span><span class="n">Xy</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span>
|
||
<span class="n">Xy</span> <span class="o">=</span> <span class="n">Xy</span><span class="p">[</span><span class="o">~</span><span class="n">nan_rows</span><span class="p">]</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">Xy</span><span class="p">[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">Xy</span><span class="p">[:,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]}</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"mammographic"</span><span class="p">:</span>
|
||
<span class="c1"># remove rows with nan values</span>
|
||
<span class="n">Xy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">([</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">[:,</span> <span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]])</span>
|
||
<span class="n">nan_rows</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">isnan</span><span class="p">(</span><span class="n">Xy</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span>
|
||
<span class="n">Xy</span> <span class="o">=</span> <span class="n">Xy</span><span class="p">[</span><span class="o">~</span><span class="n">nan_rows</span><span class="p">]</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">Xy</span><span class="p">[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">Xy</span><span class="p">[:,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]}</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"tictactoe"</span><span class="p">:</span>
|
||
<span class="n">_array_replace</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">repl</span><span class="o">=</span><span class="p">{</span><span class="s2">"o"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"x"</span><span class="p">:</span> <span class="mi">2</span><span class="p">})</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">y</span><span class="p">}</span>
|
||
<span class="k">elif</span> <span class="n">group</span> <span class="o">==</span> <span class="s2">"wine-quality"</span><span class="p">:</span>
|
||
<span class="c1"># add color data to split the final datasets</span>
|
||
<span class="n">color</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">original</span><span class="p">[</span><span class="s2">"color"</span><span class="p">]</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">y</span><span class="p">,</span> <span class="s2">"color"</span><span class="p">:</span> <span class="n">color</span><span class="p">}</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"X"</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">:</span> <span class="n">y</span><span class="p">}</span>
|
||
|
||
<span class="k">return</span> <span class="n">data</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">binarize_data</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">)</span> <span class="o">-></span> <span class="n">LabelledCollection</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Filter and transform data to extract a binary dataset.</span>
|
||
|
||
<span class="sd"> :param name: name of the dataset</span>
|
||
<span class="sd"> :param data: dictionary containing X and y fields, plus additional data when needed</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.LabelledCollection` with the extracted dataset</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">if</span> <span class="n">name</span> <span class="o">==</span> <span class="s2">"acute.a"</span><span class="p">:</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">"X"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"y"</span><span class="p">][:,</span> <span class="mi">0</span><span class="p">]</span>
|
||
<span class="k">elif</span> <span class="n">name</span> <span class="o">==</span> <span class="s2">"acute.b"</span><span class="p">:</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">"X"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"y"</span><span class="p">][:,</span> <span class="mi">1</span><span class="p">]</span>
|
||
<span class="k">elif</span> <span class="n">name</span> <span class="o">==</span> <span class="s2">"wine-q-red"</span><span class="p">:</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">"X"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"y"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"color"</span><span class="p">]</span>
|
||
<span class="n">red_idx</span> <span class="o">=</span> <span class="n">color</span> <span class="o">==</span> <span class="s2">"red"</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">red_idx</span><span class="p">,</span> <span class="p">:],</span> <span class="n">y</span><span class="p">[</span><span class="n">red_idx</span><span class="p">]</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="p">(</span><span class="n">y</span> <span class="o">></span> <span class="mi">5</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
|
||
<span class="k">elif</span> <span class="n">name</span> <span class="o">==</span> <span class="s2">"wine-q-white"</span><span class="p">:</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">"X"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"y"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"color"</span><span class="p">]</span>
|
||
<span class="n">white_idx</span> <span class="o">=</span> <span class="n">color</span> <span class="o">==</span> <span class="s2">"white"</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">white_idx</span><span class="p">,</span> <span class="p">:],</span> <span class="n">y</span><span class="p">[</span><span class="n">white_idx</span><span class="p">]</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="p">(</span><span class="n">y</span> <span class="o">></span> <span class="mi">5</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">"X"</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="s2">"y"</span><span class="p">]</span>
|
||
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">binarize</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">pos_class</span><span class="o">=</span><span class="n">pos_class</span><span class="p">[</span><span class="n">name</span><span class="p">])</span>
|
||
|
||
<span class="k">return</span> <span class="n">LabelledCollection</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickled_resource</span><span class="p">(</span><span class="n">file</span><span class="p">,</span> <span class="n">download</span><span class="p">,</span> <span class="n">identifier</span><span class="p">,</span> <span class="n">dataset_group</span><span class="p">)</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">binarize_data</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">standardize</span><span class="p">:</span>
|
||
<span class="n">stds</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
|
||
<span class="n">data</span><span class="o">.</span><span class="n">instances</span> <span class="o">=</span> <span class="n">stds</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">instances</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
|
||
<span class="n">data</span><span class="o">.</span><span class="n">stats</span><span class="p">()</span>
|
||
|
||
<span class="k">return</span> <span class="n">data</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_UCIMulticlassDataset">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_UCIMulticlassDataset">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_UCIMulticlassDataset</span><span class="p">(</span>
|
||
<span class="n">dataset_name</span><span class="p">,</span>
|
||
<span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||
<span class="n">min_test_split</span><span class="o">=</span><span class="mf">0.3</span><span class="p">,</span>
|
||
<span class="n">max_train_instances</span><span class="o">=</span><span class="mi">25000</span><span class="p">,</span>
|
||
<span class="n">min_class_support</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span>
|
||
<span class="n">standardize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||
<span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dataset</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a UCI multiclass dataset as an instance of :class:`quapy.data.base.Dataset`. </span>
|
||
|
||
<span class="sd"> The list of available datasets is taken from https://archive.ics.uci.edu/, following these criteria:</span>
|
||
<span class="sd"> - It has more than 1000 instances</span>
|
||
<span class="sd"> - It is suited for classification</span>
|
||
<span class="sd"> - It has more than two classes</span>
|
||
<span class="sd"> - It is available for Python import (requires ucimlrepo package)</span>
|
||
|
||
<span class="sd"> >>> import quapy as qp</span>
|
||
<span class="sd"> >>> dataset = qp.datasets.fetch_UCIMulticlassDataset("dry-bean")</span>
|
||
<span class="sd"> >>> train, test = dataset.train_test</span>
|
||
<span class="sd"> >>> ...</span>
|
||
|
||
<span class="sd"> The list of valid dataset names can be accessed in `quapy.data.datasets.UCI_MULTICLASS_DATASETS`</span>
|
||
|
||
<span class="sd"> The datasets are downloaded only once and pickled into disk, saving time for consecutive calls.</span>
|
||
|
||
<span class="sd"> :param dataset_name: a dataset name</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :param min_test_split: minimum proportion of instances to be included in the test set. This value is interpreted</span>
|
||
<span class="sd"> as a minimum proportion, meaning that the real proportion could be higher in case the training proportion</span>
|
||
<span class="sd"> (1-`min_test_split`% of the instances) surpasses `max_train_instances`. In such case, only `max_train_instances`</span>
|
||
<span class="sd"> are taken for training, and the rest (irrespective of `min_test_split`) is taken for test.</span>
|
||
<span class="sd"> :param max_train_instances: maximum number of instances to keep for training (defaults to 25000);</span>
|
||
<span class="sd"> set to -1 or None to avoid this check</span>
|
||
<span class="sd"> :param min_class_support: integer or float, the minimum number or proportion of istances per class.</span>
|
||
<span class="sd"> Classes with fewer instances are discarded (deafult is 100).</span>
|
||
<span class="sd"> :param standardize: indicates whether the covariates should be standardized or not (default is True). If requested,</span>
|
||
<span class="sd"> standardization applies after the LabelledCollection is split, that is, the mean an std are computed only on the</span>
|
||
<span class="sd"> training portion of the data.</span>
|
||
<span class="sd"> :param verbose: set to True (default is False) to get information (stats) about the dataset</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.Dataset` instance</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">fetch_UCIMulticlassLabelledCollection</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">data_home</span><span class="p">,</span> <span class="n">min_class_support</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">)</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
|
||
<span class="n">train_prop</span> <span class="o">=</span> <span class="p">(</span><span class="mf">1.</span><span class="o">-</span><span class="n">min_test_split</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="p">(</span><span class="n">max_train_instances</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">and</span> <span class="p">(</span><span class="n">max_train_instances</span> <span class="o">></span> <span class="mi">0</span><span class="p">):</span>
|
||
<span class="n">n_train</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n</span><span class="o">*</span><span class="n">train_prop</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">n_train</span> <span class="o">></span> <span class="n">max_train_instances</span><span class="p">:</span>
|
||
<span class="n">train_prop</span> <span class="o">=</span> <span class="p">(</span><span class="n">max_train_instances</span> <span class="o">/</span> <span class="n">n</span><span class="p">)</span>
|
||
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="o">*</span><span class="n">data</span><span class="o">.</span><span class="n">split_stratified</span><span class="p">(</span><span class="n">train_prop</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="n">dataset_name</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">standardize</span><span class="p">:</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">standardizer</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">data</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_UCIMulticlassLabelledCollection">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_UCIMulticlassLabelledCollection">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_UCIMulticlassLabelledCollection</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">min_class_support</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">standardize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">LabelledCollection</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a UCI multiclass collection as an instance of :class:`quapy.data.base.LabelledCollection`.</span>
|
||
|
||
<span class="sd"> The list of available datasets is taken from https://archive.ics.uci.edu/, following these criteria:</span>
|
||
<span class="sd"> - It has more than 1000 instances</span>
|
||
<span class="sd"> - It is suited for classification</span>
|
||
<span class="sd"> - It has more than two classes</span>
|
||
<span class="sd"> - It is available for Python import (requires ucimlrepo package)</span>
|
||
<span class="sd"> </span>
|
||
<span class="sd"> >>> import quapy as qp</span>
|
||
<span class="sd"> >>> collection = qp.datasets.fetch_UCIMulticlassLabelledCollection("dry-bean")</span>
|
||
<span class="sd"> >>> X, y = collection.Xy</span>
|
||
<span class="sd"> >>> ...</span>
|
||
|
||
<span class="sd"> The list of valid dataset names can be accessed in `quapy.data.datasets.UCI_MULTICLASS_DATASETS`</span>
|
||
|
||
<span class="sd"> The datasets are downloaded only once and pickled into disk, saving time for consecutive calls.</span>
|
||
|
||
<span class="sd"> :param dataset_name: a dataset name</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where the dataset will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :param min_class_support: minimum number of istances per class. Classes with fewer instances</span>
|
||
<span class="sd"> are discarded (deafult is 100)</span>
|
||
<span class="sd"> :param standardize: indicates whether the covariates should be standardized or not (default is True). </span>
|
||
<span class="sd"> :param verbose: set to True (default is False) to get information (stats) about the dataset</span>
|
||
<span class="sd"> :return: a :class:`quapy.data.base.LabelledCollection` instance</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">assert</span> <span class="n">dataset_name</span> <span class="ow">in</span> <span class="n">UCI_MULTICLASS_DATASETS</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Name </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1"> does not match any known dataset from the '</span> \
|
||
<span class="sa">f</span><span class="s1">'UCI Machine Learning datasets repository (multiclass). '</span> \
|
||
<span class="sa">f</span><span class="s1">'Valid ones are </span><span class="si">{</span><span class="n">UCI_MULTICLASS_DATASETS</span><span class="si">}</span><span class="s1">'</span>
|
||
|
||
<span class="k">assert</span> <span class="p">(</span><span class="n">min_class_support</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span>
|
||
<span class="p">((</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">min_class_support</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span> <span class="ow">and</span> <span class="n">min_class_support</span> <span class="o">>=</span> <span class="mi">0</span><span class="p">)</span> <span class="ow">or</span>
|
||
<span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">min_class_support</span><span class="p">,</span> <span class="nb">float</span><span class="p">)</span> <span class="ow">and</span> <span class="mf">0.</span> <span class="o"><=</span> <span class="n">min_class_support</span> <span class="o"><</span> <span class="mf">1.</span><span class="p">))),</span> \
|
||
<span class="sa">f</span><span class="s1">'invalid value for </span><span class="si">{</span><span class="n">min_class_support</span><span class="si">=}</span><span class="s1">; expected non negative integer or float in [0,1)'</span>
|
||
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">identifiers</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'dry-bean'</span><span class="p">:</span> <span class="mi">602</span><span class="p">,</span>
|
||
<span class="s1">'wine-quality'</span><span class="p">:</span> <span class="mi">186</span><span class="p">,</span>
|
||
<span class="s1">'academic-success'</span><span class="p">:</span> <span class="mi">697</span><span class="p">,</span>
|
||
<span class="s1">'digits'</span><span class="p">:</span> <span class="mi">80</span><span class="p">,</span>
|
||
<span class="s1">'letter'</span><span class="p">:</span> <span class="mi">59</span><span class="p">,</span>
|
||
<span class="s1">'abalone'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||
<span class="s1">'obesity'</span><span class="p">:</span> <span class="mi">544</span><span class="p">,</span>
|
||
<span class="s1">'nursery'</span><span class="p">:</span> <span class="mi">76</span><span class="p">,</span>
|
||
<span class="s1">'yeast'</span><span class="p">:</span> <span class="mi">110</span><span class="p">,</span>
|
||
<span class="s1">'hand_digits'</span><span class="p">:</span> <span class="mi">81</span><span class="p">,</span>
|
||
<span class="s1">'satellite'</span><span class="p">:</span> <span class="mi">146</span><span class="p">,</span>
|
||
<span class="s1">'shuttle'</span><span class="p">:</span> <span class="mi">148</span><span class="p">,</span>
|
||
<span class="s1">'cmc'</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
|
||
<span class="s1">'isolet'</span><span class="p">:</span> <span class="mi">54</span><span class="p">,</span>
|
||
<span class="s1">'waveform-v1'</span><span class="p">:</span> <span class="mi">107</span><span class="p">,</span>
|
||
<span class="s1">'molecular'</span><span class="p">:</span> <span class="mi">69</span><span class="p">,</span>
|
||
<span class="s1">'poker_hand'</span><span class="p">:</span> <span class="mi">158</span><span class="p">,</span>
|
||
<span class="s1">'connect-4'</span><span class="p">:</span> <span class="mi">26</span><span class="p">,</span>
|
||
<span class="s1">'mhr'</span><span class="p">:</span> <span class="mi">863</span><span class="p">,</span>
|
||
<span class="s1">'chess'</span><span class="p">:</span> <span class="mi">23</span><span class="p">,</span>
|
||
<span class="s1">'page_block'</span><span class="p">:</span> <span class="mi">78</span><span class="p">,</span>
|
||
<span class="s1">'phishing'</span><span class="p">:</span> <span class="mi">379</span><span class="p">,</span>
|
||
<span class="s1">'image_seg'</span><span class="p">:</span> <span class="mi">147</span><span class="p">,</span>
|
||
<span class="s1">'hcv'</span><span class="p">:</span> <span class="mi">503</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">full_names</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'dry-bean'</span><span class="p">:</span> <span class="s1">'Dry Bean Dataset'</span><span class="p">,</span>
|
||
<span class="s1">'wine-quality'</span><span class="p">:</span> <span class="s1">'Wine Quality'</span><span class="p">,</span>
|
||
<span class="s1">'academic-success'</span><span class="p">:</span> <span class="s1">'Predict students</span><span class="se">\'</span><span class="s1"> dropout and academic success'</span><span class="p">,</span>
|
||
<span class="s1">'digits'</span><span class="p">:</span> <span class="s1">'Optical Recognition of Handwritten Digits'</span><span class="p">,</span>
|
||
<span class="s1">'letter'</span><span class="p">:</span> <span class="s1">'Letter Recognition'</span><span class="p">,</span>
|
||
<span class="s1">'abalone'</span><span class="p">:</span> <span class="s1">'Abalone'</span><span class="p">,</span>
|
||
<span class="s1">'obesity'</span><span class="p">:</span> <span class="s1">'Estimation of Obesity Levels Based On Eating Habits and Physical Condition'</span><span class="p">,</span>
|
||
<span class="s1">'nursery'</span><span class="p">:</span> <span class="s1">'Nursery'</span><span class="p">,</span>
|
||
<span class="s1">'yeast'</span><span class="p">:</span> <span class="s1">'Yeast'</span><span class="p">,</span>
|
||
<span class="s1">'hand_digits'</span><span class="p">:</span> <span class="s1">'Pen-Based Recognition of Handwritten Digits'</span><span class="p">,</span>
|
||
<span class="s1">'satellite'</span><span class="p">:</span> <span class="s1">'Statlog Landsat Satellite'</span><span class="p">,</span>
|
||
<span class="s1">'shuttle'</span><span class="p">:</span> <span class="s1">'Statlog Shuttle'</span><span class="p">,</span>
|
||
<span class="s1">'cmc'</span><span class="p">:</span> <span class="s1">'Contraceptive Method Choice'</span><span class="p">,</span>
|
||
<span class="s1">'isolet'</span><span class="p">:</span> <span class="s1">'ISOLET'</span><span class="p">,</span>
|
||
<span class="s1">'waveform-v1'</span><span class="p">:</span> <span class="s1">'Waveform Database Generator (Version 1)'</span><span class="p">,</span>
|
||
<span class="s1">'molecular'</span><span class="p">:</span> <span class="s1">'Molecular Biology (Splice-junction Gene Sequences)'</span><span class="p">,</span>
|
||
<span class="s1">'poker_hand'</span><span class="p">:</span> <span class="s1">'Poker Hand'</span><span class="p">,</span>
|
||
<span class="s1">'connect-4'</span><span class="p">:</span> <span class="s1">'Connect-4'</span><span class="p">,</span>
|
||
<span class="s1">'mhr'</span><span class="p">:</span> <span class="s1">'Maternal Health Risk'</span><span class="p">,</span>
|
||
<span class="s1">'chess'</span><span class="p">:</span> <span class="s1">'Chess (King-Rook vs. King)'</span><span class="p">,</span>
|
||
<span class="s1">'page_block'</span><span class="p">:</span> <span class="s1">'Page Blocks Classification'</span><span class="p">,</span>
|
||
<span class="s1">'phishing'</span><span class="p">:</span> <span class="s1">'Website Phishing'</span><span class="p">,</span>
|
||
<span class="s1">'image_seg'</span><span class="p">:</span> <span class="s1">'Statlog (Image Segmentation)'</span><span class="p">,</span>
|
||
<span class="s1">'hcv'</span><span class="p">:</span> <span class="s1">'Hepatitis C Virus (HCV) for Egyptian patients'</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">identifier</span> <span class="o">=</span> <span class="n">identifiers</span><span class="p">[</span><span class="n">dataset_name</span><span class="p">]</span>
|
||
<span class="n">fullname</span> <span class="o">=</span> <span class="n">full_names</span><span class="p">[</span><span class="n">dataset_name</span><span class="p">]</span>
|
||
|
||
<span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Loading UCI Muticlass </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1"> (</span><span class="si">{</span><span class="n">fullname</span><span class="si">}</span><span class="s1">)'</span><span class="p">)</span>
|
||
|
||
<span class="n">file</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'uci_multiclass'</span><span class="p">,</span> <span class="n">dataset_name</span><span class="o">+</span><span class="s1">'.pkl'</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">dummify_categorical_features</span><span class="p">(</span><span class="n">df_features</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">):</span>
|
||
<span class="n">categorical_features</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="mi">158</span><span class="p">:</span> <span class="p">[</span><span class="s2">"S1"</span><span class="p">,</span> <span class="s2">"C1"</span><span class="p">,</span> <span class="s2">"S2"</span><span class="p">,</span> <span class="s2">"C2"</span><span class="p">,</span> <span class="s2">"S3"</span><span class="p">,</span> <span class="s2">"C3"</span><span class="p">,</span> <span class="s2">"S4"</span><span class="p">,</span> <span class="s2">"C4"</span><span class="p">,</span> <span class="s2">"S5"</span><span class="p">,</span> <span class="s2">"C5"</span><span class="p">],</span> <span class="c1"># poker_hand</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">categorical</span> <span class="o">=</span> <span class="n">categorical_features</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dataset_id</span><span class="p">,</span> <span class="p">[])</span>
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">df_features</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="k">if</span> <span class="n">categorical</span><span class="p">:</span>
|
||
<span class="n">X</span><span class="p">[</span><span class="n">categorical</span><span class="p">]</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">categorical</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">"category"</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">get_dummies</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">categorical</span><span class="p">,</span> <span class="n">drop_first</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">X</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download</span><span class="p">(</span><span class="nb">id</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">_fetch_ucirepo</span><span class="p">(</span><span class="nb">id</span><span class="o">=</span><span class="nb">id</span><span class="p">)</span>
|
||
|
||
<span class="n">X_df</span> <span class="o">=</span> <span class="n">dummify_categorical_features</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">features</span><span class="p">,</span> <span class="nb">id</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">X_df</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">targets</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span>
|
||
|
||
<span class="k">assert</span> <span class="n">y</span><span class="o">.</span><span class="n">ndim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'error: the dataset </span><span class="si">{</span><span class="nb">id</span><span class="si">=}</span><span class="s1"> </span><span class="si">{</span><span class="n">name</span><span class="si">=}</span><span class="s1"> has more than one target variable'</span>
|
||
|
||
<span class="n">classes</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">y</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">classes</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">LabelledCollection</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">filter_classes</span><span class="p">(</span><span class="n">data</span><span class="p">:</span> <span class="n">LabelledCollection</span><span class="p">,</span> <span class="n">min_class_support</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">min_class_support</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">min_class_support</span> <span class="o">==</span> <span class="mf">0.</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">data</span>
|
||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">min_class_support</span><span class="p">,</span> <span class="nb">float</span><span class="p">):</span>
|
||
<span class="n">min_class_support</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">*</span> <span class="n">min_class_support</span><span class="p">)</span>
|
||
<span class="n">classes</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">classes_</span>
|
||
<span class="c1"># restrict classes to only those with at least min_class_support instances</span>
|
||
<span class="n">classes</span> <span class="o">=</span> <span class="n">classes</span><span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">counts</span><span class="p">()</span> <span class="o">>=</span> <span class="n">min_class_support</span><span class="p">]</span>
|
||
<span class="c1"># filter X and y keeping only datapoints belonging to valid classes</span>
|
||
<span class="n">filter_idx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">y</span><span class="p">,</span> <span class="n">classes</span><span class="p">)</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">X</span><span class="p">[</span><span class="n">filter_idx</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">y</span><span class="p">[</span><span class="n">filter_idx</span><span class="p">]</span>
|
||
<span class="c1"># map classes to range(len(classes))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">classes</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">LabelledCollection</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickled_resource</span><span class="p">(</span><span class="n">file</span><span class="p">,</span> <span class="n">download</span><span class="p">,</span> <span class="n">identifier</span><span class="p">,</span> <span class="n">dataset_name</span><span class="p">)</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">filter_classes</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">min_class_support</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">data</span><span class="o">.</span><span class="n">n_classes</span> <span class="o"><=</span> <span class="mi">2</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||
<span class="sa">f</span><span class="s1">'After filtering out classes with less than </span><span class="si">{</span><span class="n">min_class_support</span><span class="si">=}</span><span class="s1"> instances, the dataset </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1"> '</span>
|
||
<span class="sa">f</span><span class="s1">'is no longer multiclass. Try a reducing this value.'</span>
|
||
<span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">standardize</span><span class="p">:</span>
|
||
<span class="n">stds</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
|
||
<span class="n">data</span><span class="o">.</span><span class="n">instances</span> <span class="o">=</span> <span class="n">stds</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">instances</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
|
||
<span class="n">data</span><span class="o">.</span><span class="n">stats</span><span class="p">()</span>
|
||
|
||
<span class="k">return</span> <span class="n">data</span></div>
|
||
|
||
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">_df_replace</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">repl</span><span class="o">=</span><span class="p">{</span><span class="s1">'yes'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'no'</span><span class="p">:</span><span class="mi">0</span><span class="p">},</span> <span class="n">astype</span><span class="o">=</span><span class="nb">float</span><span class="p">):</span>
|
||
<span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span><span class="n">repl</span><span class="p">[</span><span class="n">x</span><span class="p">])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">astype</span><span class="p">,</span> <span class="n">copy</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">_array_replace</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">repl</span><span class="o">=</span><span class="p">{</span><span class="s2">"yes"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"no"</span><span class="p">:</span> <span class="mi">0</span><span class="p">}):</span>
|
||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">repl</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||
<span class="n">arr</span><span class="p">[</span><span class="n">arr</span> <span class="o">==</span> <span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_lequa2022">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_lequa2022">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_lequa2022</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads the official datasets provided for the `LeQua 2022 <https://lequa2022.github.io/index>`_ competition.</span>
|
||
<span class="sd"> In brief, there are 4 tasks (T1A, T1B, T2A, T2B) having to do with text quantification</span>
|
||
<span class="sd"> problems. Tasks T1A and T1B provide documents in vector form, while T2A and T2B provide raw documents instead.</span>
|
||
<span class="sd"> Tasks T1A and T2A are binary sentiment quantification problems, while T2A and T2B are multiclass quantification</span>
|
||
<span class="sd"> problems consisting of estimating the class prevalence values of 28 different merchandise products.</span>
|
||
<span class="sd"> We refer to the `Esuli, A., Moreo, A., Sebastiani, F., & Sperduti, G. (2022).</span>
|
||
<span class="sd"> A Detailed Overview of LeQua@ CLEF 2022: Learning to Quantify.</span>
|
||
<span class="sd"> <https://ceur-ws.org/Vol-3180/paper-146.pdf>`_ for a detailed description</span>
|
||
<span class="sd"> on the tasks and datasets.</span>
|
||
|
||
<span class="sd"> The datasets are downloaded only once, and stored for fast reuse.</span>
|
||
|
||
<span class="sd"> See `4.lequa2022_experiments.py` provided in the example folder, that can serve as a guide on how to use these</span>
|
||
<span class="sd"> datasets.</span>
|
||
|
||
<span class="sd"> :param task: a string representing the task name; valid ones are T1A, T1B, T2A, and T2B</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :return: a tuple `(train, val_gen, test_gen)` where `train` is an instance of</span>
|
||
<span class="sd"> :class:`quapy.data.base.LabelledCollection`, `val_gen` and `test_gen` are instances of</span>
|
||
<span class="sd"> :class:`quapy.data._lequa.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,</span>
|
||
<span class="sd"> that return a series of samples stored in a directory which are labelled by prevalence.</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data._lequa</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_raw_documents</span><span class="p">,</span> <span class="n">load_vector_documents_2022</span><span class="p">,</span> <span class="n">SamplesFromDir</span>
|
||
|
||
<span class="k">assert</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">LEQUA2022_TASKS</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Unknown task </span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">. Valid ones are </span><span class="si">{</span><span class="n">LEQUA2022_TASKS</span><span class="si">}</span><span class="s1">'</span>
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">URL_TRAINDEV</span><span class="o">=</span><span class="sa">f</span><span class="s1">'https://zenodo.org/record/6546188/files/</span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">.train_dev.zip'</span>
|
||
<span class="n">URL_TEST</span><span class="o">=</span><span class="sa">f</span><span class="s1">'https://zenodo.org/record/6546188/files/</span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">.test.zip'</span>
|
||
<span class="n">URL_TEST_PREV</span><span class="o">=</span><span class="sa">f</span><span class="s1">'https://zenodo.org/record/6546188/files/</span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">.test_prevalences.zip'</span>
|
||
|
||
<span class="n">lequa_dir</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'lequa2022'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download_unzip_and_remove</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
|
||
<span class="n">tmp_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span> <span class="o">+</span> <span class="s1">'_tmp.zip'</span><span class="p">)</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">tmp_path</span><span class="p">)</span>
|
||
<span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">tmp_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">file</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Unzipping </span><span class="si">{</span><span class="n">tmp_path</span><span class="si">}</span><span class="s1">...'</span><span class="p">,</span> <span class="n">end</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span>
|
||
<span class="n">file</span><span class="o">.</span><span class="n">extractall</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'[done]'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">tmp_path</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">)):</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">URL_TRAINDEV</span><span class="p">)</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">URL_TEST</span><span class="p">)</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">URL_TEST_PREV</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">task</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'T1A'</span><span class="p">,</span> <span class="s1">'T1B'</span><span class="p">]:</span>
|
||
<span class="n">load_fn</span> <span class="o">=</span> <span class="n">load_vector_documents_2022</span>
|
||
<span class="k">elif</span> <span class="n">task</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'T2A'</span><span class="p">,</span> <span class="s1">'T2B'</span><span class="p">]:</span>
|
||
<span class="n">load_fn</span> <span class="o">=</span> <span class="n">load_raw_documents</span>
|
||
|
||
<span class="n">tr_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'training_data.txt'</span><span class="p">)</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">tr_path</span><span class="p">,</span> <span class="n">loader_func</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
|
||
<span class="n">val_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'dev_samples'</span><span class="p">)</span>
|
||
<span class="n">val_true_prev_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'dev_prevalences.txt'</span><span class="p">)</span>
|
||
<span class="n">val_gen</span> <span class="o">=</span> <span class="n">SamplesFromDir</span><span class="p">(</span><span class="n">val_samples_path</span><span class="p">,</span> <span class="n">val_true_prev_path</span><span class="p">,</span> <span class="n">load_fn</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
|
||
<span class="n">test_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'test_samples'</span><span class="p">)</span>
|
||
<span class="n">test_true_prev_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'test_prevalences.txt'</span><span class="p">)</span>
|
||
<span class="n">test_gen</span> <span class="o">=</span> <span class="n">SamplesFromDir</span><span class="p">(</span><span class="n">test_samples_path</span><span class="p">,</span> <span class="n">test_true_prev_path</span><span class="p">,</span> <span class="n">load_fn</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">train</span><span class="p">,</span> <span class="n">val_gen</span><span class="p">,</span> <span class="n">test_gen</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_lequa2024">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_lequa2024">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_lequa2024</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">merge_T3</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads the official datasets provided for the `LeQua 2024 <https://lequa2024.github.io/index>`_ competition.</span>
|
||
<span class="sd"> LeQua 2024 defines four tasks (T1, T2, T3, T4) related to the problem of quantification;</span>
|
||
<span class="sd"> all tasks are affected by some type of dataset shift. Tasks T1 and T2 are akin to tasks T1A and T1B of LeQua 2022,</span>
|
||
<span class="sd"> while T3 and T4 are new tasks introduced in LeQua 2024.</span>
|
||
|
||
<span class="sd"> - Task T1 evaluates binary quantifiers under prior probability shift (akin to T1A of LeQua 2022).</span>
|
||
<span class="sd"> - Task T2 evaluates single-label multi-class quantifiers (for n > 2 classes) under prior probability shift (akin to T1B of LeQua 2022).</span>
|
||
<span class="sd"> - Task T3 evaluates ordinal quantifiers, where the classes are totally ordered.</span>
|
||
<span class="sd"> - Task T4 also evaluates binary quantifiers, but under some mix of covariate shift and prior probability shift.</span>
|
||
|
||
<span class="sd"> For a broader discussion, we refer to the `online official documentation <https://lequa2024.github.io/tasks/>`_</span>
|
||
|
||
<span class="sd"> The datasets are downloaded only once, and stored locally for future reuse.</span>
|
||
|
||
<span class="sd"> See `4b.lequa2024_experiments.py` provided in the example folder, which can serve as a guide on how to use these</span>
|
||
<span class="sd"> datasets.</span>
|
||
|
||
<span class="sd"> :param task: a string representing the task name; valid ones are T1, T2, T3, and T4</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quapy_data/ directory)</span>
|
||
<span class="sd"> :param merge_T3: bool, if False (default), returns a generator of training collections, corresponding to natural</span>
|
||
<span class="sd"> groups of reviews; if True, returns one single :class:`quapy.data.base.LabelledCollection` representing the</span>
|
||
<span class="sd"> entire training set, as a concatenation of all the training collections</span>
|
||
<span class="sd"> :return: a tuple `(train, val_gen, test_gen)` where `train` is an instance of</span>
|
||
<span class="sd"> :class:`quapy.data.base.LabelledCollection`, `val_gen` and `test_gen` are instances of</span>
|
||
<span class="sd"> :class:`quapy.data._lequa.SamplesFromDir`, a subclass of :class:`quapy.protocol.AbstractProtocol`,</span>
|
||
<span class="sd"> that return a series of samples stored in a directory which are labelled by prevalence.</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data._lequa</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_vector_documents_2024</span><span class="p">,</span> <span class="n">SamplesFromDir</span><span class="p">,</span> <span class="n">LabelledCollectionsFromDir</span>
|
||
|
||
<span class="k">assert</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">LEQUA2024_TASKS</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Unknown task </span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">. Valid ones are </span><span class="si">{</span><span class="n">LEQUA2024_TASKS</span><span class="si">}</span><span class="s1">'</span>
|
||
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">lequa_dir</span> <span class="o">=</span> <span class="n">data_home</span>
|
||
|
||
<span class="n">LEQUA2024_ZENODO</span> <span class="o">=</span> <span class="s1">'https://zenodo.org/records/11661820'</span> <span class="c1"># v3, last one with labels</span>
|
||
|
||
<span class="n">URL_TRAINDEV</span><span class="o">=</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">LEQUA2024_ZENODO</span><span class="si">}</span><span class="s1">/files/</span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">.train_dev.zip'</span>
|
||
<span class="n">URL_TEST</span><span class="o">=</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">LEQUA2024_ZENODO</span><span class="si">}</span><span class="s1">/files/</span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">.test.zip'</span>
|
||
<span class="n">URL_TEST_PREV</span><span class="o">=</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">LEQUA2024_ZENODO</span><span class="si">}</span><span class="s1">/files/</span><span class="si">{</span><span class="n">task</span><span class="si">}</span><span class="s1">.test_prevalences.zip'</span>
|
||
|
||
<span class="n">lequa_dir</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'lequa2024'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download_unzip_and_remove</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
|
||
<span class="n">tmp_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span> <span class="o">+</span> <span class="s1">'_tmp.zip'</span><span class="p">)</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">tmp_path</span><span class="p">)</span>
|
||
<span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">tmp_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">file</span><span class="p">:</span>
|
||
<span class="n">file</span><span class="o">.</span><span class="n">extractall</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">tmp_path</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">)):</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">URL_TRAINDEV</span><span class="p">)</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">URL_TEST</span><span class="p">)</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">URL_TEST_PREV</span><span class="p">)</span>
|
||
|
||
<span class="n">load_fn</span> <span class="o">=</span> <span class="n">load_vector_documents_2024</span>
|
||
|
||
<span class="n">val_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'dev_samples'</span><span class="p">)</span>
|
||
<span class="n">val_true_prev_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'dev_prevalences.txt'</span><span class="p">)</span>
|
||
<span class="n">val_gen</span> <span class="o">=</span> <span class="n">SamplesFromDir</span><span class="p">(</span><span class="n">val_samples_path</span><span class="p">,</span> <span class="n">val_true_prev_path</span><span class="p">,</span> <span class="n">load_fn</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
|
||
<span class="n">test_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'test_samples'</span><span class="p">)</span>
|
||
<span class="n">test_true_prev_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'test_prevalences.txt'</span><span class="p">)</span>
|
||
<span class="n">test_gen</span> <span class="o">=</span> <span class="n">SamplesFromDir</span><span class="p">(</span><span class="n">test_samples_path</span><span class="p">,</span> <span class="n">test_true_prev_path</span><span class="p">,</span> <span class="n">load_fn</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">task</span> <span class="o">==</span> <span class="s1">'T3'</span><span class="p">:</span>
|
||
<span class="n">training_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'training_samples'</span><span class="p">)</span>
|
||
<span class="n">training_true_prev_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'training_prevalences.txt'</span><span class="p">)</span>
|
||
<span class="n">train_gen</span> <span class="o">=</span> <span class="n">LabelledCollectionsFromDir</span><span class="p">(</span><span class="n">training_samples_path</span><span class="p">,</span> <span class="n">training_true_prev_path</span><span class="p">,</span> <span class="n">load_fn</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">merge_T3</span><span class="p">:</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="o">*</span><span class="nb">list</span><span class="p">(</span><span class="n">train_gen</span><span class="p">()))</span>
|
||
<span class="k">return</span> <span class="n">train</span><span class="p">,</span> <span class="n">val_gen</span><span class="p">,</span> <span class="n">test_gen</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">train_gen</span><span class="p">,</span> <span class="n">val_gen</span><span class="p">,</span> <span class="n">test_gen</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">tr_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lequa_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="s1">'public'</span><span class="p">,</span> <span class="s1">'training_data.txt'</span><span class="p">)</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">tr_path</span><span class="p">,</span> <span class="n">loader_func</span><span class="o">=</span><span class="n">load_fn</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">train</span><span class="p">,</span> <span class="n">val_gen</span><span class="p">,</span> <span class="n">test_gen</span></div>
|
||
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_IFCB">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_IFCB">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_IFCB</span><span class="p">(</span><span class="n">single_sample_train</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">for_model_selection</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads the IFCB dataset for quantification from `Zenodo <https://zenodo.org/records/10036244>`_ (for more</span>
|
||
<span class="sd"> information on this dataset, please follow the zenodo link).</span>
|
||
<span class="sd"> This dataset is based on the data available publicly at</span>
|
||
<span class="sd"> `WHOI-Plankton repo <https://github.com/hsosik/WHOI-Plankton>`_.</span>
|
||
<span class="sd"> The dataset already comes with processed features.</span>
|
||
<span class="sd"> The scripts used for the processing are available at `P. González's repo <https://github.com/pglez82/IFCB_Zenodo>`_.</span>
|
||
|
||
<span class="sd"> The datasets are downloaded only once, and stored for fast reuse.</span>
|
||
|
||
<span class="sd"> :param single_sample_train: a boolean. If true, it will return the train dataset as a</span>
|
||
<span class="sd"> :class:`quapy.data.base.LabelledCollection` (all examples together).</span>
|
||
<span class="sd"> If false, a generator of training samples will be returned. Each example in the training set has an individual label.</span>
|
||
<span class="sd"> :param for_model_selection: if True, then returns a split 30% of the training set (86 out of 286 samples) to be used for model selection; </span>
|
||
<span class="sd"> if False, then returns the full training set as training set and the test set as the test set</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :return: a tuple `(train, test_gen)` where `train` is an instance of</span>
|
||
<span class="sd"> :class:`quapy.data.base.LabelledCollection`, if `single_sample_train` is true or</span>
|
||
<span class="sd"> :class:`quapy.data._ifcb.IFCBTrainSamplesFromDir`, i.e. a sampling protocol that returns a series of samples</span>
|
||
<span class="sd"> labelled example by example. test_gen will be a :class:`quapy.data._ifcb.IFCBTestSamples`, </span>
|
||
<span class="sd"> i.e., a sampling protocol that returns a series of samples labelled by prevalence.</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">quapy.data._ifcb</span><span class="w"> </span><span class="kn">import</span> <span class="n">IFCBTrainSamplesFromDir</span><span class="p">,</span> <span class="n">IFCBTestSamples</span><span class="p">,</span> <span class="n">get_sample_list</span><span class="p">,</span> <span class="n">generate_modelselection_split</span>
|
||
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">URL_TRAIN</span><span class="o">=</span><span class="sa">f</span><span class="s1">'https://zenodo.org/records/10036244/files/IFCB.train.zip'</span>
|
||
<span class="n">URL_TEST</span><span class="o">=</span><span class="sa">f</span><span class="s1">'https://zenodo.org/records/10036244/files/IFCB.test.zip'</span>
|
||
<span class="n">URL_TEST_PREV</span><span class="o">=</span><span class="sa">f</span><span class="s1">'https://zenodo.org/records/10036244/files/IFCB.test_prevalences.zip'</span>
|
||
|
||
<span class="n">ifcb_dir</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'ifcb'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download_unzip_and_remove</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">,</span> <span class="n">url</span><span class="p">):</span>
|
||
<span class="n">tmp_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span> <span class="s1">'ifcb_tmp.zip'</span><span class="p">)</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">tmp_path</span><span class="p">)</span>
|
||
<span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">tmp_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">file</span><span class="p">:</span>
|
||
<span class="n">file</span><span class="o">.</span><span class="n">extractall</span><span class="p">(</span><span class="n">unzipped_path</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">tmp_path</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span><span class="s1">'train'</span><span class="p">)):</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span> <span class="n">URL_TRAIN</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span><span class="s1">'test'</span><span class="p">)):</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span> <span class="n">URL_TEST</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span><span class="s1">'test_prevalences.csv'</span><span class="p">)):</span>
|
||
<span class="n">download_unzip_and_remove</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span> <span class="n">URL_TEST_PREV</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Load test prevalences and classes</span>
|
||
<span class="n">test_true_prev_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span> <span class="s1">'test_prevalences.csv'</span><span class="p">)</span>
|
||
<span class="n">test_true_prev</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">test_true_prev_path</span><span class="p">)</span>
|
||
<span class="n">classes</span> <span class="o">=</span> <span class="n">test_true_prev</span><span class="o">.</span><span class="n">columns</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
|
||
|
||
<span class="c1">#Load train and test samples</span>
|
||
<span class="n">train_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span><span class="s1">'train'</span><span class="p">)</span>
|
||
<span class="n">test_samples_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">ifcb_dir</span><span class="p">,</span><span class="s1">'test'</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">for_model_selection</span><span class="p">:</span>
|
||
<span class="c1"># In this case, return 70% of training data as the training set and 30% as the test set</span>
|
||
<span class="n">samples</span> <span class="o">=</span> <span class="n">get_sample_list</span><span class="p">(</span><span class="n">train_samples_path</span><span class="p">)</span>
|
||
<span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">generate_modelselection_split</span><span class="p">(</span><span class="n">samples</span><span class="p">,</span> <span class="n">test_prop</span><span class="o">=</span><span class="mf">0.3</span><span class="p">)</span>
|
||
<span class="n">train_gen</span> <span class="o">=</span> <span class="n">IFCBTrainSamplesFromDir</span><span class="p">(</span><span class="n">path_dir</span><span class="o">=</span><span class="n">train_samples_path</span><span class="p">,</span> <span class="n">classes</span><span class="o">=</span><span class="n">classes</span><span class="p">,</span> <span class="n">samples</span><span class="o">=</span><span class="n">train</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Test prevalence is computed from class labels</span>
|
||
<span class="n">test_gen</span> <span class="o">=</span> <span class="n">IFCBTestSamples</span><span class="p">(</span><span class="n">path_dir</span><span class="o">=</span><span class="n">train_samples_path</span><span class="p">,</span> <span class="n">test_prevalences</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">samples</span><span class="o">=</span><span class="n">test</span><span class="p">,</span> <span class="n">classes</span><span class="o">=</span><span class="n">classes</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># In this case, we use all training samples as the training set and the test samples as the test set</span>
|
||
<span class="n">train_gen</span> <span class="o">=</span> <span class="n">IFCBTrainSamplesFromDir</span><span class="p">(</span><span class="n">path_dir</span><span class="o">=</span><span class="n">train_samples_path</span><span class="p">,</span> <span class="n">classes</span><span class="o">=</span><span class="n">classes</span><span class="p">)</span>
|
||
<span class="n">test_gen</span> <span class="o">=</span> <span class="n">IFCBTestSamples</span><span class="p">(</span><span class="n">path_dir</span><span class="o">=</span><span class="n">test_samples_path</span><span class="p">,</span> <span class="n">test_prevalences</span><span class="o">=</span><span class="n">test_true_prev</span><span class="p">)</span>
|
||
|
||
<span class="c1"># In the case the user wants it, join all the train samples in one LabelledCollection</span>
|
||
<span class="k">if</span> <span class="n">single_sample_train</span><span class="p">:</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="o">*</span><span class="p">[</span><span class="n">lc</span> <span class="k">for</span> <span class="n">lc</span> <span class="ow">in</span> <span class="n">train_gen</span><span class="p">()])</span>
|
||
<span class="k">return</span> <span class="n">train</span><span class="p">,</span> <span class="n">test_gen</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">train_gen</span><span class="p">,</span> <span class="n">test_gen</span></div>
|
||
|
||
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">_fetch_image_embedding_splits</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">embedding</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="n">LabelledCollection</span><span class="p">,</span><span class="n">LabelledCollection</span><span class="p">,</span><span class="n">LabelledCollection</span><span class="p">]:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads a pre-generated embedding set (train, val, or test) of an image dataset from `Zenodo <https://zenodo.org/records/21131944>`_.</span>
|
||
<span class="sd"> </span>
|
||
<span class="sd"> Embeddings were extracted using `this script <https://github.com/pglez82/visiondatasets_quapy>`_. </span>
|
||
|
||
<span class="sd"> :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'</span>
|
||
<span class="sd"> :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation values), 'predictions' (posterior probabilities)</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :return: a tuple (train, val, test) where each entry is an instance of :class:`quapy.data.base.LabelledCollection`</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">assert</span> <span class="n">dataset_name</span> <span class="ow">in</span> <span class="n">IMAGE_DATASETS</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Name </span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1"> does not match any known dataset. Valid ones are </span><span class="si">{</span><span class="n">IMAGE_DATASETS</span><span class="si">}</span><span class="s1">'</span>
|
||
<span class="k">assert</span> <span class="n">embedding</span> <span class="ow">in</span> <span class="n">IMAGE_EMBEDDINGS</span><span class="p">,</span> \
|
||
<span class="sa">f</span><span class="s1">'Name </span><span class="si">{</span><span class="n">embedding</span><span class="si">}</span><span class="s1"> does not match any known type of embedding. Valid ones are </span><span class="si">{</span><span class="n">IMAGE_EMBEDDINGS</span><span class="si">}</span><span class="s1">'</span>
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">dataset_network</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'cifar10'</span><span class="p">:</span> <span class="s1">'resnet18'</span><span class="p">,</span>
|
||
<span class="s1">'cifar100'</span><span class="p">:</span> <span class="s1">'resnet18'</span><span class="p">,</span>
|
||
<span class="s1">'cifar100coarse'</span><span class="p">:</span> <span class="s1">'resnet18'</span><span class="p">,</span>
|
||
<span class="s1">'svhn'</span><span class="p">:</span> <span class="s1">'resnet18'</span><span class="p">,</span>
|
||
<span class="s1">'fashionmnist'</span><span class="p">:</span> <span class="s1">'basiccnn'</span><span class="p">,</span>
|
||
<span class="s1">'mnist'</span><span class="p">:</span> <span class="s1">'basiccnn'</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="n">trained_network</span> <span class="o">=</span> <span class="n">dataset_network</span><span class="p">[</span><span class="n">dataset_name</span><span class="p">]</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">download_embedding_npz</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">trained_network</span><span class="p">,</span> <span class="n">embedding</span><span class="p">):</span>
|
||
<span class="n">target_file</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">dataset_name</span><span class="si">}</span><span class="s1">_</span><span class="si">{</span><span class="n">trained_network</span><span class="si">}</span><span class="s1">_</span><span class="si">{</span><span class="n">embedding</span><span class="si">}</span><span class="s1">.npz'</span>
|
||
<span class="n">URL</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'https://zenodo.org/records/21131944/files/</span><span class="si">{</span><span class="n">target_file</span><span class="si">}</span><span class="s1">'</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'image'</span><span class="p">),</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">file_path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">data_home</span><span class="p">,</span> <span class="s1">'image'</span><span class="p">,</span> <span class="n">target_file</span><span class="p">)</span>
|
||
<span class="n">download_file_if_not_exists</span><span class="p">(</span><span class="n">URL</span><span class="p">,</span> <span class="n">file_path</span><span class="p">)</span>
|
||
<span class="n">npz_file</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">npz_file</span>
|
||
|
||
<span class="n">embedding_dict</span> <span class="o">=</span> <span class="n">download_embedding_npz</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">trained_network</span><span class="p">,</span> <span class="n">embedding</span><span class="o">=</span><span class="n">embedding</span><span class="p">)</span>
|
||
<span class="n">labels_dict</span> <span class="o">=</span> <span class="n">download_embedding_npz</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">trained_network</span><span class="p">,</span> <span class="n">embedding</span><span class="o">=</span><span class="s1">'targets'</span><span class="p">)</span>
|
||
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="p">(</span><span class="n">embedding_dict</span><span class="p">[</span><span class="s1">'train'</span><span class="p">],</span> <span class="n">labels_dict</span><span class="p">[</span><span class="s1">'train'</span><span class="p">])</span>
|
||
<span class="n">val</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="p">(</span><span class="n">embedding_dict</span><span class="p">[</span><span class="s1">'val'</span><span class="p">],</span> <span class="n">labels_dict</span><span class="p">[</span><span class="s1">'val'</span><span class="p">],</span> <span class="n">classes</span><span class="o">=</span><span class="n">train</span><span class="o">.</span><span class="n">classes</span><span class="p">)</span>
|
||
<span class="n">test</span> <span class="o">=</span> <span class="n">LabelledCollection</span><span class="p">(</span><span class="n">embedding_dict</span><span class="p">[</span><span class="s1">'test'</span><span class="p">],</span> <span class="n">labels_dict</span><span class="p">[</span><span class="s1">'test'</span><span class="p">],</span> <span class="n">classes</span><span class="o">=</span><span class="n">train</span><span class="o">.</span><span class="n">classes</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">train</span><span class="p">,</span> <span class="n">val</span><span class="p">,</span> <span class="n">test</span>
|
||
|
||
|
||
<div class="viewcode-block" id="fetch_image_embeddings">
|
||
<a class="viewcode-back" href="../../../quapy.data.html#quapy.data.datasets.fetch_image_embeddings">[docs]</a>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_image_embeddings</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">embedding</span><span class="p">,</span> <span class="n">heldout_only</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">data_home</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">Dataset</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""</span>
|
||
<span class="sd"> Loads an image dataset with pre-generated embeddings. Available datasets include:</span>
|
||
|
||
<span class="sd"> - 'cifar10', 'cifar100', 'cifar100coarse': see `Alex Krizhevsky and Geoffrey Hinton. Learning multiple layers of features from tiny images. Technical report, University of Toronto, Toronto, Ontario, 2009. <https://cave.cs.toronto.edu/kriz/learning-features-2009-TR.pdf>`_</span>
|
||
<span class="sd"> - 'mnist': `Yann LeCun, Corinna Cortes, and Christopher J. C. Burges. The MNIST database of handwritten digits. 1998. <http://yann.lecun.com/exdb/mnist/>`_</span>
|
||
<span class="sd"> - 'fashionmnist': `Han Xiao, Kashif Rasul, and Roland Vollgraf. Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. arXiv preprint arXiv:1708.07747, 2017. <https://arxiv.org/abs/1708.07747>`_</span>
|
||
<span class="sd"> - 'svhn': `Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Baolin Wu, Andrew Y Ng, et al. Reading digits in natural images with unsupervised feature learning. In NIPS workshop on deep learning and unsupervised feature learning, volume 2011, page 4. Granada, 2011. <https://static.googleusercontent.com/media/research.google.com/es//pubs/archive/37648.pdf>`_</span>
|
||
<span class="sd"> </span>
|
||
<span class="sd"> The image dataset are stored in `Zenodo <https://zenodo.org/records/21131944>`_ and were extracted using `this script <https://github.com/pglez82/visiondatasets_quapy>`_. </span>
|
||
|
||
<span class="sd"> These embeddings were generated using a resnet18 or a simple cnn. In all cases, the network was trained using ~60% of the data, validated on ~25% of the data, and the remaining ~15% was used for test. Splits were created with stratification.</span>
|
||
<span class="sd"> Once the network is trained, it was used with frozen weights to generate embeddings for the training, validation, and test, in different formats (see below).</span>
|
||
<span class="sd"> It would therefore be convenient to use only heldout data (validation and test) for training and testing quantifiers (this is the default behavior), although the training+validation data can be accessed with `heldout_only=False`.</span>
|
||
|
||
<span class="sd"> :param dataset_name: the name of the dataset: valid ones are 'cifar10', 'cifar100', 'cifar100coarse', 'svhn', 'fashionmnist', 'mnist'</span>
|
||
<span class="sd"> :param embedding: the type of embedding: valid ones are 'features' (next-to-last representations), 'logits' (pre-activation outputs), 'predictions' (post-softmax outputs, or predicted posterior probabilities)</span>
|
||
<span class="sd"> :param heldout_only: whether to discard the part of the training data used to train the neural model that generated the embeddings (default: True); set to False</span>
|
||
<span class="sd"> to obtain, as the training data, the original training+validation splits.</span>
|
||
<span class="sd"> :param data_home: specify the quapy home directory where collections will be dumped (leave empty to use the default</span>
|
||
<span class="sd"> ~/quay_data/ directory)</span>
|
||
<span class="sd"> :return: an instance of :class:`quapy.data.base.Dataset`</span>
|
||
<span class="sd"> """</span>
|
||
<span class="k">if</span> <span class="n">data_home</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">data_home</span> <span class="o">=</span> <span class="n">get_quapy_home</span><span class="p">()</span>
|
||
|
||
<span class="n">network_train</span><span class="p">,</span> <span class="n">val</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">_fetch_image_embedding_splits</span><span class="p">(</span><span class="n">dataset_name</span><span class="p">,</span> <span class="n">embedding</span><span class="p">,</span> <span class="n">data_home</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">heldout_only</span><span class="p">:</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">val</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">network_train</span> <span class="o">+</span> <span class="n">val</span>
|
||
|
||
<span class="k">return</span> <span class="n">Dataset</span><span class="p">(</span><span class="n">train</span><span class="p">,</span> <span class="n">test</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="n">dataset_name</span><span class="p">)</span></div>
|
||
|
||
|
||
|
||
</pre></div>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
</footer>
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script defer src="../../../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9"></script>
|
||
<script defer src="../../../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9"></script>
|
||
|
||
<footer class="bd-footer">
|
||
<div class="bd-footer__inner bd-page-width">
|
||
|
||
<div class="footer-items__start">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2024, Alejandro Moreo.
|
||
<br/>
|
||
|
||
</p>
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="sphinx-version">
|
||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.0.4.
|
||
<br/>
|
||
</p>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="footer-items__end">
|
||
|
||
<div class="footer-item">
|
||
<p class="theme-version">
|
||
<!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
|
||
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.20.0.
|
||
</p></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</footer>
|
||
</body>
|
||
</html> |