OpenML Benchmark generator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import openml as oml\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "import math\n",
    "from scipy.stats import norm\n",
    "from matplotlib import pyplot\n",
    "import sklearn.tree\n",
    "import sklearn.preprocessing\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import StratifiedShuffleSplit, cross_val_score\n",
    "sys.path.insert(0,'Python')\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 0: Get all datasets from OpenML\n",
    "- Only classification datasets\n",
    "- Only active (verified) datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1375 active classification datasets\n"
     ]
    }
   ],
   "source": [
    "# Get all OpenML datasets\n",
    "status_type = 'active'\n",
    "openml_list = oml.datasets.list_datasets(status=status_type) # Returns a dict\n",
    "datalist = pd.DataFrame.from_dict(openml_list, orient='index') # Transform to pandas\n",
    "#datalist = datalist[datalist.status == 'active'] # Only use active (verified) datasets\n",
    "datalist = datalist[datalist.NumberOfClasses>=2] # Only classification\n",
    "print(\"{} {} classification datasets\".format(len(datalist), status_type))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bookkeeping\n",
    "data_names = {k: v for (k, v) in datalist[['did','name']].values} # dataset names\n",
    "data_status = {k: 'OK' for k in datalist.index} # dataset status (OK or reason for removal)\n",
    "datalist_full = datalist.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Apply simple preconditions\n",
    "- Number of observations larger than 500 (meaningful evaluations)\n",
    "- Number of observations smaller than 100000 (keep runtime manageable)\n",
    "- Number of features does not exceed 5000 (keep runtime manageable)\n",
    "- The ratio of the minority class and the majority class is > 0.05 (severely imbalanced datasets complicate analysis)\n",
    "- Number of values for categorical features must not exceed 100 (severely slows down some algorithms)\n",
    "- Sparsely formatted data (requires special data readers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Extreme imbalance', 207],\n",
       " ['OK', 492],\n",
       " ['High-dimensional', 78],\n",
       " ['Too large', 190],\n",
       " ['Too small', 378],\n",
       " ['Sparse format', 30]]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Apply preconditions\n",
    "data_status.update({k: 'Too small' for k in datalist.index[datalist.NumberOfInstances<500]})\n",
    "data_status.update({k: 'Too large' for k in datalist.index[datalist.NumberOfInstances>100000]})\n",
    "data_status.update({k: 'High-dimensional' for k in datalist.index[datalist.NumberOfFeatures>5000]})\n",
    "data_status.update({k: 'Extreme imbalance' for k in datalist.index[datalist.MinorityClassSize / datalist.MajorityClassSize < 0.05]})\n",
    "#data_status.update({k: 'Too many categories' for k in datalist.index[datalist.MaxNominalAttDistinctValues > 100]})\n",
    "data_status.update({k: 'Sparse format' for k in datalist.index[datalist.format == 'Sparse_ARFF']})\n",
    "\n",
    "# Filter dataset list\n",
    "datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})] \n",
    "\n",
    "# Status update\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Filter out special datasets\n",
    "- Artificial datasets (may bias the results)\n",
    "- Time series dataset (cannot use random sampling for evaluation)\n",
    "- Text data (contains string features which need additional preprocessing)\n",
    "- Multilabel data (multiple targets need to be predicted)\n",
    "- Derived versions of datasets (with additional preprocessing)\n",
    "- Datasets where the intended classification target is unclear\n",
    "- Binarized regression problems\n",
    "- Unknown origin (no description how data was collected and what the problem is)\n",
    "- Grouped data (instances form groups (blocks) and can't be randomly sampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-7-75846ec54bd3>:28: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['Extreme imbalance', 141],\n",
       " ['Binarized regression problem', 88],\n",
       " ['OK', 307],\n",
       " ['Multi-label data', 7],\n",
       " ['High-dimensional', 78],\n",
       " ['Unspecified target feature', 5],\n",
       " ['Label leakage', 2],\n",
       " ['Text data', 2],\n",
       " ['Derived (non-original) data', 42],\n",
       " ['Too large', 103],\n",
       " ['Too small', 377],\n",
       " ['Time series data', 7],\n",
       " ['Grouped data', 2],\n",
       " ['Sparse format', 30],\n",
       " ['Artificial data', 285],\n",
       " ['Unknown origin', 6]]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get lists of special datasets\n",
    "artificial_set = set(oml.datasets.list_datasets(tag=\"artificial\", status=status_type).keys())\n",
    "timeseries_set = set(oml.datasets.list_datasets(tag=\"time_series\", status=status_type).keys())\n",
    "text_set = set(oml.datasets.list_datasets(tag=\"text_data\", status=status_type).keys())\n",
    "multilabel_set = set(oml.datasets.list_datasets(tag=\"multi_label\", status=status_type).keys())\n",
    "derived_set = set(oml.datasets.list_datasets(tag=\"derived\", status=status_type).keys())\n",
    "unspecified_set = set(oml.datasets.list_datasets(tag=\"unspecified_target_feature\", status=status_type).keys())\n",
    "binarized_set = set(oml.datasets.list_datasets(tag=\"binarized_regression_problem\", status=status_type).keys())\n",
    "unknown_set = set(oml.datasets.list_datasets(tag=\"origin_unknown\", status=status_type).keys())\n",
    "grouped_set = set(oml.datasets.list_datasets(tag=\"grouped_data\", status=status_type).keys())\n",
    "label_leakage = set(oml.datasets.list_datasets(tag=\"label_leakage\", status=status_type).keys())\n",
    "single_feature = set(oml.datasets.list_datasets(tag=\"single_feature_predictable\", status=status_type).keys())\n",
    "\n",
    "data_status.update({k: 'Artificial data' for k in artificial_set})\n",
    "data_status.update({k: 'Time series data' for k in timeseries_set})\n",
    "data_status.update({k: 'Text data' for k in text_set})\n",
    "data_status.update({k: 'Multi-label data' for k in multilabel_set})\n",
    "data_status.update({k: 'Derived (non-original) data' for k in derived_set})\n",
    "data_status.update({k: 'Unspecified target feature' for k in unspecified_set})\n",
    "data_status.update({k: 'Binarized regression problem' for k in binarized_set})\n",
    "data_status.update({k: 'Unknown origin' for k in unknown_set})\n",
    "data_status.update({k: 'Grouped data' for k in grouped_set})\n",
    "data_status.update({k: 'Label leakage' for k in label_leakage})\n",
    "data_status.update({k: 'Single feature' for k in single_feature})\n",
    "#data_status.update({k: 'OpenML100' for k in openml100_set})\n",
    "\n",
    "# Filter dataset list\n",
    "datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})] \n",
    "\n",
    "# Status update\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Remove alternative versions of datasets\n",
    "- Remove binarized versions of multi-class datasets\n",
    "- Check other possible duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-8-e0e88a86dc5a>:43: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['Multi-label data', 6],\n",
       " ['Label leakage', 2],\n",
       " ['Too large', 103],\n",
       " ['Extreme imbalance', 139],\n",
       " ['Duplicate of 40984', 2],\n",
       " ['Duplicate of 40994', 1],\n",
       " ['Derived (non-original) data', 35],\n",
       " ['Binarized version of multiclass dataset', 88],\n",
       " ['Time series data', 7],\n",
       " ['Too small', 336],\n",
       " ['Grouped data', 2],\n",
       " ['Sparse format', 29],\n",
       " ['Artificial data', 284],\n",
       " ['Unknown origin', 6],\n",
       " ['Binarized regression problem', 86],\n",
       " ['OK', 257],\n",
       " ['Unspecified target feature', 5],\n",
       " ['Duplicate of 1590', 1],\n",
       " ['Duplicate of 23380', 2],\n",
       " ['Duplicate of 40979', 2],\n",
       " ['Duplicate of 40945', 1],\n",
       " ['High-dimensional', 78],\n",
       " ['Possible duplicate', 7],\n",
       " ['Text data', 2],\n",
       " ['Duplicate of 40982', 1]]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sorting makes things easier\n",
    "# We need the full list because there may be binarized versions of already removed datasets\n",
    "datalist_full = datalist_full.sort_values(by=['name','NumberOfClasses'], ascending=[True, False])\n",
    "\n",
    "checked_datasets = {\n",
    "    40979: [1022, 20], # Correct version of mfeat-pixel\n",
    "    40984: [958, 36], # Correct version of segment\n",
    "    40994: [40990, 40989, 1467], # Correct version of climate-model-simulation-crashes\n",
    "    1590: [179], # Correct version of adult\n",
    "    40983: [1570], # Correct version of wilt\n",
    "    40945: [40704], # Correct version of Titanic\n",
    "    772: [948], # Correct version of classification version of the quake dataset\n",
    "    40966: [40965, 40964], # Correct version of MiceProtein (needed when including in_preparation datasets)\n",
    "    40982: [40973, 1504], # Correct version of steel-plates-fault (needed when including in_preparation datasets)\n",
    "    23380: [1024, 473], # Correct version of cjs (needed when including in_preparation datasets)\n",
    "    40597 : [40733], # Correct version of yeast (needed when including in_preparation datasets)\n",
    "    1046 : [40829], # Correct version of mozilla4 (needed when including in_preparation datasets)\n",
    "    # : [40958], # Correct version of Bankdata (needed when including in_preparation datasets)\n",
    "    \n",
    "}\n",
    "duplicates_of = {}\n",
    "# Mark the duplicates of datasets where we know which version is the correct one!\n",
    "for cd in checked_datasets:\n",
    "    for dup_id in checked_datasets[cd]:\n",
    "        duplicates_of[dup_id] = cd\n",
    "\n",
    "data_unique = {}\n",
    "for index, row in datalist_full.iterrows():\n",
    "    if row['did'] in duplicates_of:\n",
    "        data_status[row['did']] = 'Duplicate of %d' % duplicates_of[row['did']]\n",
    "    elif row['did'] in checked_datasets and data_status[row['did']] in ('OK', 'Possible duplicate'):\n",
    "        data_status[row['did']] = 'OK'\n",
    "    elif row['name'] not in data_unique:\n",
    "        data_unique[row['name']] = row\n",
    "    else:\n",
    "        previous = data_unique[row['name']]\n",
    "        if previous['NumberOfClasses'] > 2 and row['NumberOfClasses'] == 2:\n",
    "            data_status[row['did']] = 'Binarized version of multiclass dataset'\n",
    "        elif data_status[row['did']] in ('OK', 'Possible duplicate'):\n",
    "            data_status[row['did']] = 'Possible duplicate'\n",
    "\n",
    "# Filter dataset list\n",
    "datalist = datalist[pd.Series({k:(v=='OK') for k,v in data_status.items()})] \n",
    "               \n",
    "# Status update\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[42193, 42345, 42493, 42734, 42804, 42805, 42883]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These need to be checked\n",
    "[k for k,v in data_status.items() if v=='Possible duplicate']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Remove trivial datasets\n",
    "- See if a model (e.g. random forest) based on 1 feature can get perfect CV performance (JvR, removed this code. But Irish and cjs came out of this check. I will tag them. )\n",
    "- See if a CV Decision Tree (flow id 7777) or a CV logistic regression (flow id 7778) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Multi-label data', 6],\n",
       " ['Dataset not checked for triviality by DT!', 176],\n",
       " ['Label leakage', 2],\n",
       " ['Too large', 103],\n",
       " ['Too easy (perfect score decision tree)', 2],\n",
       " ['Extreme imbalance', 139],\n",
       " ['Duplicate of 40984', 2],\n",
       " ['Duplicate of 40994', 1],\n",
       " ['Derived (non-original) data', 35],\n",
       " ['Binarized version of multiclass dataset', 88],\n",
       " ['Time series data', 7],\n",
       " ['Too small', 336],\n",
       " ['Grouped data', 2],\n",
       " ['Sparse format', 29],\n",
       " ['Artificial data', 284],\n",
       " ['Unknown origin', 6],\n",
       " ['Binarized regression problem', 86],\n",
       " ['OK', 79],\n",
       " ['Unspecified target feature', 5],\n",
       " ['Duplicate of 1590', 1],\n",
       " ['Duplicate of 23380', 2],\n",
       " ['Duplicate of 40979', 2],\n",
       " ['Duplicate of 40945', 1],\n",
       " ['High-dimensional', 78],\n",
       " ['Possible duplicate', 7],\n",
       " ['Text data', 2],\n",
       " ['Duplicate of 40982', 1]]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perfect_scores_dt = {int(v.data_id) for k, v in oml.evaluations.list_evaluations('predictive_accuracy', flows=[7777]).items() if v.value == 1}\n",
    "perfect_scores_lr = {int(v.data_id) for k, v in oml.evaluations.list_evaluations('predictive_accuracy', flows=[7778]).items() if v.value == 1}\n",
    "\n",
    "for did in perfect_scores_dt:\n",
    "    if data_status[did] == 'OK':\n",
    "        data_status[did] = 'Too easy (perfect score decision tree)'\n",
    "        \n",
    "for did in perfect_scores_lr:\n",
    "    if data_status[did] == 'OK':\n",
    "        data_status[did] = 'Too easy (perfect score logistic regression)'\n",
    "     \n",
    "checked_datasets = {int(v.data_id) for v in oml.evaluations.list_evaluations('predictive_accuracy', flows=[7777]).values()}\n",
    "for did in data_status:\n",
    "    if data_status[did] == 'OK' and did not in checked_datasets:\n",
    "        data_status[did] = 'Dataset not checked for triviality by DT!'\n",
    "checked_datasets = {int(v.data_id) for v in oml.evaluations.list_evaluations('predictive_accuracy', flows=[7778]).values()}\n",
    "# for did in data_status:\n",
    "#     if data_status[did] == 'OK' and did not in checked_datasets:\n",
    "#         data_status[did] = 'Dataset not checked for triviality by LR!'\n",
    "\n",
    "[[x,list(data_status.values()).count(x)] for x in set(data_status.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm, tqdm_notebook\n",
    "\n",
    "datasets = [k for k,v in data_status.items() if v=='OK']\n",
    "\n",
    "max_score_per_dataset = {}\n",
    "for idx, dataset_id in enumerate(tqdm_notebook(datasets)):\n",
    "    try:\n",
    "        \n",
    "        dataset = oml.datasets.get_dataset(dataset_id)\n",
    "        #print('processing dataset', dataset_id, dataset.name, '(',idx+1, '/', len(datasets), ')')\n",
    "        if dataset.default_target_attribute is None:\n",
    "            data_status[dataset_id] = 'No target specified'\n",
    "            print('No target')\n",
    "            continue\n",
    "        X, y = dataset.get_data(target=dataset.default_target_attribute)\n",
    "\n",
    "        #print('One hot encoding dataset to check for the true amount of total features.')\n",
    "        categorical_indices = dataset.get_features_by_type('nominal', [dataset.default_target_attribute])\n",
    "        clf = sklearn.pipeline.Pipeline(\n",
    "            steps=[\n",
    "                (\n",
    "                    'imputer', ConditionalImputer(\n",
    "                        strategy='median', \n",
    "                        strategy_nominal='mean', \n",
    "                        categorical_features=categorical_indices, \n",
    "                        fill_empty=-1,\n",
    "                    )\n",
    "                ), \n",
    "                (\n",
    "                    'encoder', \n",
    "                    sklearn.preprocessing.OneHotEncoder(categorical_features=categorical_indices)\n",
    "                )\n",
    "            ]\n",
    "        )\n",
    "        hotencoded = clf.fit_transform(X)\n",
    "        if hotencoded.shape[1] > 5000:\n",
    "            data_status[dataset_id] = 'High-dimensional (after one hot encoding.)'\n",
    "            print(dataset.name, 'too high one-hot-encoded dimensionality', hotencoded.shape[1])\n",
    "            continue\n",
    "\n",
    "        #print('building decision stump')\n",
    "        clf = sklearn.pipeline.Pipeline(steps=[('imputer', sklearn.preprocessing.Imputer(strategy='median')), \n",
    "                                               ('classifier', sklearn.tree.DecisionTreeClassifier(max_depth=1))])\n",
    "        _ = clf.fit(X, y)\n",
    "        score = clf.score(X, y)\n",
    "        \n",
    "        #print('obtaining cv task .. ')\n",
    "        # TODO\n",
    "\n",
    "        max_score_per_dataset[dataset_id] = {\n",
    "            'score': score,\n",
    "            'name': dataset.name\n",
    "        }\n",
    "    except ValueError as e:\n",
    "        data_status[dataset_id] = 'Python ValueError'\n",
    "        print(dataset_id, e)\n",
    "        continue\n",
    "    except Exception as e:\n",
    "        data_status[dataset_id] = 'Python Exception'\n",
    "        print(dataset_id, e)\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    if max_score_per_dataset[dataset_id][\"score\"] == 1.00:\n",
    "        data_status[dataset_id] = 'Too easy (decisionstump on trainset)'\n",
    "        print(\"Dataset \", dataset.name, \"is too easy.\")\n",
    "    \n",
    "results = pd.DataFrame(max_score_per_dataset).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1493</th>\n",
       "      <td>one-hundred-plants-texture</td>\n",
       "      <td>0.0193871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1492</th>\n",
       "      <td>one-hundred-plants-shape</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1491</th>\n",
       "      <td>one-hundred-plants-margin</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40923</th>\n",
       "      <td>Devnagari-Script</td>\n",
       "      <td>0.0401522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>letter</td>\n",
       "      <td>0.0718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>isolet</td>\n",
       "      <td>0.0766962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40971</th>\n",
       "      <td>collins</td>\n",
       "      <td>0.109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>vowel</td>\n",
       "      <td>0.167677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40499</th>\n",
       "      <td>texture</td>\n",
       "      <td>0.180545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1515</th>\n",
       "      <td>micro-mass</td>\n",
       "      <td>0.182137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>mfeat-zernike</td>\n",
       "      <td>0.186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>mfeat-karhunen</td>\n",
       "      <td>0.189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>mfeat-fourier</td>\n",
       "      <td>0.1915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mfeat-factors</td>\n",
       "      <td>0.192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>optdigits</td>\n",
       "      <td>0.195018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40979</th>\n",
       "      <td>mfeat-pixel</td>\n",
       "      <td>0.1975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1501</th>\n",
       "      <td>semeion</td>\n",
       "      <td>0.19774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554</th>\n",
       "      <td>mnist_784</td>\n",
       "      <td>0.198743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40996</th>\n",
       "      <td>Fashion-MNIST</td>\n",
       "      <td>0.199229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>mfeat-morphological</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>pendigits</td>\n",
       "      <td>0.204785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1468</th>\n",
       "      <td>cnae-9</td>\n",
       "      <td>0.205556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>analcatdata_dmft</td>\n",
       "      <td>0.212045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>soybean</td>\n",
       "      <td>0.234261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40966</th>\n",
       "      <td>MiceProtein</td>\n",
       "      <td>0.273148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40984</th>\n",
       "      <td>segment</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1478</th>\n",
       "      <td>har</td>\n",
       "      <td>0.373823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>vehicle</td>\n",
       "      <td>0.410165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1475</th>\n",
       "      <td>first-order-theorem-proving</td>\n",
       "      <td>0.417457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>satimage</td>\n",
       "      <td>0.424417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>credit-g</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1480</th>\n",
       "      <td>ilpd</td>\n",
       "      <td>0.713551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>diabetes</td>\n",
       "      <td>0.735677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4134</th>\n",
       "      <td>Bioresponse</td>\n",
       "      <td>0.73687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1494</th>\n",
       "      <td>qsar-biodeg</td>\n",
       "      <td>0.745972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1489</th>\n",
       "      <td>phoneme</td>\n",
       "      <td>0.754626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>electricity</td>\n",
       "      <td>0.757393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1590</th>\n",
       "      <td>adult</td>\n",
       "      <td>0.760718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1464</th>\n",
       "      <td>blood-transfusion-service-center</td>\n",
       "      <td>0.762032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>spambase</td>\n",
       "      <td>0.79374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053</th>\n",
       "      <td>jm1</td>\n",
       "      <td>0.806523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40536</th>\n",
       "      <td>SpeedDating</td>\n",
       "      <td>0.835283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1067</th>\n",
       "      <td>kc1</td>\n",
       "      <td>0.845424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4534</th>\n",
       "      <td>PhishingWebsites</td>\n",
       "      <td>0.847309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>kc2</td>\n",
       "      <td>0.850575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1486</th>\n",
       "      <td>nomao</td>\n",
       "      <td>0.852807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1462</th>\n",
       "      <td>banknote-authentication</td>\n",
       "      <td>0.853499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>credit-approval</td>\n",
       "      <td>0.855072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40701</th>\n",
       "      <td>churn</td>\n",
       "      <td>0.8706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1049</th>\n",
       "      <td>pc4</td>\n",
       "      <td>0.877915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1461</th>\n",
       "      <td>bank-marketing</td>\n",
       "      <td>0.883015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050</th>\n",
       "      <td>pc3</td>\n",
       "      <td>0.897633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40705</th>\n",
       "      <td>tokyo1</td>\n",
       "      <td>0.89781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40994</th>\n",
       "      <td>climate-model-simulation-crashes</td>\n",
       "      <td>0.914815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1510</th>\n",
       "      <td>wdbc</td>\n",
       "      <td>0.922671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>breast-w</td>\n",
       "      <td>0.924177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1068</th>\n",
       "      <td>pc1</td>\n",
       "      <td>0.935978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1487</th>\n",
       "      <td>ozone-level-8hr</td>\n",
       "      <td>0.936859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40983</th>\n",
       "      <td>wilt</td>\n",
       "      <td>0.946063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>sick</td>\n",
       "      <td>0.965536</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   name      score\n",
       "1493         one-hundred-plants-texture  0.0193871\n",
       "1492           one-hundred-plants-shape       0.02\n",
       "1491          one-hundred-plants-margin       0.02\n",
       "40923                  Devnagari-Script  0.0401522\n",
       "6                                letter     0.0718\n",
       "300                              isolet  0.0766962\n",
       "40971                           collins      0.109\n",
       "307                               vowel   0.167677\n",
       "40499                           texture   0.180545\n",
       "1515                         micro-mass   0.182137\n",
       "22                        mfeat-zernike      0.186\n",
       "16                       mfeat-karhunen      0.189\n",
       "14                        mfeat-fourier     0.1915\n",
       "12                        mfeat-factors      0.192\n",
       "28                            optdigits   0.195018\n",
       "40979                       mfeat-pixel     0.1975\n",
       "1501                            semeion    0.19774\n",
       "554                           mnist_784   0.198743\n",
       "40996                     Fashion-MNIST   0.199229\n",
       "18                  mfeat-morphological        0.2\n",
       "32                            pendigits   0.204785\n",
       "1468                             cnae-9   0.205556\n",
       "469                    analcatdata_dmft   0.212045\n",
       "42                              soybean   0.234261\n",
       "40966                       MiceProtein   0.273148\n",
       "40984                           segment   0.285714\n",
       "1478                                har   0.373823\n",
       "54                              vehicle   0.410165\n",
       "1475        first-order-theorem-proving   0.417457\n",
       "182                            satimage   0.424417\n",
       "...                                 ...        ...\n",
       "31                             credit-g        0.7\n",
       "1480                               ilpd   0.713551\n",
       "37                             diabetes   0.735677\n",
       "4134                        Bioresponse    0.73687\n",
       "1494                        qsar-biodeg   0.745972\n",
       "1489                            phoneme   0.754626\n",
       "151                         electricity   0.757393\n",
       "1590                              adult   0.760718\n",
       "1464   blood-transfusion-service-center   0.762032\n",
       "44                             spambase    0.79374\n",
       "1053                                jm1   0.806523\n",
       "40536                       SpeedDating   0.835283\n",
       "1067                                kc1   0.845424\n",
       "4534                   PhishingWebsites   0.847309\n",
       "1063                                kc2   0.850575\n",
       "1486                              nomao   0.852807\n",
       "1462            banknote-authentication   0.853499\n",
       "29                      credit-approval   0.855072\n",
       "40701                             churn     0.8706\n",
       "1049                                pc4   0.877915\n",
       "1461                     bank-marketing   0.883015\n",
       "1050                                pc3   0.897633\n",
       "40705                            tokyo1    0.89781\n",
       "40994  climate-model-simulation-crashes   0.914815\n",
       "1510                               wdbc   0.922671\n",
       "15                             breast-w   0.924177\n",
       "1068                                pc1   0.935978\n",
       "1487                    ozone-level-8hr   0.936859\n",
       "40983                              wilt   0.946063\n",
       "38                                 sick   0.965536\n",
       "\n",
       "[75 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.sort_values(by='score')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Remove datasets for other reasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_status[40705] = 'Missing description.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6a: Final selection\n",
    "Final list of selected datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38 datasets selected\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{1478: 'har',\n",
       " 1480: 'ilpd',\n",
       " 1485: 'madelon',\n",
       " 1486: 'nomao',\n",
       " 1487: 'ozone-level-8hr',\n",
       " 1489: 'phoneme',\n",
       " 1491: 'one-hundred-plants-margin',\n",
       " 1492: 'one-hundred-plants-shape',\n",
       " 1493: 'one-hundred-plants-texture',\n",
       " 1494: 'qsar-biodeg',\n",
       " 1497: 'wall-robot-navigation',\n",
       " 1501: 'semeion',\n",
       " 1510: 'wdbc',\n",
       " 1515: 'micro-mass',\n",
       " 1590: 'adult',\n",
       " 4134: 'Bioresponse',\n",
       " 4135: 'Amazon_employee_access',\n",
       " 4534: 'PhishingWebsites',\n",
       " 4538: 'GesturePhaseSegmentationProcessed',\n",
       " 6332: 'cylinder-bands',\n",
       " 23381: 'dresses-sales',\n",
       " 23517: 'numerai28.6',\n",
       " 40499: 'texture',\n",
       " 40536: 'SpeedDating',\n",
       " 40668: 'connect-4',\n",
       " 40670: 'dna',\n",
       " 40701: 'churn',\n",
       " 40705: 'tokyo1',\n",
       " 40923: 'Devnagari-Script',\n",
       " 40966: 'MiceProtein',\n",
       " 40971: 'collins',\n",
       " 40979: 'mfeat-pixel',\n",
       " 40982: 'steel-plates-fault',\n",
       " 40983: 'wilt',\n",
       " 40984: 'segment',\n",
       " 40994: 'climate-model-simulation-crashes',\n",
       " 40996: 'Fashion-MNIST',\n",
       " 41027: 'jungle_chess_2pcs_raw_endgame_complete'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_datasets = [k for k,v in data_status.items() if v=='OK']\n",
    "print('{} datasets selected'.format(len(final_datasets)))\n",
    "{k:v for k,v in data_names.items() if k in final_datasets}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6b: Check tags\n",
    "Passed all tests, but not in CC-18:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1491: 'one-hundred-plants-margin',\n",
       " 1492: 'one-hundred-plants-shape',\n",
       " 1493: 'one-hundred-plants-texture',\n",
       " 1515: 'micro-mass',\n",
       " 4135: 'Amazon_employee_access',\n",
       " 40536: 'SpeedDating',\n",
       " 40705: 'tokyo1',\n",
       " 40971: 'collins'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CC18_set = set(oml.datasets.list_datasets(tag=\"OpenML-CC18\").keys())\n",
    "\n",
    "new_datasets = [k for k,v in data_status.items() if v=='OK' and k not in CC18_set]\n",
    "{k:v for k,v in data_names.items() if k in new_datasets}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Datasets tagged with CC18 that did not pass all tests:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{3: 'kr-vs-kp',\n",
       " 6: 'letter',\n",
       " 11: 'balance-scale',\n",
       " 12: 'mfeat-factors',\n",
       " 14: 'mfeat-fourier',\n",
       " 15: 'breast-w',\n",
       " 16: 'mfeat-karhunen',\n",
       " 18: 'mfeat-morphological',\n",
       " 22: 'mfeat-zernike',\n",
       " 23: 'cmc',\n",
       " 28: 'optdigits',\n",
       " 29: 'credit-approval',\n",
       " 31: 'credit-g',\n",
       " 32: 'pendigits',\n",
       " 37: 'diabetes',\n",
       " 38: 'sick',\n",
       " 44: 'spambase',\n",
       " 46: 'splice',\n",
       " 50: 'tic-tac-toe',\n",
       " 54: 'vehicle',\n",
       " 151: 'electricity',\n",
       " 182: 'satimage',\n",
       " 188: 'eucalyptus',\n",
       " 300: 'isolet',\n",
       " 307: 'vowel',\n",
       " 375: 'JapaneseVowels',\n",
       " 458: 'analcatdata_authorship',\n",
       " 469: 'analcatdata_dmft',\n",
       " 554: 'mnist_784',\n",
       " 1049: 'pc4',\n",
       " 1050: 'pc3',\n",
       " 1053: 'jm1',\n",
       " 1063: 'kc2',\n",
       " 1067: 'kc1',\n",
       " 1068: 'pc1',\n",
       " 1461: 'bank-marketing',\n",
       " 1462: 'banknote-authentication',\n",
       " 1464: 'blood-transfusion-service-center',\n",
       " 1468: 'cnae-9',\n",
       " 1475: 'first-order-theorem-proving',\n",
       " 40927: 'CIFAR_10',\n",
       " 40975: 'car',\n",
       " 40978: 'Internet-Advertisements'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_datasets = [k for k,v in data_status.items() if k in CC18_set and v!='OK']\n",
    "{k:v for k,v in data_names.items() if k in new_datasets}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6c: Overview of exclusions\n",
    "Reasons to exclude datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Extreme imbalance': '[2, 5, 7, 9, 10, 26, 30, 34, 39, 57, 149, 150, 155, 171, 180, 181, 183, 184, 185, 186, 275, 279, 310, 311, 313, 316, 343, 372, 374, 379, 443, 449, 453, 454, 474, 477, 488, 947, 950, 951, 1039, 1047, 1056, 1057, 1069, 1102, 1109, 1110, 1111, 1113, 1142, 1146, 1472, 1481, 1483, 1509, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1567, 1569, 1596, 1597, 4154, 4340, 4552, 40474, 40475, 40476, 40477, 40478, 40497, 40498, 40588, 40590, 40593, 40596, 40672, 40685, 40691, 40707, 40708, 40713, 40900, 40910, 41138, 41168, 41169, 41533, 41703, 41939, 41960, 41972, 41981, 42078, 42087, 42088, 42089, 42123, 42132, 42133, 42166, 42167, 42169, 42177, 42206, 42252, 42256, 42397, 42553, 42680, 42716, 42742, 42746, 42757, 42774, 43028, 43029, 43033, 43035, 43044, 43051]',\n",
       " 'Python ValueError': '[3, 6, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 38, 42, 44, 46, 50, 54, 151, 182, 188, 300, 307, 469, 554, 1049, 1050, 1053, 1063, 1067, 1068, 1112, 1114, 1116, 1461, 1462, 1464, 1468, 1475]',\n",
       " 'Too small': '[4, 13, 25, 27, 35, 40, 41, 43, 48, 49, 52, 53, 55, 56, 59, 61, 62, 163, 164, 172, 187, 276, 277, 278, 285, 327, 328, 329, 336, 337, 338, 339, 340, 342, 346, 376, 380, 444, 446, 448, 450, 452, 455, 457, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 472, 475, 476, 479, 480, 481, 682, 683, 685, 694, 713, 714, 716, 719, 721, 724, 726, 729, 730, 731, 732, 733, 736, 738, 739, 744, 745, 746, 747, 748, 753, 754, 755, 756, 758, 759, 760, 762, 763, 764, 765, 767, 768, 769, 771, 773, 775, 776, 777, 778, 780, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 793, 794, 796, 798, 800, 801, 804, 808, 810, 811, 812, 814, 815, 817, 820, 828, 829, 830, 831, 832, 834, 835, 836, 840, 842, 844, 848, 850, 851, 852, 854, 857, 858, 859, 860, 861, 862, 863, 864, 865, 868, 873, 874, 875, 876, 877, 878, 880, 882, 885, 887, 889, 890, 891, 892, 893, 894, 895, 898, 899, 900, 902, 905, 906, 907, 908, 909, 911, 915, 916, 918, 919, 921, 922, 924, 925, 927, 928, 929, 932, 933, 935, 938, 939, 941, 942, 944, 945, 946, 952, 956, 968, 982, 984, 1001, 1045, 1048, 1054, 1055, 1059, 1060, 1061, 1062, 1064, 1065, 1066, 1071, 1073, 1075, 1100, 1101, 1115, 1117, 1121, 1167, 1412, 1413, 1416, 1417, 1418, 1441, 1442, 1446, 1447, 1448, 1449, 1450, 1455, 1463, 1465, 1473, 1482, 1484, 1488, 1490, 1495, 1498, 1499, 1500, 1506, 1508, 1511, 1512, 1513, 1514, 1516, 1517, 1518, 1519, 1520, 1523, 1556, 1559, 1561, 1562, 1563, 1564, 1565, 1600, 4153, 4329, 23499, 40660, 40663, 40665, 40669, 40671, 40681, 40682, 40686, 40700, 40709, 40710, 40711, 40714, 41083, 41430, 41496, 41511, 41521, 41538, 41544, 41568, 41583, 41674, 41675, 41679, 41680, 41682, 41684, 41685, 41950, 41976, 41977, 41978, 41997, 41998, 42003, 42011, 42016, 42021, 42026, 42031, 42036, 42041, 42046, 42051, 42056, 42066, 42071, 42098, 42172, 42186, 42223, 42261, 42544, 42554, 42585, 42665, 42700, 42793]',\n",
       " 'Artificial data': '[11, 60, 70, 71, 72, 73, 74, 75, 76, 77, 78, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 271, 272, 333, 334, 335, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 650, 651, 652, 653, 654, 655, 656, 657, 658, 1120, 1177, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1459, 1460, 1479, 1496, 1507, 1547, 1548, 1549, 1551, 1552, 1553, 1554, 1555, 40496, 40514, 40515, 40516, 40518, 40519, 40520, 40645, 40646, 40647, 40648, 40649, 40650, 40677, 40678, 40680, 40690, 40693, 40706]',\n",
       " 'Duplicate of 40979': '[20, 1022]',\n",
       " 'Too easy (perfect score decision tree)': '[24, 1466]',\n",
       " 'Duplicate of 40984': '[36, 958]',\n",
       " 'Binarized version of multiclass dataset': '[51, 293, 720, 741, 818, 867, 897, 953, 954, 955, 957, 959, 960, 961, 962, 963, 964, 965, 966, 967, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 983, 985, 986, 987, 988, 989, 990, 991, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1023, 1025, 1026, 1178, 1216, 1217, 1218, 1219, 1220, 1524, 40597, 40683, 40702, 41434, 41964, 41966, 41967, 41973, 42717, 42733]',\n",
       " 'Too large': '[152, 153, 154, 156, 157, 158, 159, 160, 161, 162, 274, 1169, 1235, 1236, 1237, 1238, 1240, 1502, 1503, 4541, 40517, 41039, 41147, 41150, 41167, 41228, 41672, 41708, 41710, 41711, 41714, 41717, 41728, 41729, 41732, 41735, 41738, 41741, 41743, 41745, 41747, 41748, 41749, 41751, 41752, 41755, 41757, 41761, 41770, 41771, 41772, 41774, 41775, 41776, 41778, 41786, 41789, 41793, 41794, 41795, 41796, 41798, 41800, 41802, 41807, 41808, 41809, 41810, 41813, 41814, 41817, 41819, 41822, 41823, 41836, 41837, 41840, 41845, 41850, 41852, 41853, 41854, 41855, 41861, 41869, 41876, 41880, 41887, 41890, 41892, 41895, 41949, 41991, 42344, 42395, 42396, 42435, 42468, 42718, 42732, 42750, 42769, 42803]',\n",
       " 'Duplicate of 1590': '[179]',\n",
       " 'Sparse format': '[273, 350, 351, 354, 357, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 1241, 1242, 4136, 4137, 41026]',\n",
       " 'Multi-label data': '[312, 40589, 40591, 40592, 40594, 40595]',\n",
       " 'Text data': '[373, 40945]',\n",
       " 'Grouped data': '[375, 1044]',\n",
       " 'Time series data': '[377, 1046, 1471, 1476, 1477, 40922, 40985]',\n",
       " 'Derived (non-original) data': '[378, 381, 382, 1037, 1038, 1040, 1041, 1042, 1119, 1226, 1525, 1526, 1557, 1558, 1560, 1566, 1568, 23512, 40664, 40666, 40926, 40981, 40992, 40993, 40997, 40998, 40999, 41000, 41001, 41002, 41003, 41004, 41005, 41006, 41007]',\n",
       " 'Label leakage': '[451, 23380]',\n",
       " 'Dataset not checked for triviality by DT!': '[458, 40927, 40975, 40978, 41081, 41082, 41142, 41143, 41144, 41145, 41146, 41156, 41158, 41159, 41160, 41161, 41162, 41163, 41164, 41166, 41440, 41526, 41671, 41701, 41705, 41707, 41709, 41712, 41713, 41715, 41716, 41718, 41719, 41720, 41721, 41722, 41723, 41724, 41725, 41726, 41727, 41730, 41731, 41733, 41734, 41736, 41737, 41739, 41740, 41742, 41744, 41746, 41750, 41753, 41754, 41756, 41758, 41759, 41760, 41762, 41763, 41764, 41765, 41766, 41767, 41768, 41769, 41773, 41777, 41779, 41780, 41781, 41782, 41783, 41784, 41785, 41787, 41788, 41790, 41791, 41792, 41797, 41799, 41801, 41803, 41804, 41805, 41806, 41811, 41812, 41815, 41816, 41818, 41820, 41821, 41824, 41825, 41826, 41827, 41828, 41829, 41830, 41831, 41832, 41833, 41834, 41835, 41838, 41839, 41841, 41842, 41843, 41844, 41846, 41847, 41848, 41849, 41851, 41856, 41857, 41858, 41859, 41860, 41862, 41863, 41864, 41865, 41866, 41867, 41868, 41870, 41871, 41872, 41873, 41874, 41875, 41877, 41878, 41879, 41881, 41882, 41883, 41884, 41885, 41886, 41888, 41889, 41891, 41893, 41894, 41896, 41897, 41898, 41919, 41945, 41946, 41982, 41983, 41986, 41988, 41989, 41990, 42140, 42141, 42178, 42192, 42343, 42477, 42530, 42532, 42638, 42641, 42670, 42738, 42768, 42882]',\n",
       " 'Unspecified target feature': '[470, 679, 40687, 40869, 40976]',\n",
       " 'Duplicate of 23380': '[473, 1024]',\n",
       " 'Binarized regression problem': '[715, 717, 718, 722, 723, 725, 727, 728, 734, 735, 737, 740, 742, 743, 749, 750, 751, 752, 757, 761, 766, 770, 772, 774, 779, 792, 795, 797, 799, 802, 803, 805, 806, 807, 813, 816, 819, 821, 823, 824, 825, 826, 827, 833, 837, 838, 839, 841, 843, 845, 846, 847, 849, 853, 855, 866, 869, 870, 871, 879, 881, 884, 886, 888, 896, 901, 903, 904, 910, 912, 913, 914, 917, 920, 923, 926, 930, 931, 934, 936, 937, 940, 943, 949, 981, 993]',\n",
       " 'High-dimensional': '[1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1104, 1106, 1107, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1144, 1145, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1233, 1457, 1458, 41084, 41103, 41157, 41165, 42758, 42759, 42763, 42766, 42773, 42775, 42809, 42810, 42811, 42812, 42819, 42820, 43072]',\n",
       " 'Unknown origin': '[1222, 1443, 1444, 1451, 1452, 1453]',\n",
       " 'Duplicate of 40994': '[1467]',\n",
       " 'OK': '[1478, 1480, 1485, 1486, 1487, 1489, 1491, 1492, 1493, 1494, 1497, 1501, 1510, 1515, 1590, 4134, 4135, 4534, 4538, 6332, 23381, 23517, 40499, 40536, 40668, 40670, 40701, 40705, 40923, 40966, 40971, 40979, 40982, 40983, 40984, 40994, 40996, 41027]',\n",
       " 'Duplicate of 40982': '[1504]',\n",
       " 'Duplicate of 40945': '[40704]',\n",
       " 'Possible duplicate': '[42193, 42345, 42493, 42734, 42804, 42805, 42883]'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = {}\n",
    "for key, value in sorted(data_status.items()):\n",
    "    v.setdefault(value, []).append(key)\n",
    "{k:str(v) for k,v in v.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}