{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook presents example usage of package for solving classification problem on `seismic-bumps` dataset. You can access dataset [here](https://raw.githubusercontent.com/adaa-polsl/RuleKit/master/data/seismic-bumps/seismic-bumps.arff).\n",
    "\n",
    "This tutorial will cover topics such as:   \n",
    "- training model   \n",
    "- changing model hyperparameters   \n",
    "- hyperparameters tuning   \n",
    "- calculating metrics for model    \n",
    "- getting RuleKit inbuilt   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genergy</th>\n",
       "      <th>gimpuls</th>\n",
       "      <th>goenergy</th>\n",
       "      <th>goimpuls</th>\n",
       "      <th>nbumps</th>\n",
       "      <th>nbumps2</th>\n",
       "      <th>nbumps3</th>\n",
       "      <th>nbumps4</th>\n",
       "      <th>nbumps5</th>\n",
       "      <th>nbumps6</th>\n",
       "      <th>nbumps7</th>\n",
       "      <th>nbumps89</th>\n",
       "      <th>senergy</th>\n",
       "      <th>maxenergy</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.584000e+03</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.0</td>\n",
       "      <td>2584.0</td>\n",
       "      <td>2584.0</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "      <td>2584.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.024252e+04</td>\n",
       "      <td>538.579334</td>\n",
       "      <td>12.375774</td>\n",
       "      <td>4.508901</td>\n",
       "      <td>0.859520</td>\n",
       "      <td>0.393576</td>\n",
       "      <td>0.392802</td>\n",
       "      <td>0.067724</td>\n",
       "      <td>0.004644</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4975.270898</td>\n",
       "      <td>4278.850619</td>\n",
       "      <td>0.065789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.292005e+05</td>\n",
       "      <td>562.652536</td>\n",
       "      <td>80.319051</td>\n",
       "      <td>63.166556</td>\n",
       "      <td>1.364616</td>\n",
       "      <td>0.783772</td>\n",
       "      <td>0.769710</td>\n",
       "      <td>0.279059</td>\n",
       "      <td>0.068001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20450.833222</td>\n",
       "      <td>19357.454882</td>\n",
       "      <td>0.247962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000e+02</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>-96.000000</td>\n",
       "      <td>-96.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.166000e+04</td>\n",
       "      <td>190.000000</td>\n",
       "      <td>-37.000000</td>\n",
       "      <td>-36.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.548500e+04</td>\n",
       "      <td>379.000000</td>\n",
       "      <td>-6.000000</td>\n",
       "      <td>-6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.283250e+04</td>\n",
       "      <td>669.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>30.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2600.000000</td>\n",
       "      <td>2000.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.595650e+06</td>\n",
       "      <td>4518.000000</td>\n",
       "      <td>1245.000000</td>\n",
       "      <td>838.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>402000.000000</td>\n",
       "      <td>400000.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            genergy      gimpuls     goenergy     goimpuls       nbumps  \\\n",
       "count  2.584000e+03  2584.000000  2584.000000  2584.000000  2584.000000   \n",
       "mean   9.024252e+04   538.579334    12.375774     4.508901     0.859520   \n",
       "std    2.292005e+05   562.652536    80.319051    63.166556     1.364616   \n",
       "min    1.000000e+02     2.000000   -96.000000   -96.000000     0.000000   \n",
       "25%    1.166000e+04   190.000000   -37.000000   -36.000000     0.000000   \n",
       "50%    2.548500e+04   379.000000    -6.000000    -6.000000     0.000000   \n",
       "75%    5.283250e+04   669.000000    38.000000    30.250000     1.000000   \n",
       "max    2.595650e+06  4518.000000  1245.000000   838.000000     9.000000   \n",
       "\n",
       "           nbumps2      nbumps3      nbumps4      nbumps5  nbumps6  nbumps7  \\\n",
       "count  2584.000000  2584.000000  2584.000000  2584.000000   2584.0   2584.0   \n",
       "mean      0.393576     0.392802     0.067724     0.004644      0.0      0.0   \n",
       "std       0.783772     0.769710     0.279059     0.068001      0.0      0.0   \n",
       "min       0.000000     0.000000     0.000000     0.000000      0.0      0.0   \n",
       "25%       0.000000     0.000000     0.000000     0.000000      0.0      0.0   \n",
       "50%       0.000000     0.000000     0.000000     0.000000      0.0      0.0   \n",
       "75%       1.000000     1.000000     0.000000     0.000000      0.0      0.0   \n",
       "max       8.000000     7.000000     3.000000     1.000000      0.0      0.0   \n",
       "\n",
       "       nbumps89        senergy      maxenergy        class  \n",
       "count    2584.0    2584.000000    2584.000000  2584.000000  \n",
       "mean        0.0    4975.270898    4278.850619     0.065789  \n",
       "std         0.0   20450.833222   19357.454882     0.247962  \n",
       "min         0.0       0.000000       0.000000     0.000000  \n",
       "25%         0.0       0.000000       0.000000     0.000000  \n",
       "50%         0.0       0.000000       0.000000     0.000000  \n",
       "75%         0.0    2600.000000    2000.000000     0.000000  \n",
       "max         0.0  402000.000000  400000.000000     1.000000  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from rulekit.arff import read_arff\n",
    "\n",
    "DATASET_URL: str = (\n",
    "    'https://raw.githubusercontent.com/'\n",
    "    'adaa-polsl/RuleKit/refs/heads/master/data/seismic-bumps/'\n",
    "    'seismic-bumps.arff'\n",
    ")\n",
    "\n",
    "df_full: pd.DataFrame = read_arff(DATASET_URL)\n",
    "df_full['class'] = df_full['class'].astype(int)\n",
    "df_full.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision class distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGLCAYAAABa0JF/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABE1klEQVR4nO3dd3xc1YE2/ufe6UVt1Ltsucu9G4PBBWxjShqQhBACm2R3k81mQ3bfN/tu/2V3s5stIbRAIBASIJRgOhiDwQ033Lst2apWb9Prvff3h4yxLMmW7Zk5U57v56OP7JmR9EiyNY/OOfccSdM0DURERJS2ZNEBiIiISCyWASIiojTHMkBERJTmWAaIiIjSHMsAERFRmmMZICIiSnMsA0RERGmOZYCIiCjNsQwQERGlOZYBIiKiNMcyQEQJbfPmzbj11ltRUlICSZLw+uuvi45ElHJYBogooXm9XsyYMQOPPvqo6ChEKUsvOgAR0cWsXr0aq1evFh2DKKVxZICIiCjNsQwQERGlOZYBIiKiNMcyQERElOZYBoiIiNIcryYgooTm8XhQV1d37u/19fXYv38/HA4HKioqBCYjSh2Spmma6BBERCPZuHEjli5dOuT2e++9F7/97W/jH4goBbEMEBERpTmuGSAiIkpzLANERERpjmWAiIgozbEMEBERpTmWASIiojTHMkBERJTmWAaIiIjSHHcgJEoBgbCCLncQXZ4gus++7nIH0X32db8vjIiqIaJqUFQVEUWDop590TRsMv4VIMlnX3SAzgAYbYDRDpjsgCkDMGYM/Pmz26y5QEYJkFEEZBQDeqPoLwMRXSGWAaIE1+8L4WibC829voEnfHcQ3Z7QoCd7dzBydR/E3HCVKSXA6hgoB5nFAwUhsxRwVAN54wdejLar/BhEFCssA0QJQtM0NPT4cKzNhWNtLhxtHXjd6gyIjjYKGuDrGXjpODTM/dJAOcgbD+RN+Px1wRTAnh/3tEQ0GLcjJhLAH1JwvN2Fo+c98Z9od8MbUoTkaTB/XcjHBTBQEopnAiUzgZJZQOmcgVEGIooblgGiOKjtcGPTyS7sa+7HsTYXGrq9UBPof57QMjAcRzVQNg8onwdULQHyJ4hORJTSWAaIYsATjGBrbTc2nezC5pNdONPvFx3pohKuDFwoowQYez0w9oaBl4wi0YmIUgrLAFGUHGl1YtPJLmw80YV9TX0IK8nzXyvhy8CF8iaeLQbXA2OuH7i6gYiuGMsA0RVy+sLYXDvw5L+ltgud7qDoSFcs6crA+XQmoHopMPlWYOLNXG9AdAVYBoguw8kON9471I6NJztxsMUJJZEm/q9CUpeB80k6oPIaYPJtwORbgMwS0YmIkgLLANEldHuCeGN/K9bubcGRVpfoODGRMmVgEGngyoSaLwDT7wLsBaIDESUslgGiYQQjCj442oG1e89g88kuRFJkBGAkqVkGziPrgXE3ArPuBiasGthhkYjOYRkgOs/hM078YVcT3jrQClfgKnf1SyIpXwbOZ80Dpt8JzLwbKJoqOg1RQmAZoLTnC0Xw1oFWvLCzCQdanKLjCJFWZeB8xTOAWfcAM746cP4CUZpiGaC0dbTVhRd2NeKNfa1Xv7d/kkvbMvAZUyYw8+vA/O8CudWi0xDFHcsApZ2NJzrx6Md1+LShT3SUhJH2ZeAcCR/MfhS2mpW4pjpPdBiiuOFBRZQ2PjregYc21GF/c7/oKJSgVEsuHthlh3vbTkwrzcJ3l4zFzdOKoZMl0dGIYoojA5TyPjjagYc/qsXBNF0PMBocGRiws/zbuKt22aDbyh0W/MniMbhrXgUsRp2gZESxxTJAKUnTNLx/ZKAEpOreANHEMgBoOiNWS4/huMc67P3ZVgPuXVSF7y4ZC5uJg6qUWlgGKKVomob3DrfjoQ21ON7uFh0nabAMAI1lt+H6uq9e8nF5dhP+asV4fG1+BacPKGWwDFBKUFUN7xxqw8Mf1eJkh0d0nKTDMgDcr/8ZPvJUjvrx4wrs+MmqSVgxpTCGqYjig2WAkpqqanjrYCse/qgOdZ0sAVcq3cvAEaUSa8I/u6K3XTQ2F3+3ZjKmlmZFORVR/LAMUNLa29SHv3/tMI62cU3A1Ur3MvD3+D6eCyy+4reXJOD2GSX4m1WTUJptiWIyovhgGaCk4/SF8R/rjuHFT5vBf73Rkc5loEfKxTz/g1Bx9VcKmPQyvrW4Ct9fOg6ZZp5/QMmDS2Ipqbyyuxn/8d5x9HhDoqNQingVy6NSBAAgGFHxxKbTeGV3C36wbBy+sbASBp0clfdNFEscGaCkcLLDjb977RB3DYyRdB0ZCMCERcGH0afZY/L+x+TZ8H9XTcSqqcUxef9E0cKRAUpo/pCCX26oxVNbTqf8McIUfx/J18SsCABAfbcXf/bcXiybVID/+NI0FGSaY/axiK4GRwYoYX1wtAP//OZhnOkPiI6S8tJxZECFhDXh/8IxpSQuHy/basC/3FaD22eWxuXjEV0OjgxQwmnp8+Gf3zyCD491io5CKWy/PC1uRQAA+n1h/PDF/Vh/pAM//cJUOGzGuH1sokthGaCEEVZUPLWlHr/ccBKBsCo6DqW4JyOrhXzcdw61YWd9L372pWm4kRsWUYLgNAElhIZuL77/wl6eIyBIuk0TNMtluM73c9Ex8KXZpfjn22p4GSIJx2teSLi3D7ZizUObWQQobl5QbxIdAQCwdu8ZrPzFZmyp7RIdhdIcRwZImEBYwT++fhAv72kVHSXtpdPIgEvKwDz/IwgisX4bv3tBBf5uzWRYjZy9pfjjyAAJcarTjVX/+xGLAMXdO9INCVcEAOD5nU1Y9eAW7KrvFR2F0hDLAMXdiztOY/WDm9DQx10EKb7C0OOhwCrRMUbU1OvDV3+9Hf/2zlFEFC6ipfjheBTFTSCs4IHnd+Ld430AeA48xd82eR7a1BzRMS5K1YAnt9TjSKsLj909G9lWXoJIsceRAYqL2nYnVvz8g7NFgEiMR0JrREcYtW2nenDbI5+gtsMtOgqlAZYBirnnt9VhzUNb0OJWREehNHZcnoBPI2NFx7gsTb0+fPGxbdhwrEN0FEpxLAMUM4Gwgj97Zhv+7s0TCKmcFiCxnlESd63AxXiCEXznd7vx2MY60VEohXHNAMVEc48HX3t8K0cDKCF0yfl4yTdfdIwrpmrAz9edwIl2N/7zy9NhNkTnyGWiz3BkgKJuR20bVv9iI4sAJYxXtBVIhR93b+xvxV1PbEeHi4d3UXQl//8OSijPbTyEe57eDU+E0wKUGPySBb8KLBcdI2oOtDhx2yNbcaC5X3QUSiEsAxQViqLgH5/fiH9Y14iwxn9WlDg+kBbDrVlFx4iqDlcQdz6xHa/vOyM6CqUI/tSmq+b2eHHPg2/hd4e80Lh/ACUQFTIeCt4sOkZMBCMq/uql/fjZe8egqtxVnq4OywBdldauHnzlF+9hW1fibe9KtEc3A3VKkegYMfXEptP47u/3IBjhGh26ciwDdMVO1Lfgzoc+wgmvRXQUomE9HkrNUYELfXisA3/y293wh1gI6MqwDNAV2XnoBL7x5Da0hFNrLpZSR4NciQ3hGtEx4mZrXTe++fROeIIR0VEoCbEM0GXRNA3vbdmN7zx/EF2qTXQcohE9r94kOkLcfdrQh7uf3AGnLyw6CiUZlgEaNVVV8fy7m/DjtxvgAkcEKHE5pSw8G7hWdAwhDrQ48dUnd6DHExQdhZIIywCNSiQSwRN/XI+fbuqBT+IaAUpsb0hLEUL6Lmo91ubCnU9sRyc3J6JRYhmgSwqFwnji5ffw4G4vgrJZdByiiwrDgEf8yXkOQTSd6vLiq0/uQJebIwR0aSwDdFH+QAC/evFNPLrPxyJASWGzvACdWqboGAnhdJcXX+eUAY0CywCNyO3x4rHn1uLJg0H4dHbRcYguSYOER0JrRMdIKLWdHtz91E70eUOio1ACYxmgYfU5XXjs93/E744r8OizRMchGpWj8iTsi1SKjpFwjre78Y3f7ORVBjQilgEawu3x4qkXXsPLtSqceofoOESj9htltegICetIqwv3PL0TrgALAQ3FMkCD+AMB/PaVN/HH2iB6jAWi4xCNWrtchLXBuaJjJLSDLU7c/8yn3LqYhmAZoHNCoTCeW/sOXjzYhw5jqeg4RJflZW2F6AhJYXdjH37y6iHRMSjBsAwQgIEjiF9+ez2e39WCMybOuVJy8UlWPB5YJjpG0nht3xk8+nGd6BiUQFgGCJqm4Y31G/Hs5hNoMFeLjkN02dZJS+DTeOnr5fjv9Sfw3qE20TEoQbAMpDlN07Bu4zb8Zv1e1JknAJBERyK6LAp0eCjAhYOXS9OAB14+gEMtTtFRKAGwDKS5Lbv24sl3tuGYaRI0FgFKQrvkWWhQ80XHSEr+sIJv/+5TdHDb4rTHMpDGdu0/jKfWrsch/Xio/KdASerx8M2iIyS1DlcQ3352N/whXmGQzvgMkKYOHa/FMy+/if0Yg3AaH+hCye2UPAabwpNEx0h6h8448cDL+6FpmugoJAjLQBqqa2jC0y+9joOhArikDNFxiK7Y79WVoiOkjPcOt+O/158QHYMEYRlIM82t7XjyhbU42ifhjL5YdByiK9Yn5eD3gcWiY6SURz8+hbV7W0THIAFYBtJIn9OFp/7wGk60OXHaMlF0HEpwZ1wqvrHWj9yfu2H5Nxem/cqD3a0Xn1cORjT83YYAKh90w/SvLlQ96MbT+z4/IOeDUxFMeNiDzJ+5cM9rfoSUz4elnQENEx72oLFfHVW+17AcCnRX9snRiH6y9hB2N/SKjkFxphcdgOIjEongxTfW4WhdIxodCxBW2ANpZH1+DYuf9mLpGD3eu9uKfKuE2l4VOeaLX3Fy5x/96PBo+M1tFoxzyGhzq1DPPt+rmoavr/Xjb681YmW1Hl95xY9f7wnjL+YbAQA/+TCAP5trQGX2pf9tBmHCo4Ebr/rzpKFCERV/+vs9eP37i1HusIqOQ3HCMpAm1m/ejq2f7kNX3gz0hY2i41CC+89PgijPkvHM7ZZzt43JufiT9Lq6CDY1RHD6hxlwWAZKQ9V5T+zdPg3dPg3fm2eEWS/htgl6HOsaGGnY1hzBp60KHrl5dBsHbZQXokfjepdY6fGG8P0X9mLtn18DvY6/OKQDfpfTwMFjJ/H6+x/DnVGFhnCm6DiUBN48EcHcYh3ueMWHgv9yY9YTHjy5J3SJtwljbokOP/8kiNL/dWPCwx789foA/OGBoYF8q4Riu4T1pyLwhTVsaVIwvVCHsKLhz98J4IlbLNDJl97rQoOEh4NrovJ50sgOtjjx0IZa0TEoTlgGUlxnTy9eeP09dIf0OKLy8CEandN9Kn61O4TxDhnvf8OKP59rxF+uC+DZ/SMXgtN9KrY2KTjcqeK1u6x4cJUJfzwaxvfeHdjQRpIkvHyHBT/dHETNYx7MKpJx/ywD/mNrCEur9DDrgcVPezHxEQ8e2TXyxzko1+CwUhb1z5mGenTjKexr6hMdg+KA0wQpLBgM4fm176KuuR21OQuhKNxhkEZH1YC5JTr8+/KBYftZxToc7lTx+J4w7p05/DSTqgGSBDz/JQuyzBIAHf53JfCVl/147GYzLAYJ11bo8el37Ofe5mSPgt8dDGPfn9qw5BkvfrjAiNXj9Zj6mBdLKnWYXjh0geBTEW49HC+KquGBlw/gnb+8FlYjny5SGUcGUpSmaXjrw03Ytf8Q2nNnwa3wPzKNXnGGhCn5g388TM6T0eQceaV/cYaM0gzpbBH4/G00AC2u4d/uT98O4H9uMkHVgH3tKu6oMaDAJuP6Kh02NQy9cqFVLsZboVlX9knRFanv9uLf3jkmOgbFGMtAitp98Cje/WgrnNkT0BK2iY5DSWZxuQ4negY/gZ/sUVGZNfKPjMXlOrS6NXhC2qC3kSWgLHPo2/1mbwgOi4TbJhqgnP1QYeXz18owu+H9QbvpCj4bulrP72zCxyc6RcegGGIZSEEtbR148Y334IIFx5RC0XEoCf1ooQk7WhT8+5Yg6npVvHAojF/vDeH78z6fIvjbDwP45mv+c3//+jQDcq0S7nvDj6NdCjY3RvA3HwRx/0wDLIbBU1SdXhX/uiWIh1cPTEPkWCRMzpPx4I4QtjdHsKE+gsXlg0ezPJIdT/mXxvCzpov5P388iD7vxReRUvJiGUgxPn8Az7/2Ds50dKHBMpEnEdIVmVeqw2t3WfCHw2FMfcyDn24O4sGVZtw9/fNzLNo82qBpA7tRwgf3WNEf0DD3117cvdaPWyfo8dDqoZcL/nBdAD9eZEJJxuc/gn77BQtePBLGLX/w42+uMWFe6eD1Au9KS+AHL4sVpcsdxN+uPSQ6BsWIpPFkipShaRqef+1dvP3hZkSKarDblys6EiWJBvPXRUe4qAh0WBp6EM0q/02L9t93zMBX5vBqjlTDkYEUsnXXPqzfvB05hSU44M8RHYcoanbIc1kEEsS/vHkELX0+0TEoylgGUkTTmTa88s56mIwGHNFKEdb4raXU8Vj4ZtER6Cx3MIIfv3wAqspB5VTCZ4wUEIlE8Md3P0RnTx8Uxxg0BSyXfiOiJFErV2NbeLzoGHSenfW9eGrradExKIpYBlLAJ7v3Y++hoygrK8MOV7boOERR9Vt1legINIz/Xn8SJzvcomNQlLAMJLmunj68uX4TzCYzjoQL4VN5pCuljh4pF38ILBQdg4YRiqj4l7eOiI5BUcIykMQ0TcOb6zeipb0D5vwKHPfxuFFKLa9iOVSw4CaqT+p68OHRDtExKApYBpLYviPHsfXTfSgtKsInrhzuKUApJQAzHgvcKDoGXcK/v3sMYWXkbaopObAMJCmP14fX130ERVXRpCtGX8Rw6TciSiIb5GvQr3Er7UR3utuL329vFB2DrhLLQJJ6f9M2nDjViNySSuxzZ4iOQxRVKiQ8HFwjOgaN0i831KLfx62KkxnLQBKqa2jCB5u3oyDPgZ2eXCicHqAUs1+ejuNKsegYNEpOfxgPflgrOgZdBZaBJBMKhbH2vY/g8njhtZWgNWQSHYko6n4dWS06Al2m53Y04lSXR3QMukIsA0lm88492H/0OCrLS7HHkyk6DlHUNctlWBeaLjoGXaaIquHf3zkmOgZdIZaBJNLe1YO3N2yG3WpDi+qAk4sGKQW9oN4kOgJdoQ3HO7Gltkt0DLoCLANJQlVVvL5uA9o6u1FcVIC9HrvoSERR55Iy8HTgetEx6Cr869vHoPDcgqTDMpAk9hw6hu17D6KytBjH/RnwKnrRkYii7m1pKYLgiFcyO9HhxoufNomOQZeJZSAJBIMhvPfxVmgaYLZl4ABHBSgFhaHHw4GVomNQFPzig5NwB8KiY9BlYBlIArv2H8aJUw2oKC3GYa8NAZ4/QCnoE3k+2tQc0TEoCro9ITzycZ3oGHQZWAYSnM8fwPubt8FoNEAymHCIowKUoh4JcZOhVPLbTxrQ6Q6IjkGjxDKQ4Lbt3o9TDc0oLynCEa8NYY3fMko9x+UJ2B0ZIzoGRVEwouLprQ2iY9Ao8ZklgbncHqzfvB02mxXQGXGEowKUop5RVomOQDHw/I5GuLh2ICmwDCSwLbv2oulMO8qKC3HUa0OQowKUgjrlfLwUnC86BsWAOxjBczt4iFEy4LNLgurtd+LDrTuRnZUBTdLjsJent1FqekW7EfxRlLqe3tqAQFgRHYMugf8DE9S23fvR1tGF4oJ8HPNZeQUBpSS/ZMHjgWWiY1AMdXuCeGVPi+gYdAksAwmo3+XGhk92ITsrE5os8woCSlnrpWvh1qyiY1CMPbn5NHclTHAsAwlo+54DaOvoRnFBHk77LfBzVIBSkAIZDwVvFh2D4qCp14d1h9tFx6CLYBlIMC63Bxu27kRmhh06nQ7HuVaAUtRe3QycUgpFx6A4+e22etER6CJYBhLM9r0Hcaa9EyVF+egN69EZNoqORBQTj4c4KpBOPm3ow+EzTtExaAQsAwnE4/Vhw9adsNtt0Ot0OO7jXCqlpga5EhvCNaJjUJw9/QlHBxIVy0AC+fTAETS3tqO0MB8RDTjFMkAp6jn1JtERSIC3D7Shyx0UHYOGwTKQIBRFwZZde2E2maDX61Hvt3CTIUpJTikLvw1cJzoGCRBSVLywk8cbJyI+2ySIk6cbcbqpBYUFuQDAKQJKWW9IyxCBXnQMEuS5nY0IRVTRMegCLAMJYs/BYwiGQrBbregL69ERMomORBR1IRjxiH+l6BgkUJc7iA+PdYiOQRdgGUgAfU4Xdh04hLycgbPcT3BUgFLUFnk+OrVM0TFIsDf3t4qOQBdgGUgAB46eRGdPH/Ic2VA0oJZlgFKQBgmPhNaIjkEJ4OMTnfAEI6Jj0HlYBgRTFAWffLofZqMROp2OCwcpZR2RJ2FfpFJ0DEoAwYiK9Ue4I2Ei4bOOYHUNzahrbEJRQR4ALhyk1PUbZbXoCJRA3jrAqYJEwjIg2O6DRxAIBGG3WdEf0aGdCwcpBbXLRXgtOFd0DEogW+u60ecNiY5BZ7EMCOR0e7Br/2Hk5mQDAE7wHAJKUS9pN4qOQAkmrGh4j4cXJQyWAYH2HzmOzu5e5OfmQNOAOr9FdCSiqPNKVjwRWCo6BiUgThUkDpYBQVRVxbbdB2A8u3CwK2zgUcWUkt6XlsCnmUXHoAS0s74Hna6A6BgElgFh6hqaUVvfhOKzCwebAvxhSalHgQ6/DPB0QhqeqgFvH2wTHYPAMiDM7oNH4PMHYLcNXD3QHOTCQUo9u+RZaFTzRMegBPbWQU4VJAKWAQFc5xYOZgEAvIqMnrBRcCqi6Hs8zFEBurh9Tf1o7vWJjpH2WAYEOFnfhK6ePuTnDmw/3MwpAkpBp+Qx2BSeJDoGJQFOFYjHMiDAiVP1UDUNev3AyW1NnCKgFPQ7dZXoCJQk3uRVBcKxDMRZOBzBwaO1yMqwAwAUDWhlGaAU0yvl4LnANaJjUJI41ubCqS6P6BhpjWUgzhpaWtHZ0wtH9sB6gdagCRGeRUAp5nUshwJeKkujt7W2W3SEtMZnoTirrW9CIBiExTwwGsCrCCjVBGHCI4GbRMegJLOrvld0hLTGMhBHmqZh/5HjMJtMkCQJABcPUurZKC9Cr2YXHYOSzE6WAaFYBuKos6cXTa3tcJy9pLAvrIdb0QtORRQ9GiQ8FFwjOgYloW5PEHWdXDcgCstAHNXVN8Hp9pxbPMhdBynVHJRrcEQpFR2DkhSnCsRhGYijo7WnIUsSZHngy871ApRqnoqsFh2BktjO+h7REdIWy0Cc+PwBHDl5CjnZmQCAoCqhI8RdByl1nJFL8FZolugYlMQ4MiAOy0CcnGpsRndvPxxZA+sFOkJGaJAEpyKKnhdVXkFAV6fNGUBTD7cmFoFlIE5q65ugKAqMRgMAoDtsEJyIKHo8kh1PBW4QHYNSwA5OFQjBMhAHqqpi3+Hj504oBIBuThFQCnlHuh5+8N80Xb2dpzlVIALLQBy0tHWgrbPr3K6DAEcGKHVEoMPDAS4cpOjgIkIxWAbioK6hGV6f/9zIgE+R4VO5VSulhh3yXLSoDtExKEW09PnR2u8XHSPtsAzEQdOZdsiyfG7XwR6OClAKeTTETYYoujg6EH8sAzGmaRpONTbBZrWcu41TBJQqTsrjsD0yTnQMSjFcNxB/LAMx1u9yo7vPOXjxIMsApYhn1ZWiI1AK2tXAMhBvLAMx1tbZDY/XN6gMcJqAUkG3lIc/BBaKjkEpqKHbi1BEFR0jrbAMxFh7ZzciEQVGw0ABCCgyPDyciFLAq1gOFVwIS9GnakBjj1d0jLTCMhBjZ9o7IZ230SCnCCgVBGDGrwIrRMegFHa6m2UgnlgGYkjTNNTWN8HG9QKUYjbI16Bfs4mOQSnsdBfLQDyxDMSQ0+1Bd18/7FaWAUodKiQ8HOTlhBRb9d0e0RHSCstADLV3dsPj9cJu+/yyQi4epGS3T56O40qx6BiU4uo5TRBXLAMx1NbZjXBYgck4sGd7QJXg5uJBSnJPRrj1MMUepwnii2UghgYWD36+erCfowKU5JrlMqwLTRcdg9JAjzcEpz8sOkbaYBmIkYGdB5thtZrP3eZReBkWJbfnuckQxRGnCuKHZSBGXB4vOrt7By0eZBmgZOaSMvFMYInoGJRGTndxEWG8sAzESPvZnQcz7CwDlBrelm5AEJzqovjhyED8sAzESHtXN0KRyLmdBwGWAUpeYejxcIBTBBRf3HgoflgGYqTP6QY0bdACQpYBSlafyPPRpuaIjkFphlcUxA/LQIx09/ZBpxv85O9lGaAk9UiImwxR/DX2eKFpmugYaYFlIEY6u3thMhnP/T2oSghr/HJT8jkmT8TuyBjRMSgN+UIK2l0B0THSAp+dYkBVVfT0O2E2fl4GfBwVoCT1jLJKdARKY+1OloF4YBmIAbfXB78/MGhkwK/yS03Jp1POx8vBeaJjUBrr58ZDccFnqBhwutwIhkIsA5T0XtZuBH9MkEj9vpDoCGmB/8tjwOn2IBgMDZom8HOagJKMT7LgicAy0TEozfX7ODIQDywDMeD2eKFq2qCrCTgyQMnmA+lauDXrpR9IFEN9LANxwWeoGPD4/ENuC7AMUBJRIOOh4M2iYxDByWmCuOAzVAz4fH5ceGUspwkomezRzcQppVB0DCKODMQJy0AMON0eyOftPAhwZICSyxMhjgpQYuDVBPHBZ6gY6HO6Bp1JAACqoCxEl6tBrsSG8BTRMYgAAL5gRHSEtMAyEAP9LjcMBv2g27ihJiWL36s8kIgShz+siI6QFlgGokxVVbg8XhguGBnQNGmEtyBKHP1SNp4NXCs6BtE5/hDLQDywDESZzx9AKBSGQc+RAUo+b0hLEYH+0g8kihMfy0BcsAxEWSAYQkRRoNcPvnqAawYo0YVgxKN+ThFQYuE0QXywDESZpmnQNA3SBVcTcJqAEt1meQE6tUzRMYgG4TRBfLAMRJmqqcBwZUBQHqLR0CDhYV5OSAkopKiIKBxbjTWWgSjTtIGXC8cBWAYokR2RJ+FApFJ0DKJhRVT+BI01loEoU1UVGjhNQMnlN8pq0RGIRmTS86kq1vgVjrKBNQPD3B7/KESj9lpwrugIRMMy6uUhv1xR9LEMRJmmacAwIwOc8SIiunxmjgrEBb/KUXZuZOCCIsuRASKiy2c28JC3eGAZiLJzlxaCawaIiK4Wy0B8sAxEmXp21euFU1wcGSAiunxmA5+m4oFf5SgbcdMhQXmIiJKZSc+RgXhgGYgyVRu4tPDCRQN6iXWAiOhycWQgPvhVjrLPFhBeOE1glnk9ARHR5eKagfhgGYgyVR3YgvDCBYQsA0REl4/TBPHBMhBlmqYNuwMhywAR0eXjNEF88KscZdpw2w+CZYCI6EpwmiA+WAaizGI2Qa/TIxyJDLqdZYCI6PJxZCA++FWOMovFDINBjwjLABHRVTNzzUBcsAxEmdVshkHPkQEiomiwGFkG4oFlIMqsFpYBIqJoKcw0i46QFlgGokyWZdjtVoTDF5QBnSIoERFR8ip3WEVHSAssAzGQZbcPGRmwcGSAiOiyleVYREdICywDMZCVaR8yMmBiGSAiuiySxDIQLywDMZCVkQFFGTwtoJcAg8RCQEQ0Wvl2E3cgjBOWgRiwWodf8MLRASKi0eN6gfhhGYgBq9k89KQicN0AEdHlKOcUQdywDMSA1WIGhtmW2MorCoiIRq0shyMD8cIyEANWixmSJEFVB48E5OgjI7wFERFdqNzBkYF4YRmIgZE2HsoxhAUlIiJKPuUcGYgbloEYsJgHzie48PJCB0cGiIhGjdME8cMyEANWixn6YUYGsvQRyBj+iGMiIvqcTpZQks2tiOOFZSAGMu02WC1m+APBQbfLEpDN0QEioksqyjRDr+NTVLzwKx0DOp0OFSVF8Pn8Q+7jugEiokvjzoPxxTIQIxUlRQiFhz7x84oCIqJL44ZD8cUyECP5uQ4AgHbBfgMOjgwQEV3SpKIM0RHSCstAjBTkOWDQ64eMDnBkgIjo0mZV5IiOkFZYBmIkPzcHVqsF3gvWDdh1Cg8sIiK6CKNOxtTSTNEx0grLQIxkZ2YgOzMDPn9g0O2SxNEBIqKLqSnN5GmFccYyECOSJKGytHjIyADAKwqIiC5mVjmnCOKNZSCGSosLoChDDyfiyAAR0chmV2aLjpB2WAZiqCDXAQ28ooCI6HLM5uLBuGMZiKGCXAfMJuOQnQgHygC3JSYiulBRphkl2dxwKN5YBmIoPzcHNosFPv/gdQNmWeNUARHRMGZVZIuOkJZYBmLIZrUgNycL3guuKACAElNwmLcgIkpvnCIQg2UghiRJQlVZKfzDlIFiY0hAIiKixMbFg2KwDMRYcWEeVGXoJkPFpiAkrhsgIjrHqJNRU5IlOkZaYhmIseKCfMg6GeHI4DUCJlmDg+sGiIjOmVySCbOBmw2JwDIQY5VlxcjKsMPp8gy5r5jrBoiIzpnNxYPCsAzEWHZmBspLiuB0uYfcxzJARPQ5Hk4kDstAHNRMrEYgOPSJv9gYgsx1A0REkCXgmupc0THSFstAHFSWFkOn0w05ztgoayjiVQVERJhRno08u0l0jLTFMhAHlWUlyM7MgNM9dN1AmXnoZYdEROlmxeRC0RHSGstAHGRl2FFZVgync+i6gXKuGyAiwo1TWAZEYhmIkynjqxEMDZ0SyDFEYNfxEkMiSl8VDismFGaIjpHWWAbiZGxFKYwGw7ALCcs4OkBEaWz55ALREdIey0CcjKkoRa4jG739riH3caqAiNLZjVwvIBzLQJyYTSZMnVA97H4DJaYg9NLQLYuJiFJdlsWAeWMcomOkPZaBOJo0bgxUTYOqDn7iN8gaqnhVARGloZU1hTDo+FQkGr8DcVRdVT6wNfEwlxiOt/oEJCIiEmvN9BLREQgsA3FVkOtARUkR+oZZN1BiDMEmKwJSERGJ4bAZsZi7DiYEloE4kiQJ06dMgD8QgKZpF9wHjOPoABGlkZU1hdBziiAh8LsQZxPHVsFiNsPr8w+5j1MFRJRO1kzjFEGiYBmIs+rKMoypKEVHd8+Q+7L1CvINPKuAiFJfnt2IRZwiSBgsA3EmyzIWzp4Ovz845KoCgKMDRJQeVk0tgk6WRMegs1gGBJg+eTxysjLQ5xy6kLDa4oeOxxoTUYq7e0Gl6Ah0HpYBAQrzcjF14jh0dvcOuc8ka6jgngNElMLmVzkwuThTdAw6D8uAIPNm1AAAwuGhhxRxqoCIUtm911SJjkAXYBkQpGbiOBTkOdDVM3R0oMwUhJl7DhBRCirOMmNlDc8iSDQsA4LYrBYsmDl12HUDsgSMswy99JCIKNndvaCCewskIH5HBJoxZSJMRiO8fu45QESpz6iX8bX5FaJj0DBYBgQaP6YCFWXF6OwaOlWQa4ggl3sOEFEKuWVaMXLtJtExaBgsAwLp9Xosmj0dHp9vyPbEADDV5hWQiogoNrhwMHGxDAg2bdJ4ZGdmoN/lHnJftcWPDN3Qqw2IiJLNzPJszCjPFh2DRsAyIFhpUQEmVY8Zds8BWQKm24ced0xElGy+xVGBhMYyIJgkSZg/swaKqiCiDL2ccILVBysvMySiJJZnN+HmacWiY9BFsAwkgGmTxqMg14GOrqGHF+kkYBpHB4goiX19fjmMej7dJDJ+dxJAZoYd1y+ci75+57CHF02y+rgJERElJYNOwt0LeQ5BomMZSBCL581EniMHncPsSGiQNdTwygIiSkIra4pQmGkWHYMugWUgQRTkOnDd/Fno6ukb9jLDKTYvDNLQUQMiokQlS8APlo0XHYNGgWUggVy3YA4c2Vno6u0bcp9J1jCFowNElES+OKsME4syRMegUWAZSCAlhfm4Zs4MdHb1jLgJkY6jA0SUBIx6GQ/cNEF0DBolloEEc/3COcjKsKO3f+gBRhadikk8s4CIksA3F1aiNNsiOgaNEstAgikvKcL8mdPQ3tk17OjAdLsHMobeTkSUKDLMevzFsnGiY9BlYBlIMJIk4fpFc2CzWuF0Dd1fwKZTMY6jA0SUwP7s+mpkW42iY9BlYBlIQNWV5Zg9bTJaO7uGvX8GRweIKEEVZppw/+IxomPQZWIZSECSJGHporkwG41weYZeQZClV7jvABElpB8unwCLUSc6Bl0mloEENWncGEyfPB5n2jqGvX9WhptnFhBRQhmbb8Nd88pFx6ArwDKQoCRJwrLF86HX6eDxDV0jYJQ1zM8cesUBEZEo/2flROhkSXQMugIsAwmsZkI1aiaOw5nW4UcHxln9KDIG45yKiGiomeXZWDWVJxMmK5aBBKbT6bD82gWQdTLcw6wdAIBrspyQuJiQiAT729WTREegq8AykOBm1UzEvBlT0Ximbdh9BxyGCBcTEpFQSyfmY8HYXNEx6CqwDCQ4WZZx+003wJGViY6unmEfMzvDDQsXExKRAAadhJ+sniw6Bl0lloEkUF5ShJuuX4Tu3n6EI5Eh93MxIRGJ8r0bxvEwohTAMpAkVly7EOOqytHY0jbs/eOtfhRyMSERxdHk4kxuO5wiWAaShN1mxW03Xg9FUeDxDr8dMRcTElG86GQJ//WV6TDo+DSSCvhdTCJzZ9Rg3owaNLa0DruYMNcQwWQuJiSiOPjeDdWYWpolOgZFCctAEpFlGV9YuRTZmRno6B5+MeHcDDfMXExIRDE0Lt+KHywbLzoGRRHLQJKpKC3GjUsWobunDxEuJiSiONNJwC/umg2jnk8fqYTfzSR043ULMbaybMTFhBOsfpSbAnFORUTp4LtLxmJaGacHUo2kDTf5TAlvx96DePTZl1BaVAC7zTrkfp8iY21XPgIqTw8TTQ360L/lOfhqt0P1OWEsGIucFd+FqXgCAKB/6/PwHtsCxd0FSdbDWDQO2Uu+CVPJxFG9f+eOV9C/6VlkzLkNjhXfPXd774Yn4T28AZLBjOzr74W9Zum5+7zHt8J7eAMKvvJP0f1kKaWNzbXgvR9dD5OeP1dSDUcGktS8GTWYO33KiIsJrToVS7L74x+MhuhZ9zACDfuRd8uPUXz/IzCPmYWOF/8eEXc3AMDgKIXjxj9D8f2PovDun0OfVYiOl/4Bis95yfcdbDsJ9/51MORXDbrdV7cT3mObUHDnT5Fzw33oXffwufenBr3o3/w7OG7686h/rpS6ZAn436/OZhFIUSwDSUqn0+H2mwYWE3Z29w77mApzEJOsvLpAJDUchO/EJ8heeh/M5VNhyClB9rV3w5BTDPe+9wAAtik3wFI1E4bsIhjzK5Gz7NvQQj6EOusv/r5DfnS/9d/IXfUDyGb7oPvCPc0wl0+DqXg8bFOuh2S0IuIcOPCq7+NnkDHrZugzC2LzSVNK+vZ1YzCzPFt0DIoRloEkVlVeghXXLURXTx9CofCwj1mY6UKWfvj7KA5UBdBUSDrDoJslvQnBliNDHq4pYbj3r4NkssFYMOai77r3g1/BUj0PlqqZQ+4z5o9BqL0OSsCDYHsdtEgQ+pwSBFqOINRxChlzbr2qT4vSS5XDgh/fNLppK0pOetEB6OqsumExjtaexvG6ekwaNwaSNPgscb2sYWl2P97szoMKnjMeb7LJClPJJDi3vQhDbjl0tmx4j21GsPU49DmfH/fqq9uF7jd/Di0chM6eg8K7fgqddeRFWt6jmxBqP4Xie38x7P2WsXNgq7kB7c/+CJLeiLw1P4JsMKH3/ceQu+ZHcO97F+69b0NnyYRj5V/AmF8Z9c+dUoME4H+/OovTAymOCwhTQF1DE/7nid9BkmWUFOYP+5iDHht2ubgCWIRwXxt63vslgs2HAUmGsagahpxSBNvrUPqdxwEAaigAxdsL1eeC+8D7CDQdRPE9/wOdLXvI+4u4utD27I9QeNdPz40etL/wExgLxg5aQHih/q0vQA16YZ+2Ah0v/wNK7n8U/rpdcO99G8Xf+mVMPndKft+/oRp/s4rHE6c6loEU8d5HW/H7tW+jsrwENotlyP2aBnzQ60BT0CwgHQEDT/hqyAe93YGuN/4TWsiPgjv+edjHnvn1d2CfdiOyFt055D7fye3oeu3fAOm8WT5NBSABkoSKv34Nkjz4t7hwTzM6X/3/UPyth+A5+AGCLUeR/4WfQA0F0PyLr6D8r16GbBp6VQqlt/mVWXjxTxdDljmqmOo4TZAiVly3ECdON2DH3kOYMmEsZHnwchBJApbk9OH1rnx4FH7bRZCNZshGM5SAB/76vci54b6RH6xp0JTh13qYK2eg+P5HBt3W8+4vYcgtQ+aCLw8pApqmoef9R5Gz7NuQjRZAU6GpZzes+uy1pl7x50WpKdeiw+PfnM8ikCb4rJAiDAY97rptFZpb29HY0ooxFWVDHmOWNSzL6cPbXD8QV/7TewAAekcpIn1t6Nv4NAyOMtinrYAaCsC5/SVYxy2Azu6A4nfBvfdtRNw9sE689tz76Hjx/8EyfhEy59wK2WSF8YJLCSWDCbI5Y8jtAOA58D50lkxYxy0AAJhKJ6N/6wsInjkO/+k9MORWDLkagdKbTtLw5L3z4LAZRUehOGEZSCHFBXn4ypob8fhzr6DP6UJOVuaQxxQYw5iX6cJOrh+IGzXoQ//mZxFxd0NnzoB14jXIXvJNSDo9oKkI97ag6/UNUPwu6CyZMBaNR9Hd/zloUV+4rx0m/+VvM614++Dc/jKKvvFf524zlUxE5vwvovOP/wLZmoW8NT+KyudJqePHy8didlWu6BgUR1wzkGJUVcVza9/Bux9twYSxVTAaDcM+7oPeHDQGhq4tIKL0dl2FBb//3jLRMSjOuM9AipFlGV9avRxTJ41HXUPTsLsTAsCS7H7uP0BEg5TYgF9/e4noGCQAy0AKstusuOdLa5DnyEbjmeEPMzLJGlY5emHhccdEhIEtzF/402thMXL2OB2xDKSoyrIS3HnLSoRCYfQ5h59rztArWOnohUHiSnKidKaDiofumoaqAq4lSlcsAyls8byZuPG6hTjT1oFgKDTsY/KMYSzL6YMELh0hSld/eW0xVkyvEh2DBGIZSGGSJOFLq5dj+uQJqKtvhqoOPwJQbg5icdalT8gjotSzcowJP7xlrugYJBjLQIqzWS2458u3oLQoH3X1Iy8onGTzYZbdHed0RCTS+AwFj9x/vegYlABYBtJAeUkR7rvrC8iw29DQ0jri4+ZkujHe4otjMiISpcAYwvPfuwEGw/CXH1N6YRlIEzUTqnHPl28BNKCto2vEx12X3Y9SUyCOyYgo3rLlIH7/7UUoyBm6MRmlJ5aBNLJg1jTcccuNcHm86Okbfo2ALAHLc/rg4B4ERCnJDj8e/9pUTKwoEh2FEgjLQBqRJAk3LVmEW1csQXtXN9we77CPM8oaVub2wKaLxDkhEcWSRfXjP1ZXYuG0CaKjUIJhGUgzsizji6uWY9k189DY0gp/IDjs42w6FascvTByDwKilGBS/fibRVlYs4RXDtBQLANpyGDQ4+4vrsH8WdNQ19CEcHj4EYAcQwQ3Onqh4x4EREnNqAbwnSkS7rllKSSJJ5bSUCwDacpqMeNbd9yGmgnVOHG6AYoy/AhAsSmEGx290HOEgCgpGdQg7pug4i++dgsMBm41TMNjGUhjjuws/MlXv4iqshLU1jeOuAdBmTnIKQOiJKRXQ/hGdQQ/vPtWmE0m0XEogbEMpLnSogLcf9cXkJuThdONLSMWgiJTCDfn9sDMg42IkoJeDeOrVQH89T23wmoxi45DCY5lgDBhbCW++ZXbYDTocaa9c8TH5RnDWJPbAysLAVFC02kRfLnci//7zdtgs1pEx6EkwDJAAIA50ybjq7evRiAYROtFNiXKMURwS143MnjZIVFCkrUIbi924/9963Zk2G2i41CSYBmgc25YNBdf/8LNCASCaG5tH/FxmXoFt+R1I5sbExElFJ0Wxi0FTvzDfbchK8MuOg4lEUkbaZKY0pKmadiyay+ee/UdKJqKqrKSES9FCigy3ut1oCdsjHNKIrqQSfXj1iIX/va+LyM3J0t0HEoyLAM0rB17D+LZV95EIBTC2IqyEQtBSJXwfq8DHSGuVCYSxR5x4tZiD370ra+gINchOg4lIZYBGtHew8fwzEtvoN/txoQxlSMWgogq4cO+HLQEuWKZKN4c4U6sLvLjB9+6C0X5uaLjUJJiGaCLOnyiDr958TV09fRhwthKyPLwy0wUDfi4LwcNAa5cJoqX0lATbihS8L1v3onSogLRcSiJsQzQJZ083Yin/rAWZ9o7MWFsFXS64QuBqgHbnFk47uMKZqJYkqChyn8S15aZ8N27v4zyEp5ASFeHZYBGpaG5Fb9+/o+obz6DidVV0OtH3tb0uNeK7c4sKOAe6ETRZoCCau9hLK8pwbfuuB35uTmiI1EKYBmgUWtp68Cvn/8jTp5uxMTqMRfd57wrZMCGvhx4FO6FThQtFoQwwXsIaxbW4O4v3swNhShqWAbosnR09+DXz7+KwyfqMGFsJUzGkS8rDCgyPurLQSuvNCC6alnwYnLgGL68YhG+sHIZDx2iqGIZoMvW0+fEU394FfsOHcfYqvKL7nuuasAedwYOeDLimJAoteSpvZgpNeBrt96EZYvn8xhiijqWAboiTrcHz7z0OnbsO4Tigjzk5mRf9PENfjM29WcjrHHTS6LRkqChNNSMORke3HvHLZg9dbLoSJSiWAboigWDIbyx/mO8+9FW6PV6VJYVX/Q3lv6IDh/2OtAfMcQxJVFyssoRVLmPYHZ5Ju6/6wsYW1kmOhKlMJYBuiqapmH7ngN48c116O51YvyYiovOZYZVCZv7s1HP/QiIRlRucKOw5wDmTR2Pb915OwrzuJkQxRbLAEVFfdMZ/O7Vt3Cs9jSqyktht1kv+viDHhs+dWVC4+WHROcYJBXTDe0w9tTh+oVzcPcX1/DkQYoLlgGKGqfbgz+8/i627NqHnOxMFOXnXfTxrUEjPurLQUDVxSkhUeLKN4QwOVIHzdePNcuuxZdWr4DRyCk1ig+WAYqqSCSCdRs/wRvrNyIciWBsRdmIWxgDgE+R8YkzC42cNqA0JUHDNIsT9u7DyMqw40url2P54vkX/X9DFG0sAxQT+44cx/Nr38WZ9g6MG1Nx0f0IAOCUz4JtrkwEOUpAacSui2C2vgWhrkbUTByHr92+CuOqKkTHojTEMkAxc6a9E8+tfRv7Dh9HWXERsrMuvtcARwkonYw1+1DqPgq9pmDFdQtw+01LL7nWhihWWAYoprw+P15990N8sHk7bDYrSosKLrlhSp3Pgu3OLAS5JwGlIIOkYo6lE1r7CVSWFuOOW27C3OlTuJEQCcUyQDGnqio2bt+NV975AC63B2Mryy45beBTZGx3ZvESREoppaYAxoVOI+LpxaI5M3DnrTehINchOhYRywDFT219E155ez0OHa9FniMHBXmOS/421Bww4RNnFg88oqRm10Uwx9qLcNtJZGdl4IurlmHporkXPf2TKJ5YBiiu/IEA3t+0Des+/gRur29UowQRVcJejx2HPHbuS0BJRYaGaXYPKiIt6OrswtRJA4sEqyvLRUcjGoRlgIQ4eboRL7+9HoeP1yE/d3SjBL1hPbb2Z6MzfPHyQJQISk0BLLD3o7etAbIk46YlC3HrjTfw2GFKSCwDJIw/EMC6jduwbuMn8Hh9GFNRCrPp4scdaxpw0mfFXncGvLwMkRJQhi6C+ZkuOCLdaDrThjHlpbjjlpswe+okLhKkhMUyQMKdPN2IV9/9EAeP1yIrwz6qKw4UDTjqteGAx84dDCkhGCUVMzPcGG/oR/OZVsiShEVzZ+BLq5YjPzdHdDyii2IZoIQQDIawaecevP3hJnR296GirBiZo9iTPaxKOOy14ZDHjhAvRSQBJGiYZPVhdoYLfV0d6HO6MLG6CrfddANm1XA0gJIDywAllNaOLrz+/sfYsfcgZFlGVVnxqFZcB1QJBz12HPXaEGEpoDgpNwWwINMFOdCPpjPtyM/NxqobFmPpNfNhtZhFxyMaNZYBSjiqqmL3gSN4/f2PcaqpBQW5jlEtMAQG9ifY787AcZ8VKq88oJjQUG4KYobdgxzJg8aWNkgAFs2ZjltWXI+y4kLRAYkuG8sAJSyn24P1m7bh4+270dvXj8KCPOTlZI+qFLgjOux1Z6DOb+HliBQVEjRUW/yYbvcgSw6hpb0DHq8PE6urcMvyJZg9dRIPF6KkxTJACa+1owsfbd2FrZ/uRb/Lg5KifORkZY6qFPSF9djjzkBDwAywFNAV0EsqJlp9mGbzwqaLoLO7F109vSgtKsTqpYtx7fxZl7wKhijRsQxQ0mhsacWHW3Zix76D8Pr8KC0qQFbmxQ8/+kx3yIADHjsaA2ZOH9ComGUFU2xeTLF5YZY19LvcaGntQE5WBpYtno/l1y6AIztLdEyiqGAZoKSiaRpONTbj/U3bsefgUQSCQVSUFo/6tDefIuOEz4rjPiu83OKYhmHXRTDN7sFEix86SYXL7UFrRxdMRiMWzJqKm5ddh/KSItExiaKKZYCSkqZpOFZXj/c3bsP+oyegRCIoLyuGzTK63d1UDWgOmHHMZ0VL0AROIZBDH8Z0uwdjLX5I0NDT50RHVzdsVgumT56AZYvno2ZCNS8VpJTEMkBJTVVVHDxWi/c3fYLDJ05BkiRUlBZd1hyuK6LDMa8NJ/0WBLmBUVqRoKHUFESNzYtycxCqqqKzpxddPX3IycrE/Bk1uG7BHIyrKmcJoJTGMkApIRKJYO/h43h/0zacOFUPvU6PkqKCy7rWO6IB9X4LjnltPP8gxeUbQqi2+FFt8cOiUxFRFLR3dqPP6UK+IwfXzZ+FxfNm8TJBShssA5RSQqEwdh88gvVbduB0YwvC4QgK8h3Izc66rMu+esJ6HPPaUOe3cBOjFJGpi6Da4sc4qw9ZegXAwL+XMx2d8Pn8KCkswA2L5mLh7OncPpjSDssApaRIJIJjdfXYsfcg9h4+jr5+J+x2G4oL8i55ZPL5QqqE034LGgNmtAZNULi2IKmYZQVjLX6Ms/hRYAyfu90fCKClrRORSASVZcVYds18zJ85FZkZdoFpicRhGaCU197Vg72HjmHrrr1obm2HBqAwPxfZmRmXNQ8cViW0BE1oCpjRFDRxfUGC0ksqKs0BVFv8KDMFIZ/9FmuaBpfbg7bObkiShPFjKrB88XzMnjYZFjO3Dqb0xjJAaSMYDOHQiTps270fh0/UweX2IDsrE4X5uTCM4vyD86ka0BEyojFgRlPADBcvUxRKBw3FpiCqLX5UmQMwyJ//WPP6/ejs7oXX60eG3YqJ1WOw9Jp5mD5pPAwGft+IAJYBSkOapqGlrQOfHjiCbbv3o7WzC3qdHkUFeciwWa9o1XhfWH+uGHSGDeClirElQUO+IYwSUxAlpiAKjCHoz/uSB0MhdPX0welyw2QyoqKkGAtmTcO0SeNQXlLEKwOILsAyQGnN6/PjwNET2LprH06cboDXF0BWph25OdlXfOqcT5HRFDCjMWBGW8jIBYhRIEGDwxBGiTGEElMQRcbQoN/+ASCiKOjp7UdPXz90soziwnzMm1GD6ZMnoLqybFSnXxKlK5YBIgzsV3C6qQV7Dx3DnkPH0NHdg2AwBKvFDEdONjLttiv6bVLVgN6wAR1hAzpDRnSEjPBwSmFUsvVnf/M3hlBsCsIkD/1Rpaoq+pwudPf0QVFV5DmyMXvqFMysmYhJ46p4ZgDRKLEMEF0gHI6gvvkMTpxuwN5Dx9HS1g63xwej0YDcnCxkZ2ZCp7vy3/Z9ioyOkBGdISO6wwb0hA0IpfnogV5SkaOPINcQRvHZJ3+rTh32sZqmweXxoqu7F8FQCNmZGZgyoRpzp09BzYRqXhFAdAVYBoguQlVVtHZ04cSpBhw8Vova+kb0u9yQJAk52ZlwZGfBaDBc9cdxRXToCRvOlYOesAH+FLxaQYKGDJ2CHEMYDn0EDkMYDkMYmToFIw28aJoGr8+PfpcbLrcHmqYhw2bDmIpSLJg1DTUTq1GYlxvfT4QoxbAMEF2G7t5+1NY34sjJUzh8og7dvf1QVAWZdjscOVmwms1RW5wWUCV4FR08im7Y115FBy1BFypaZQWZ+giy9BFk6hRknf1zhj4yaKHfcDRNgy8QgNPphtPtgaqqsFosyHNko2ZCNaory1BZVoLSogIuBCSKEpYBoivk9flRW9+EE6fqse/ICXR298IfDEKCBLvNiswMGzJsVuh0sfkNX9UAvyrDM0xR8Ck6KJoEBYCqSVDP//O515d+ItVJKkySBpOsfv4iaTCe/3f588cYZRVWWR2yuO9iNE1DIBiC0zXw5B+JRGAxm+HIzsKU8WMwbkwFqspKUFKYz0WARDHCMkAUBeFwBM2t7Whp70BjSxuOn6pHd28/PF4fVFWF2WSC3WaF3WaF1RK90YOrpWgYXBTOlgX92Sd3XQxiqqoKnz8Ar88Pp9uDcDgCk9GAnKwMTKwegwljK1FVVoKy4kIYjVc/BUNEl8YyQBQDqqqiq7cPLW2dONPeiZOnG9Dc2g6P1wdfIAgJgNFggM1mhd1qgc1quayzE5KBpmkIhkLw+vzw+vzw+QNQFRWSBFgsA5/zuMoyTBw3BlVlJZd92iQRRQ/LAFGcBIJBdHT1oL2rB22d3WhoPoOmM21we33w+vznHifJMkxGA0xGI0xGI8ymgdd6vS5hRhQ+o2kawpEI/IEg/IEA/IEgAoEgNE2DBsBkNMBmscCRnYUx5SUoKSpAfq4DhXkO5DtyYDLxdEiiRMAyQCRQOBxBR3cPOnt64XJ74XR70NfvQmdPL7p7++DzBxAMhRAMhRCJDJy0p2mAwaD/vDCYjNDpdJAlCdJ5L7J89rUkQZLkc38fjqZpUFUV4YiCSCSCiHL2dURB+NzfB25TVQ2yLEE7G0av18NiNsFqMSM/14GyogLkOXKQm5OF3JwsFOQ6kHGF+zQQUXywDBAlqM8uqXO6PXB7vHC5PXCefd3V24eunj709TvhD4agKgpUTTv7pH72taZB09Rzf//sBZI07NJBWZah1+mg1+th0A+81ut0sFktsNutyLBakWG3wW6zwmwywWIeeMnOzIAjJws5mZnc658oSbEMECUxRVHg9voQCoehKCoURYGqqgN/VpWzrwduP//P6me3qSokSDCbP39y/+yJ3mwywWwyptxaBiIaimWAiIgozbHyExERpTmWAaIYePTRR1FVVQWz2YwFCxZg165doiMREY2IZYAoyl566SU88MAD+Kd/+ifs3bsXM2bMwMqVK9HZ2Sk6GhHRsLhmgCjKFixYgHnz5uGRRx4BMLABUXl5OX7wgx/gJz/5ieB0RERDcWSAKIpCoRD27NmDFStWnLtNlmWsWLEC27dvF5iMiGhkLANEUdTd3Q1FUVBYWDjo9sLCQrS3twtKRUR0cSwDREREaY5lgCiK8vLyoNPp0NHRMej2jo4OFBUVCUpFRHRxLANEUWQ0GjFnzhxs2LDh3G2qqmLDhg1YtGiRwGRERCPjRuJEUfbAAw/g3nvvxdy5czF//nw8+OCD8Hq9uO+++0RHIyIaFssAUZTddddd6Orqwj/+4z+ivb0dM2fOxLp164YsKiQiShTcZ4CIiCjNcc0AERFRmmMZICIiSnMsA0RERGmOZYCIiCjNsQwQERGlOZYBIiKiNMcyQERElOZYBoiIiNIcywAREVGaYxkgIiJKcywDREREae7/B/6Vvdek679dAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "groups = df_full['class'].value_counts()\n",
    "sizes = [groups[0], groups[1]]\n",
    "labels = [str(e) for e in groups.index]\n",
    "\n",
    "fig1, ax1 = plt.subplots()\n",
    "ax1.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=True, startangle=90)\n",
    "ax1.axis('equal')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper function for calculating metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "from rulekit.classification import RuleClassifier\n",
    "\n",
    "\n",
    "X: pd.DataFrame = df_full.drop(['class'], axis=1)\n",
    "y: pd.Series = df_full['class']\n",
    "\n",
    "\n",
    "def get_prediction_metrics(\n",
    "    measure: str,\n",
    "    y_pred: np.ndarray,\n",
    "    y_true: pd.Series,\n",
    "    classification_metrics: dict\n",
    ") -> tuple[pd.DataFrame, np.ndarray]:\n",
    "    confusion_matrix: np.ndarray = metrics.confusion_matrix(y_true, y_pred)\n",
    "    tn, fp, fn, tp = confusion_matrix.ravel()\n",
    "    sensitivity: float = tp / (tp + fn)\n",
    "    specificity: float = tn / (tn + fp)\n",
    "    npv: float = tn / (tn + fn)\n",
    "    ppv: float = tp / (tp + fp)\n",
    "\n",
    "    dictionary = {\n",
    "        'Measure': measure,\n",
    "        'Accuracy': metrics.accuracy_score(y_true, y_pred),\n",
    "        'MAE': metrics.mean_absolute_error(y_true, y_pred),\n",
    "        'Kappa': metrics.cohen_kappa_score(y_true, y_pred),\n",
    "        'Balanced accuracy': metrics.balanced_accuracy_score(y_true, y_pred),\n",
    "        'Logistic loss': metrics.log_loss(y_true, y_pred),\n",
    "        'Precision': metrics.log_loss(y_true, y_pred),\n",
    "        'Sensitivity': sensitivity,\n",
    "        'Specificity': specificity,\n",
    "        'NPV': npv,\n",
    "        'PPV': ppv,\n",
    "        'psep': ppv + npv - 1,\n",
    "        'Fall-out': fp / (fp + tn),\n",
    "        \"Youden's J statistic\": sensitivity + specificity - 1,\n",
    "        'Lift': (tp / (tp + fp)) / ((tp + fn) / (tp + tn + fp + fn)),\n",
    "        'F-measure': 2 * tp / (2 * tp + fp + fn),\n",
    "        'Fowlkes-Mallows index': metrics.fowlkes_mallows_score(y_true, y_pred),\n",
    "        'False positive': fp,\n",
    "        'False negative': fn,\n",
    "        'True positive': tp,\n",
    "        'True negative': tn,\n",
    "        'Rules per example': classification_metrics['rules_per_example'],\n",
    "        'Voting conflicts': classification_metrics['voting_conflicts'],\n",
    "        'Geometric mean': math.sqrt(specificity * sensitivity),\n",
    "        'Geometric mean': math.sqrt(specificity * sensitivity),\n",
    "    }\n",
    "    return pd.DataFrame.from_records([dictionary], index='Measure'), confusion_matrix\n",
    "\n",
    "\n",
    "def get_ruleset_stats(\n",
    "    measure: str,\n",
    "    model: RuleClassifier\n",
    ") -> pd.DataFrame:\n",
    "    return pd.DataFrame.from_records(\n",
    "        [{'Measure': measure, **model.stats.__dict__}],\n",
    "        index='Measure'\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rule induction on full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time_total_s</th>\n",
       "      <th>time_growing_s</th>\n",
       "      <th>time_pruning_s</th>\n",
       "      <th>rules_count</th>\n",
       "      <th>conditions_per_rule</th>\n",
       "      <th>induced_conditions_per_rule</th>\n",
       "      <th>avg_rule_coverage</th>\n",
       "      <th>avg_rule_precision</th>\n",
       "      <th>avg_rule_quality</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>FDR_pvalue</th>\n",
       "      <th>FWER_pvalue</th>\n",
       "      <th>fraction_significant</th>\n",
       "      <th>fraction_FDR_significant</th>\n",
       "      <th>fraction_FWER_significant</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Measure</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C2</th>\n",
       "      <td>1.218550</td>\n",
       "      <td>1.036229</td>\n",
       "      <td>0.111458</td>\n",
       "      <td>35</td>\n",
       "      <td>4.742857</td>\n",
       "      <td>22.142857</td>\n",
       "      <td>0.259410</td>\n",
       "      <td>0.670793</td>\n",
       "      <td>0.322125</td>\n",
       "      <td>0.005729</td>\n",
       "      <td>0.005879</td>\n",
       "      <td>0.010640</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.885714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Correlation</th>\n",
       "      <td>0.471475</td>\n",
       "      <td>0.339709</td>\n",
       "      <td>0.123223</td>\n",
       "      <td>21</td>\n",
       "      <td>5.142857</td>\n",
       "      <td>51.666667</td>\n",
       "      <td>0.306612</td>\n",
       "      <td>0.469157</td>\n",
       "      <td>0.201772</td>\n",
       "      <td>0.016841</td>\n",
       "      <td>0.017345</td>\n",
       "      <td>0.026655</td>\n",
       "      <td>0.904762</td>\n",
       "      <td>0.904762</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RSS</th>\n",
       "      <td>1.514044</td>\n",
       "      <td>1.327209</td>\n",
       "      <td>0.171176</td>\n",
       "      <td>14</td>\n",
       "      <td>3.714286</td>\n",
       "      <td>64.428571</td>\n",
       "      <td>0.473795</td>\n",
       "      <td>0.484564</td>\n",
       "      <td>0.253249</td>\n",
       "      <td>0.041892</td>\n",
       "      <td>0.044068</td>\n",
       "      <td>0.068063</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             time_total_s  time_growing_s  time_pruning_s  rules_count  \\\n",
       "Measure                                                                  \n",
       "C2               1.218550        1.036229        0.111458           35   \n",
       "Correlation      0.471475        0.339709        0.123223           21   \n",
       "RSS              1.514044        1.327209        0.171176           14   \n",
       "\n",
       "             conditions_per_rule  induced_conditions_per_rule  \\\n",
       "Measure                                                         \n",
       "C2                      4.742857                    22.142857   \n",
       "Correlation             5.142857                    51.666667   \n",
       "RSS                     3.714286                    64.428571   \n",
       "\n",
       "             avg_rule_coverage  avg_rule_precision  avg_rule_quality  \\\n",
       "Measure                                                                \n",
       "C2                    0.259410            0.670793          0.322125   \n",
       "Correlation           0.306612            0.469157          0.201772   \n",
       "RSS                   0.473795            0.484564          0.253249   \n",
       "\n",
       "               pvalue  FDR_pvalue  FWER_pvalue  fraction_significant  \\\n",
       "Measure                                                                \n",
       "C2           0.005729    0.005879     0.010640              0.971429   \n",
       "Correlation  0.016841    0.017345     0.026655              0.904762   \n",
       "RSS          0.041892    0.044068     0.068063              0.785714   \n",
       "\n",
       "             fraction_FDR_significant  fraction_FWER_significant  \n",
       "Measure                                                           \n",
       "C2                           0.971429                   0.885714  \n",
       "Correlation                  0.904762                   0.857143  \n",
       "RSS                          0.785714                   0.714286  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>MAE</th>\n",
       "      <th>Kappa</th>\n",
       "      <th>Balanced accuracy</th>\n",
       "      <th>Logistic loss</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>NPV</th>\n",
       "      <th>PPV</th>\n",
       "      <th>...</th>\n",
       "      <th>Lift</th>\n",
       "      <th>F-measure</th>\n",
       "      <th>Fowlkes-Mallows index</th>\n",
       "      <th>False positive</th>\n",
       "      <th>False negative</th>\n",
       "      <th>True positive</th>\n",
       "      <th>True negative</th>\n",
       "      <th>Rules per example</th>\n",
       "      <th>Voting conflicts</th>\n",
       "      <th>Geometric mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Measure</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C2</th>\n",
       "      <td>0.932663</td>\n",
       "      <td>0.067337</td>\n",
       "      <td>0.433613</td>\n",
       "      <td>0.709693</td>\n",
       "      <td>2.427088</td>\n",
       "      <td>2.427088</td>\n",
       "      <td>0.452941</td>\n",
       "      <td>0.966446</td>\n",
       "      <td>0.961665</td>\n",
       "      <td>0.487342</td>\n",
       "      <td>...</td>\n",
       "      <td>7.407595</td>\n",
       "      <td>0.469512</td>\n",
       "      <td>0.928703</td>\n",
       "      <td>81</td>\n",
       "      <td>93</td>\n",
       "      <td>77</td>\n",
       "      <td>2333</td>\n",
       "      <td>9.079334</td>\n",
       "      <td>2146.0</td>\n",
       "      <td>0.661622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Correlation</th>\n",
       "      <td>0.827399</td>\n",
       "      <td>0.172601</td>\n",
       "      <td>0.246689</td>\n",
       "      <td>0.729909</td>\n",
       "      <td>6.221157</td>\n",
       "      <td>6.221157</td>\n",
       "      <td>0.617647</td>\n",
       "      <td>0.842171</td>\n",
       "      <td>0.969018</td>\n",
       "      <td>0.216049</td>\n",
       "      <td>...</td>\n",
       "      <td>3.283951</td>\n",
       "      <td>0.320122</td>\n",
       "      <td>0.823757</td>\n",
       "      <td>381</td>\n",
       "      <td>65</td>\n",
       "      <td>105</td>\n",
       "      <td>2033</td>\n",
       "      <td>6.438854</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>0.721224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RSS</th>\n",
       "      <td>0.788313</td>\n",
       "      <td>0.211687</td>\n",
       "      <td>0.207458</td>\n",
       "      <td>0.725394</td>\n",
       "      <td>7.629984</td>\n",
       "      <td>7.629984</td>\n",
       "      <td>0.652941</td>\n",
       "      <td>0.797846</td>\n",
       "      <td>0.970277</td>\n",
       "      <td>0.185309</td>\n",
       "      <td>...</td>\n",
       "      <td>2.816694</td>\n",
       "      <td>0.288687</td>\n",
       "      <td>0.789800</td>\n",
       "      <td>488</td>\n",
       "      <td>59</td>\n",
       "      <td>111</td>\n",
       "      <td>1926</td>\n",
       "      <td>6.633127</td>\n",
       "      <td>2085.0</td>\n",
       "      <td>0.721766</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Accuracy       MAE     Kappa  Balanced accuracy  Logistic loss  \\\n",
       "Measure                                                                       \n",
       "C2           0.932663  0.067337  0.433613           0.709693       2.427088   \n",
       "Correlation  0.827399  0.172601  0.246689           0.729909       6.221157   \n",
       "RSS          0.788313  0.211687  0.207458           0.725394       7.629984   \n",
       "\n",
       "             Precision  Sensitivity  Specificity       NPV       PPV  ...  \\\n",
       "Measure                                                               ...   \n",
       "C2            2.427088     0.452941     0.966446  0.961665  0.487342  ...   \n",
       "Correlation   6.221157     0.617647     0.842171  0.969018  0.216049  ...   \n",
       "RSS           7.629984     0.652941     0.797846  0.970277  0.185309  ...   \n",
       "\n",
       "                 Lift  F-measure  Fowlkes-Mallows index  False positive  \\\n",
       "Measure                                                                   \n",
       "C2           7.407595   0.469512               0.928703              81   \n",
       "Correlation  3.283951   0.320122               0.823757             381   \n",
       "RSS          2.816694   0.288687               0.789800             488   \n",
       "\n",
       "             False negative  True positive  True negative  Rules per example  \\\n",
       "Measure                                                                        \n",
       "C2                       93             77           2333           9.079334   \n",
       "Correlation              65            105           2033           6.438854   \n",
       "RSS                      59            111           1926           6.633127   \n",
       "\n",
       "             Voting conflicts  Geometric mean  \n",
       "Measure                                        \n",
       "C2                     2146.0        0.661622  \n",
       "Correlation            1993.0        0.721224  \n",
       "RSS                    2085.0        0.721766  \n",
       "\n",
       "[3 rows x 23 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix - C2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2333</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0   1\n",
       "0  2333  81\n",
       "1    93  77"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix - Correlation\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2033</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0    1\n",
       "0  2033  381\n",
       "1    65  105"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix - RSS\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1926</td>\n",
       "      <td>488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>59</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0    1\n",
       "0  1926  488\n",
       "1    59  111"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from rulekit.classification import RuleClassifier\n",
    "from rulekit.rules import RuleSet, ClassificationRule\n",
    "from rulekit.params import Measures\n",
    "from IPython.display import display\n",
    "\n",
    "# C2\n",
    "clf = RuleClassifier(\n",
    "    induction_measure=Measures.C2,\n",
    "    pruning_measure=Measures.C2,\n",
    "    voting_measure=Measures.C2,\n",
    ")\n",
    "clf.fit(X, y)\n",
    "c2_ruleset: RuleSet[ClassificationRule] = clf.model\n",
    "prediction, classification_metrics = clf.predict(X, return_metrics=True)\n",
    "\n",
    "prediction_metric, c2_confusion_matrix = get_prediction_metrics('C2', prediction, y, classification_metrics)\n",
    "model_stats = get_ruleset_stats('C2', clf.model)\n",
    "\n",
    "# Correlation\n",
    "clf = RuleClassifier(\n",
    "    induction_measure=Measures.Correlation,\n",
    "    pruning_measure=Measures.Correlation,\n",
    "    voting_measure=Measures.Correlation,\n",
    ")\n",
    "clf.fit(X, y)\n",
    "corr_ruleset: RuleSet[ClassificationRule] = clf.model\n",
    "prediction, classification_metrics = clf.predict(X, return_metrics=True)\n",
    "\n",
    "tmp, corr_confusion_matrix = get_prediction_metrics('Correlation', prediction, y, classification_metrics)\n",
    "prediction_metric = pd.concat([prediction_metric, tmp])\n",
    "model_stats = pd.concat([model_stats, get_ruleset_stats('Correlation', clf.model)])\n",
    "\n",
    "# RSS\n",
    "clf = RuleClassifier(\n",
    "    induction_measure=Measures.RSS,\n",
    "    pruning_measure=Measures.RSS,\n",
    "    voting_measure=Measures.RSS,\n",
    ")\n",
    "clf.fit(X, y)\n",
    "rss_ruleset: RuleSet[ClassificationRule] = clf.model\n",
    "prediction, classification_metrics = clf.predict(X, return_metrics=True)\n",
    "tmp, rss_confusion_matrix = get_prediction_metrics('RSS', prediction, y, classification_metrics)\n",
    "prediction_metric = pd.concat([prediction_metric, tmp])\n",
    "model_stats = pd.concat([model_stats, get_ruleset_stats('RSS', clf.model)])\n",
    "\n",
    "display(model_stats)\n",
    "display(prediction_metric)\n",
    "\n",
    "print('Confusion matrix - C2')\n",
    "display(pd.DataFrame(c2_confusion_matrix))\n",
    "\n",
    "print('Confusion matrix - Correlation')\n",
    "display(pd.DataFrame(corr_confusion_matrix))\n",
    "\n",
    "print('Confusion matrix - RSS')\n",
    "display(pd.DataFrame(rss_confusion_matrix))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### C2 Measure generated rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IF gimpuls = (-inf, 32.50) THEN class = {0}\n",
      "IF nbumps = (-inf, 0.50) AND goenergy = <-84.50, inf) AND goimpuls = (-inf, -0.50) AND genergy = (-inf, 13675) THEN class = {0}\n",
      "IF genergy = (-inf, 17640) AND nbumps = (-inf, 0.50) AND goenergy = <-84.50, inf) THEN class = {0}\n",
      "IF genergy = <1625, 17640) AND maxenergy = (-inf, 3500) AND gimpuls = (-inf, 772.50) THEN class = {0}\n",
      "IF shift = {N} AND nbumps = (-inf, 0.50) AND goenergy = <-73.50, inf) THEN class = {0}\n",
      "IF shift = {N} AND senergy = (-inf, 6150) AND genergy = <1865, inf) AND goimpuls = (-inf, 230.50) THEN class = {0}\n",
      "IF senergy = (-inf, 550) AND gimpuls = (-inf, 380.50) AND goimpuls = (-inf, 96.50) AND goenergy = (-inf, 118) THEN class = {0}\n",
      "IF senergy = (-inf, 550) AND genergy = (-inf, 31790) AND goenergy = <-84.50, 114.50) THEN class = {0}\n",
      "IF senergy = (-inf, 550) AND goenergy = <-84.50, 87.50) AND gimpuls = (-inf, 1342.50) AND goimpuls = (-inf, 96) THEN class = {0}\n",
      "IF senergy = (-inf, 550) AND goimpuls = (-inf, 233.50) THEN class = {0}\n",
      "IF genergy = <1865, 28515) AND goenergy = (-inf, 105.50) AND nbumps = (-inf, 4.50) THEN class = {0}\n",
      "IF nbumps = <0.50, 1.50) AND gimpuls = (-inf, 1210) AND goimpuls = (-inf, 233.50) AND goenergy = <-72.50, inf) AND genergy = <12550, inf) THEN class = {0}\n",
      "IF gimpuls = (-inf, 514.50) AND nbumps = (-inf, 6.50) AND goimpuls = (-inf, 96.50) AND goenergy = <-84.50, inf) THEN class = {0}\n",
      "IF nbumps = (-inf, 2.50) AND gimpuls = (-inf, 1832.50) AND goimpuls = (-inf, 312) THEN class = {0}\n",
      "IF nbumps3 = (-inf, 2.50) AND nbumps = (-inf, 6.50) AND goenergy = <-88.50, inf) THEN class = {0}\n",
      "IF genergy = (-inf, 748755) AND goimpuls = (-inf, 95.50) AND maxenergy = (-inf, 55000) AND gimpuls = (-inf, 3096) THEN class = {0}\n",
      "IF nbumps3 = <3.50, inf) AND gimpuls = <364, 1459.50) AND senergy = <10150, inf) THEN class = {1}\n",
      "IF gimpuls = <2208.50, inf) AND genergy = <513615, 1005720) AND nbumps2 = <0.50, inf) AND nbumps = (-inf, 3.50) THEN class = {1}\n",
      "IF gimpuls = <1328, inf) AND goenergy = (-inf, -29.50) AND goimpuls = <-31.50, -14.50) AND nbumps4 = (-inf, 1.50) THEN class = {1}\n",
      "IF gimpuls = <1328, 2109) AND goimpuls = (-inf, -5.50) AND senergy = <350, 36350) AND genergy = <159155, 586025) AND nbumps2 = (-inf, 3.50) AND goenergy = <-41.50, inf) THEN class = {1}\n",
      "IF nbumps3 = <0.50, 1.50) AND gimpuls = <1408, 1959) AND goimpuls = <-20.50, 13.50) AND senergy = (-inf, 54950) THEN class = {1}\n",
      "IF senergy = <750, 38250) AND genergy = <254130, 1133675) AND goenergy = <-16.50, inf) AND gimpuls = <1438.50, inf) AND goimpuls = <-5, inf) THEN class = {1}\n",
      "IF nbumps = <4.50, inf) AND nbumps3 = <1.50, 4.50) AND gimpuls = <203.50, inf) AND senergy = <4300, 131700) AND goenergy = <-41.50, inf) THEN class = {1}\n",
      "IF nbumps = <2.50, 4.50) AND gimpuls = <740.50, inf) AND genergy = <38935, 127440) AND senergy = (-inf, 14750) AND goimpuls = (-inf, 68.50) THEN class = {1}\n",
      "IF nbumps = <2.50, inf) AND gimpuls = <379, 1742) AND senergy = (-inf, 31100) AND genergy = (-inf, 211170) AND goenergy = (-inf, 123.50) AND goimpuls = (-inf, 19.50) THEN class = {1}\n",
      "IF gimpuls = <1139.50, inf) AND goimpuls = <-46, 116.50) AND senergy = (-inf, 38250) AND genergy = <46580, 1877915) AND goenergy = (-inf, 183) AND shift = {W} AND nbumps3 = (-inf, 1.50) AND nbumps2 = (-inf, 2.50) THEN class = {1}\n",
      "IF nbumps = <1.50, 3.50) AND gimpuls = <521.50, 2344.50) AND nbumps2 = <0.50, inf) AND genergy = <34605, 656965) AND goenergy = (-inf, 137) AND goimpuls = <-39, 41.50) AND maxenergy = <450, inf) THEN class = {1}\n",
      "IF nbumps = <1.50, 3.50) AND nbumps2 = <0.50, inf) AND genergy = <18870, inf) AND nbumps3 = (-inf, 1.50) AND gimpuls = <160, inf) AND senergy = <550, inf) AND goimpuls = <-62.50, 8.50) AND goenergy = (-inf, -1.50) THEN class = {1}\n",
      "IF nbumps = <1.50, inf) AND gimpuls = <95, 1603.50) AND goenergy = (-inf, 131) AND goimpuls = <-70.50, 119) AND nbumps2 = (-inf, 4.50) AND nbumps3 = <0.50, inf) AND genergy = (-inf, 614380) AND maxenergy = (-inf, 25000) AND senergy = <2250, inf) THEN class = {1}\n",
      "IF goenergy = <-59.50, 186) AND genergy = <12415, 129940) AND gimpuls = <121.50, 793) AND senergy = <150, 1350) AND ghazard = {a} AND goimpuls = <-53.50, inf) THEN class = {1}\n",
      "IF genergy = <42215, 94300) AND gimpuls = <133.50, 813.50) AND ghazard = {a} AND goenergy = <-74.50, 160) AND senergy = (-inf, 11100) AND nbumps = (-inf, 3.50) AND nbumps3 = (-inf, 0.50) THEN class = {1}\n",
      "IF gimpuls = <537.50, 796) AND shift = {W} AND genergy = <17635, 36470) AND goimpuls = <-36.50, inf) AND goenergy = <-37.50, inf) AND nbumps = (-inf, 0.50) THEN class = {1}\n",
      "IF genergy = <18800, 52205) AND shift = {W} AND ghazard = {a} AND goimpuls = <-28.50, inf) AND goenergy = (-inf, 181) AND gimpuls = <380.50, 524.50) AND nbumps = (-inf, 0.50) THEN class = {1}\n",
      "IF gimpuls = <184.50, inf) AND goenergy = <-55.50, 128.50) AND genergy = <7265, inf) AND goimpuls = <-60.50, 37.50) AND nbumps = (-inf, 7.50) AND nbumps2 = (-inf, 4.50) AND maxenergy = (-inf, 25000) AND senergy = (-inf, 31350) THEN class = {1}\n",
      "IF gimpuls = <32.50, inf) AND goimpuls = <-74.50, inf) AND ghazard = {a} AND genergy = <1510, inf) AND goenergy = <-89.50, 124.50) THEN class = {1}\n"
     ]
    }
   ],
   "source": [
    "for rule in c2_ruleset.rules:\n",
    "    print(rule)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation Measure generated rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IF nbumps = (-inf, 1.50) AND gimpuls = (-inf, 1252.50) THEN class = {0}\n",
      "IF nbumps = (-inf, 2.50) AND gimpuls = (-inf, 1331) AND goimpuls = (-inf, 312) AND nbumps5 = (-inf, 0.50) THEN class = {0}\n",
      "IF nbumps = (-inf, 2.50) THEN class = {0}\n",
      "IF gimpuls = (-inf, 1253.50) AND nbumps3 = (-inf, 2.50) AND nbumps = (-inf, 7) THEN class = {0}\n",
      "IF nbumps4 = (-inf, 2.50) THEN class = {0}\n",
      "IF nbumps2 = <0.50, 2.50) AND maxenergy = <1500, inf) AND senergy = (-inf, 36050) AND nbumps3 = <0.50, 4.50) AND goimpuls = <-34, 95) AND genergy = (-inf, 662435) AND gimpuls = <994.50, 1959) THEN class = {1}\n",
      "IF nbumps2 = <0.50, inf) AND maxenergy = <1500, inf) AND goimpuls = <-55, 95) AND nbumps = (-inf, 6.50) AND genergy = <61250, 662435) AND goenergy = (-inf, 96) AND nbumps3 = <0.50, inf) AND gimpuls = <712, 2257.50) AND senergy = (-inf, 31100) THEN class = {1}\n",
      "IF nbumps2 = <0.50, inf) AND genergy = <58310, 934630) AND goenergy = (-inf, 186) AND senergy = (-inf, 40650) AND maxenergy = <1500, inf) AND gimpuls = <538.50, inf) AND goimpuls = <-55, inf) THEN class = {1}\n",
      "IF nbumps = <1.50, inf) AND nbumps2 = <0.50, inf) AND gimpuls = <521.50, 2374) AND genergy = <58310, 799855) AND senergy = <650, 36050) AND goimpuls = <-71, 58.50) AND ghazard = {a} THEN class = {1}\n",
      "IF nbumps = <1.50, 6.50) AND nbumps2 = <0.50, inf) AND gimpuls = <521.50, 2374) AND genergy = <34360, inf) AND maxenergy = <350, inf) AND goimpuls = <-55, 95) AND senergy = <550, inf) AND nbumps4 = (-inf, 1.50) THEN class = {1}\n",
      "IF nbumps = <1.50, inf) AND gimpuls = <306, inf) AND genergy = <28325, inf) AND goimpuls = (-inf, 19.50) THEN class = {1}\n",
      "IF nbumps = <1.50, inf) AND nbumps2 = <0.50, inf) AND gimpuls = <153.50, 321) AND genergy = <14295, 36250) AND goimpuls = <-60.50, inf) AND senergy = (-inf, 40500) AND nbumps3 = (-inf, 3.50) THEN class = {1}\n",
      "IF genergy = <96260, 1062020) AND goimpuls = <-29, inf) AND senergy = <850, 7500) AND nbumps3 = (-inf, 1.50) AND gimpuls = <1404, 2965.50) AND nbumps = (-inf, 3.50) AND goenergy = (-inf, 69.50) THEN class = {1}\n",
      "IF gimpuls = <1253.50, inf) AND goenergy = <-50.50, 131.50) AND genergy = <46580, 1789250) AND nbumps = (-inf, 7.50) AND shift = {W} AND goimpuls = <-60.50, 118) AND senergy = (-inf, 95850) AND ghazard = {a} THEN class = {1}\n",
      "IF senergy = <550, inf) AND shift = {W} AND genergy = <10495, inf) AND gimpuls = <160, inf) AND goenergy = (-inf, 126) THEN class = {1}\n",
      "IF senergy = <350, inf) AND goimpuls = <-74.50, inf) AND gimpuls = <32.50, inf) AND goenergy = <-78.50, inf) AND maxenergy = <250, inf) THEN class = {1}\n",
      "IF genergy = <43150, inf) AND gimpuls = <133.50, inf) AND goenergy = (-inf, 176.50) THEN class = {1}\n",
      "IF shift = {W} AND genergy = <31760, 49585) AND gimpuls = <362.50, 771) AND goimpuls = <-27.50, inf) AND goenergy = <-3.50, inf) AND maxenergy = (-inf, 650) THEN class = {1}\n",
      "IF shift = {W} AND genergy = <20485, 43280) AND gimpuls = <380.50, 796) AND goimpuls = <-37, 142.50) AND goenergy = <-37.50, 181) AND nbumps = (-inf, 0.50) THEN class = {1}\n",
      "IF gimpuls = <177.50, inf) AND genergy = <7265, inf) AND goimpuls = (-inf, 241.50) AND goenergy = (-inf, 124.50) THEN class = {1}\n",
      "IF gimpuls = <54.50, 90) AND genergy = <1510, 4905) AND goimpuls = <-72.50, 28.50) AND seismoacoustic = {a} THEN class = {1}\n"
     ]
    }
   ],
   "source": [
    "for rule in corr_ruleset.rules:\n",
    "    print(rule)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RSS Measure generated rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IF nbumps = (-inf, 1.50) AND genergy = (-inf, 126350) THEN class = {0}\n",
      "IF nbumps = (-inf, 1.50) AND gimpuls = (-inf, 2168) AND goimpuls = (-inf, 96.50) THEN class = {0}\n",
      "IF genergy = (-inf, 44750) AND nbumps3 = (-inf, 2.50) AND goenergy = (-inf, 105.50) THEN class = {0}\n",
      "IF gimpuls = (-inf, 725.50) AND nbumps3 = (-inf, 3.50) AND nbumps4 = (-inf, 2.50) AND goimpuls = (-inf, 117) AND goenergy = <-88.50, inf) THEN class = {0}\n",
      "IF nbumps2 = (-inf, 1.50) AND nbumps = (-inf, 4.50) THEN class = {0}\n",
      "IF goimpuls = (-inf, 312) AND nbumps5 = (-inf, 0.50) AND goenergy = <-88.50, inf) THEN class = {0}\n",
      "IF gimpuls = <521.50, inf) AND genergy = <57680, inf) THEN class = {1}\n",
      "IF nbumps = <1.50, inf) THEN class = {1}\n",
      "IF senergy = <550, inf) AND shift = {W} AND genergy = <10495, inf) THEN class = {1}\n",
      "IF nbumps = <0.50, inf) AND goimpuls = <-74.50, inf) AND gimpuls = <32.50, inf) AND goenergy = <-78.50, 124.50) THEN class = {1}\n",
      "IF genergy = <34315, 49585) AND ghazard = {a} AND gimpuls = <396, 1445.50) AND goenergy = <7, inf) AND goimpuls = <-19, inf) AND senergy = (-inf, 350) THEN class = {1}\n",
      "IF genergy = <26200, 78890) AND gimpuls = <133.50, 813.50) AND goenergy = <-74.50, 297.50) AND goimpuls = <-71, inf) AND nbumps = (-inf, 3.50) AND senergy = (-inf, 1850) THEN class = {1}\n",
      "IF genergy = <18585, 25305) AND shift = {W} AND gimpuls = <240, 588.50) AND goimpuls = <-42.50, 133) AND goenergy = <-45.50, inf) AND senergy = (-inf, 450) THEN class = {1}\n",
      "IF gimpuls = <54.50, inf) AND goimpuls = <-74.50, 28.50) AND genergy = <1510, inf) AND ghazard = {a} AND nbumps4 = (-inf, 1.50) AND senergy = (-inf, 92850) THEN class = {1}\n"
     ]
    }
   ],
   "source": [
    "for rule in rss_ruleset.rules:\n",
    "    print(rule)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratified K-Folds cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "N_SPLITS: int = 10\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cezar\\AppData\\Local\\Temp\\ipykernel_13196\\4002598548.py:22: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ppv: float = tp / (tp + fp)\n",
      "C:\\Users\\cezar\\AppData\\Local\\Temp\\ipykernel_13196\\4002598548.py:39: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  'Lift': (tp / (tp + fp)) / ((tp + fn) / (tp + tn + fp + fn)),\n",
      "C:\\Users\\cezar\\AppData\\Local\\Temp\\ipykernel_13196\\4002598548.py:22: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ppv: float = tp / (tp + fp)\n",
      "C:\\Users\\cezar\\AppData\\Local\\Temp\\ipykernel_13196\\4002598548.py:39: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  'Lift': (tp / (tp + fp)) / ((tp + fn) / (tp + tn + fp + fn)),\n"
     ]
    }
   ],
   "source": [
    "c2_ruleset_stats = pd.DataFrame()\n",
    "c2_prediction_metrics = pd.DataFrame()\n",
    "c2_confusion_matrix = np.array([[0.0, 0.0], [0.0, 0.0]])\n",
    "\n",
    "for train_index, test_index in skf.split(X, y):\n",
    "    x_train, x_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "\n",
    "    clf = RuleClassifier(\n",
    "        induction_measure=Measures.C2,\n",
    "        pruning_measure=Measures.C2,\n",
    "        voting_measure=Measures.C2,\n",
    "    )\n",
    "    clf.fit(x_train, y_train)\n",
    "    c2_ruleset = clf.model\n",
    "    prediction, classification_metrics = clf.predict(x_test, return_metrics=True)\n",
    "    tmp, confusion_matrix = get_prediction_metrics('C2', prediction, y_test, classification_metrics)\n",
    "    \n",
    "    c2_prediction_metrics = pd.concat([c2_prediction_metrics, tmp])\n",
    "    c2_ruleset_stats = pd.concat([c2_ruleset_stats, get_ruleset_stats('C2', c2_ruleset)])\n",
    "    c2_confusion_matrix += confusion_matrix\n",
    "\n",
    "c2_confusion_matrix /= N_SPLITS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rules characteristics "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "time_total_s                    0.292132\n",
       "time_growing_s                  0.227262\n",
       "time_pruning_s                  0.047455\n",
       "rules_count                    34.200000\n",
       "conditions_per_rule             4.720378\n",
       "induced_conditions_per_rule    21.124536\n",
       "avg_rule_coverage               0.239541\n",
       "avg_rule_precision              0.690010\n",
       "avg_rule_quality                0.337021\n",
       "pvalue                          0.014757\n",
       "FDR_pvalue                      0.015234\n",
       "FWER_pvalue                     0.030014\n",
       "fraction_significant            0.909792\n",
       "fraction_FDR_significant        0.909792\n",
       "fraction_FWER_significant       0.872710\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(c2_ruleset_stats.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rules evaluation (average)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Accuracy                   0.855063\n",
       "MAE                        0.144937\n",
       "Kappa                      0.107556\n",
       "Balanced accuracy          0.567020\n",
       "Logistic loss              5.224070\n",
       "Precision                  5.224070\n",
       "Sensitivity                0.235294\n",
       "Specificity                0.898747\n",
       "NPV                        0.945594\n",
       "PPV                        0.495355\n",
       "psep                       0.443788\n",
       "Fall-out                   0.101253\n",
       "Youden's J statistic       0.134041\n",
       "Lift                       7.520255\n",
       "F-measure                  0.145015\n",
       "Fowlkes-Mallows index      0.870605\n",
       "False positive            24.500000\n",
       "False negative            13.000000\n",
       "True positive              4.000000\n",
       "True negative            216.900000\n",
       "Rules per example          7.871479\n",
       "Voting conflicts         179.000000\n",
       "Geometric mean             0.344832\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(c2_prediction_metrics.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion matrix (average)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>216.9</td>\n",
       "      <td>24.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0     1\n",
       "0  216.9  24.5\n",
       "1   13.0   4.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(pd.DataFrame(c2_confusion_matrix))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters tuning\n",
    "\n",
    "This one gonna take a while..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best BAC: 0.626780 using {'induction_measure': <Measures.WeightedLaplace: 'WeightedLaplace'>, 'minsupp_new': 5, 'pruning_measure': <Measures.RSS: 'RSS'>, 'voting_measure': <Measures.WeightedLaplace: 'WeightedLaplace'>}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from rulekit.params import Measures\n",
    "\n",
    "N_SPLITS: int = 3\n",
    "\n",
    "# define models and parameters\n",
    "model = RuleClassifier()\n",
    "minsupp_new = range(3, 15, 2)\n",
    "measures_choice = [Measures.C2, Measures.RSS, Measures.WeightedLaplace, Measures.Correlation]\n",
    "\n",
    "# define grid search\n",
    "grid = {\n",
    "    'minsupp_new': minsupp_new, \n",
    "    'induction_measure': measures_choice, \n",
    "    'pruning_measure': measures_choice, \n",
    "    'voting_measure': measures_choice\n",
    "}\n",
    "cv = StratifiedKFold(n_splits=N_SPLITS)\n",
    "grid_search = GridSearchCV(\n",
    "    estimator=model, \n",
    "    param_grid=grid, \n",
    "    cv=cv, \n",
    "    scoring='balanced_accuracy', \n",
    "    n_jobs=3\n",
    ")\n",
    "grid_result = grid_search.fit(X, y)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best BAC: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building model with tuned hyperparameters\n",
    "\n",
    "### Split dataset to train and test (80%/20%)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from IPython.display import display\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True)\n",
    "\n",
    "\n",
    "clf = RuleClassifier(**grid_result.best_params_)\n",
    "clf.fit(X_train, y_train)\n",
    "ruleset: RuleSet[ClassificationRule] = clf.model\n",
    "ruleset_stats = get_ruleset_stats('Best', ruleset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rules evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "time_total_s                    0.173054\n",
       "time_growing_s                  0.120460\n",
       "time_pruning_s                  0.029986\n",
       "rules_count                    29.000000\n",
       "conditions_per_rule             2.689655\n",
       "induced_conditions_per_rule    15.310345\n",
       "avg_rule_coverage               0.491183\n",
       "avg_rule_precision              0.736226\n",
       "avg_rule_quality                1.309334\n",
       "pvalue                          0.019831\n",
       "FDR_pvalue                      0.019993\n",
       "FWER_pvalue                     0.024284\n",
       "fraction_significant            0.931034\n",
       "fraction_FDR_significant        0.931034\n",
       "fraction_FWER_significant       0.931034\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(ruleset_stats.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validate model on test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Accuracy                   0.808511\n",
       "MAE                        0.191489\n",
       "Kappa                      0.170010\n",
       "Balanced accuracy          0.679398\n",
       "Logistic loss              6.901976\n",
       "Precision                  6.901976\n",
       "Sensitivity                0.533333\n",
       "Specificity                0.825462\n",
       "NPV                        0.966346\n",
       "PPV                        0.158416\n",
       "psep                       0.124762\n",
       "Fall-out                   0.174538\n",
       "Youden's J statistic       0.358795\n",
       "Lift                       2.730033\n",
       "F-measure                  0.244275\n",
       "Fowlkes-Mallows index      0.809997\n",
       "False positive            85.000000\n",
       "False negative            14.000000\n",
       "True positive             16.000000\n",
       "True negative            402.000000\n",
       "Rules per example         14.034816\n",
       "Voting conflicts         360.000000\n",
       "Geometric mean             0.663511\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>402</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0   1\n",
       "0  402  85\n",
       "1   14  16"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prediction, classification_metrics = clf.predict(X_test, return_metrics=True)\n",
    "prediction_metrics, confusion_matrix = get_prediction_metrics('Best', prediction, y_test, classification_metrics)\n",
    "\n",
    "display(prediction_metrics.mean())\n",
    "display(pd.DataFrame(confusion_matrix))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tutorials_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}