ch7_data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ch7 data cleaning and preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "PREVIOUS_MAX_ROWS = pd.options.display.max_rows\n",
    "pd.options.display.max_rows = 20\n",
    "np.random.seed(12345)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7.1 handling missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0     aardvark\n1    artichoke\n2          NaN\n3      avocado\ndtype: object"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "string_data = pd.Series(['aardvark', 'artichoke', np.nan, 'avocado'])\n",
    "string_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    False\n1    False\n2     True\n3    False\ndtype: bool"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "string_data.isnull()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## filter out missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    3.5\n3    NaN\n4    7.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "from numpy import nan as NA\n",
    "data = pd.Series([1, NA, 3.5, NA, 7])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n2    3.5\n4    7.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    3.5\n3    NaN\n4    7.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n2    3.5\n4    7.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "data[data.notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "data = pd.DataFrame([[1., 6.5, 3.], [1., NA, NA],\n",
    "                     [NA, NA, NA], [NA, 6.5, 3.]])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "cleaned = data.dropna()\n",
    "cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "data.dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "data.dropna(how='all', axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2   4\n0  1.0  6.5  3.0 NaN\n1  1.0  NaN  NaN NaN\n2  NaN  NaN  NaN NaN\n3  NaN  6.5  3.0 NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "data[4] = NA\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "data.dropna(how='all', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.478943 -0.519439\n1 -0.555730  1.965781  1.393406\n2  0.092908  0.281746  0.769023\n3  1.246435  1.007189 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.478943</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>1.965781</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.281746</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>1.007189</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.randn(7, 3))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708       NaN -0.519439\n1 -0.555730       NaN  1.393406\n2  0.092908       NaN  0.769023\n3  1.246435       NaN -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>NaN</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>NaN</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>NaN</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>NaN</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "df.iloc[:4, 1] = NA\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708       NaN -0.519439\n1 -0.555730       NaN  1.393406\n2  0.092908       NaN       NaN\n3  1.246435       NaN -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>NaN</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>NaN</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>NaN</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "df.iloc[2, 2] = NA\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708       NaN -0.519439\n1 -0.555730       NaN  1.393406\n3  1.246435       NaN -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>NaN</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>NaN</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>NaN</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "df.dropna(thresh=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## filling in missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.000000 -0.519439\n1 -0.555730  0.000000  1.393406\n2  0.092908  0.000000  0.000000\n3  1.246435  0.000000 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.000000</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>0.000000</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>0.000000</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.500000 -0.519439\n1 -0.555730  0.500000  1.393406\n2  0.092908  0.500000  0.000000\n3  1.246435  0.500000 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.500000</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>0.500000</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.500000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>0.500000</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 20
    }
   ],
   "source": [
    "df.fillna({1: 0.5, 2: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.000000 -0.519439\n1 -0.555730  0.000000  1.393406\n2  0.092908  0.000000  0.000000\n3  1.246435  0.000000 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.000000</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>0.000000</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>0.000000</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "_ = df.fillna(0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772  0.000940  1.343810\n3 -0.713544 -0.831154 -2.370232\n4 -1.860761 -0.860757  0.560145\n5 -1.265934  0.119827 -1.063512",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>0.000940</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>-0.831154</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>-0.860757</td>\n      <td>0.560145</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>0.119827</td>\n      <td>-1.063512</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.randn(6, 3))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772       NaN  1.343810\n3 -0.713544       NaN -2.370232\n4 -1.860761       NaN       NaN\n5 -1.265934       NaN       NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>NaN</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>NaN</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "df.iloc[2:, 1] = NA\n",
    "df.iloc[4:, 2] = NA\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772  0.124121  1.343810\n3 -0.713544  0.124121 -2.370232\n4 -1.860761  0.124121 -2.370232\n5 -1.265934  0.124121 -2.370232",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>0.124121</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "df.fillna(method='ffill')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772  0.124121  1.343810\n3 -0.713544  0.124121 -2.370232\n4 -1.860761       NaN -2.370232\n5 -1.265934       NaN -2.370232",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>0.124121</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>NaN</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>NaN</td>\n      <td>-2.370232</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "df.fillna(method='ffill', limit=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    3.5\n3    NaN\n4    7.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "source": [
    "data = pd.Series([1., NA, 3.5, NA, 7])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.000000\n1    3.833333\n2    3.500000\n3    3.833333\n4    7.000000\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "data.fillna(data.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7.2 data transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## remove duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4\n6  two   4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 28
    }
   ],
   "source": [
    "data = pd.DataFrame({'k1': ['one', 'two'] * 3 + ['two'],\n",
    "                     'k2': [1, 1, 2, 3, 3, 4, 4]})\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    False\n1    False\n2    False\n3    False\n4    False\n5    False\n6     True\ndtype: bool"
     },
     "metadata": {},
     "execution_count": 29
    }
   ],
   "source": [
    "data.duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 30
    }
   ],
   "source": [
    "data.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4\n6  two   4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 31
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    k1  k2  v1\n0  one   1   0\n1  two   1   1\n2  one   2   2\n3  two   3   3\n4  one   3   4\n5  two   4   5\n6  two   4   6",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "source": [
    "data['v1'] = range(7)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    k1  k2  v1\n0  one   1   0\n1  two   1   1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 33
    }
   ],
   "source": [
    "data.drop_duplicates(['k1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    k1  k2  v1\n0  one   1   0\n1  two   1   1\n2  one   2   2\n3  two   3   3\n4  one   3   4\n6  two   4   6",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "data.drop_duplicates(['k1', 'k2'], keep='last')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## transforming data using function or mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          food  ounces\n0        bacon     4.0\n1  pulled pork     3.0\n2        bacon    12.0\n3     Pastrami     6.0\n4  corned beef     7.5\n5        Bacon     8.0\n6     pastrami     3.0\n7    honey ham     5.0\n8     nova lox     6.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "data = pd.DataFrame({'food': ['bacon', 'pulled pork', 'bacon',\n",
    "                              'Pastrami', 'corned beef', 'Bacon',\n",
    "                              'pastrami', 'honey ham', 'nova lox'],\n",
    "                     'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "meat_to_animal = {\n",
    "  'bacon': 'pig',\n",
    "  'pulled pork': 'pig',\n",
    "  'pastrami': 'cow',\n",
    "  'corned beef': 'cow',\n",
    "  'honey ham': 'pig',\n",
    "  'nova lox': 'salmon'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0          bacon\n1    pulled pork\n2          bacon\n3       pastrami\n4    corned beef\n5          bacon\n6       pastrami\n7      honey ham\n8       nova lox\nName: food, dtype: object"
     },
     "metadata": {},
     "execution_count": 37
    }
   ],
   "source": [
    "lowercased = data['food'].str.lower()\n",
    "lowercased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          food  ounces  animal\n0        bacon     4.0     pig\n1  pulled pork     3.0     pig\n2        bacon    12.0     pig\n3     Pastrami     6.0     cow\n4  corned beef     7.5     cow\n5        Bacon     8.0     pig\n6     pastrami     3.0     cow\n7    honey ham     5.0     pig\n8     nova lox     6.0  salmon",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n      <th>animal</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n      <td>salmon</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 38
    }
   ],
   "source": [
    "data['animal'] = lowercased.map(meat_to_animal)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0       pig\n1       pig\n2       pig\n3       cow\n4       cow\n5       pig\n6       cow\n7       pig\n8    salmon\nName: food, dtype: object"
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "source": [
    "data['food'].map(lambda x : meat_to_animal[x.lower()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          food  ounces  animal     ani\n0        bacon     4.0     pig     pig\n1  pulled pork     3.0     pig     pig\n2        bacon    12.0     pig     pig\n3     Pastrami     6.0     cow     cow\n4  corned beef     7.5     cow     cow\n5        Bacon     8.0     pig     pig\n6     pastrami     3.0     cow     cow\n7    honey ham     5.0     pig     pig\n8     nova lox     6.0  salmon  salmon",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n      <th>animal</th>\n      <th>ani</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n      <td>pig</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n      <td>pig</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n      <td>pig</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n      <td>cow</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n      <td>cow</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n      <td>pig</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n      <td>cow</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n      <td>pig</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n      <td>salmon</td>\n      <td>salmon</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 40
    }
   ],
   "source": [
    "data['ani'] = data['food'].map(lambda x : meat_to_animal[x.lower()])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## replacing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0       1.0\n1    -999.0\n2       2.0\n3    -999.0\n4   -1000.0\n5       3.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 41
    }
   ],
   "source": [
    "data = pd.Series([1., -999., 2., -999., -1000., 3.])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0       1.0\n1       NaN\n2       2.0\n3       NaN\n4   -1000.0\n5       3.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 42
    }
   ],
   "source": [
    "data.replace(-999, np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    NaN\n5    3.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 43
    }
   ],
   "source": [
    "data.replace([-999, -1000], np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    0.0\n5    3.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 44
    }
   ],
   "source": [
    "data.replace([-999, -1000], [np.nan, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0      1.0\n1      NaN\n2      2.0\n3      NaN\n4   -999.0\n5      3.0\ndtype: float64"
     },
     "metadata": {},
     "execution_count": 45
    }
   ],
   "source": [
    "data.replace({-999: np.nan, -1000:-999})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## renaming axis indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "          one  two  three  four\nOhio        0    1      2     3\nColorado    4    5      6     7\nNew York    8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Ohio</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>Colorado</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>New York</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 46
    }
   ],
   "source": [
    "data = pd.DataFrame(np.arange(12).reshape((3, 4)),\n",
    "                    index=['Ohio', 'Colorado', 'New York'],\n",
    "                    columns=['one', 'two', 'three', 'four'])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Index(['OHIO', 'COLO', 'NEW '], dtype='object')"
     },
     "metadata": {},
     "execution_count": 47
    }
   ],
   "source": [
    "transform_func = lambda x: x[:4].upper()\n",
    "data.index.map(transform_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.index = data.index.map(transform_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "      one  two  three  four\nOHIO    0    1      2     3\nCOLO    4    5      6     7\nNEW     8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>OHIO</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 49
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "      ONE  TWO  THREE  FOUR\nOhio    0    1      2     3\nColo    4    5      6     7\nNew     8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ONE</th>\n      <th>TWO</th>\n      <th>THREE</th>\n      <th>FOUR</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Ohio</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>Colo</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>New</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 50
    }
   ],
   "source": [
    "data.rename(index=str.title, columns=str.upper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "      one  two  three  four\nOHIO    0    1      2     3\nCOLO    4    5      6     7\nNEW     8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>OHIO</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 51
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "         one  two  peekaboo  four\nINDIANA    0    1         2     3\nCOLO       4    5         6     7\nNEW        8    9        10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>peekaboo</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>INDIANA</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 52
    }
   ],
   "source": [
    "data.rename(\n",
    "    index={'OHIO' : 'INDIANA'}, \n",
    "    columns={'three' : 'peekaboo'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "         one  two  peekaboo  four\nINDIANA    0    1         2     3\nCOLO       4    5         6     7\nNEW        8    9        10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>peekaboo</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>INDIANA</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 53
    }
   ],
   "source": [
    "data.rename(\n",
    "    index={'OHIO' : 'INDIANA', 'NY': 'new_york'}, \n",
    "    columns={'three' : 'peekaboo'}\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "         one  two  peekaboo  four\nINDIANA    0    1         2     3\nCOLO       4    5         6     7\nNEW        8    9        10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>peekaboo</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>INDIANA</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 54
    }
   ],
   "source": [
    "data.rename(\n",
    "    index={'OHIO' : 'INDIANA', 'NY': 'new_york'}, \n",
    "    columns={'three' : 'peekaboo'}, \n",
    "    inplace=True\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## discretization and binning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]\nLength: 12\nCategories (4, interval[int64]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]"
     },
     "metadata": {},
     "execution_count": 56
    }
   ],
   "source": [
    "bins = [18, 25, 35, 60, 100]\n",
    "cats = pd.cut(ages, bins)\n",
    "cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)"
     },
     "metadata": {},
     "execution_count": 57
    }
   ],
   "source": [
    "cats.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]],\n              closed='right',\n              dtype='interval[int64]')"
     },
     "metadata": {},
     "execution_count": 58
    }
   ],
   "source": [
    "cats.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(18, 25]     5\n(35, 60]     3\n(25, 35]     3\n(60, 100]    1\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 59
    }
   ],
   "source": [
    "pd.value_counts(cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[[18, 26), [18, 26), [18, 26), [26, 36), [18, 26), ..., [26, 36), [61, 100), [36, 61), [36, 61), [26, 36)]\nLength: 12\nCategories (4, interval[int64]): [[18, 26) < [26, 36) < [36, 61) < [61, 100)]"
     },
     "metadata": {},
     "execution_count": 61
    }
   ],
   "source": [
    "pd.cut(ages, [18, 26, 36, 61, 100], right=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[Youth, Youth, Youth, YoungAdult, Youth, ..., YoungAdult, Senior, MiddleAged, MiddleAged, YoungAdult]\nLength: 12\nCategories (4, object): [Youth < YoungAdult < MiddleAged < Senior]"
     },
     "metadata": {},
     "execution_count": 62
    }
   ],
   "source": [
    "group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']\n",
    "pd.cut(ages, bins, labels=group_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([0.4896, 0.3773, 0.8486, 0.9111, 0.3838, 0.3155, 0.5684, 0.1878,\n       0.1258, 0.6876, 0.7996, 0.5735, 0.9732, 0.6341, 0.8884, 0.4954,\n       0.3516, 0.7142, 0.5039, 0.2256])"
     },
     "metadata": {},
     "execution_count": 63
    }
   ],
   "source": [
    "data = np.random.rand(20)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(0.34, 0.55], (0.34, 0.55], (0.76, 0.97], (0.76, 0.97], (0.34, 0.55], ..., (0.34, 0.55], (0.34, 0.55], (0.55, 0.76], (0.34, 0.55], (0.12, 0.34]]\nLength: 20\nCategories (4, interval[float64]): [(0.12, 0.34] < (0.34, 0.55] < (0.55, 0.76] < (0.76, 0.97]]"
     },
     "metadata": {},
     "execution_count": 65
    }
   ],
   "source": [
    "pd.cut(data, 4, precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.random.randn(1000)  # Normally distributed\n",
    "# data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(-3.746, -0.612], (0.681, 3.26], (-3.746, -0.612], (-0.011, 0.681], (0.681, 3.26], ..., (-3.746, -0.612], (-0.011, 0.681], (0.681, 3.26], (-0.011, 0.681], (-0.612, -0.011]]\nLength: 1000\nCategories (4, interval[float64]): [(-3.746, -0.612] < (-0.612, -0.011] < (-0.011, 0.681] < (0.681, 3.26]]"
     },
     "metadata": {},
     "execution_count": 68
    }
   ],
   "source": [
    "cats = pd.qcut(data, 4)\n",
    "cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(0.681, 3.26]       250\n(-0.011, 0.681]     250\n(-0.612, -0.011]    250\n(-3.746, -0.612]    250\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 69
    }
   ],
   "source": [
    "pd.value_counts(cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[(-1.299, -0.011], (-0.011, 1.338], (-1.299, -0.011], (-0.011, 1.338], (-0.011, 1.338], ..., (-3.746, -1.299], (-0.011, 1.338], (-0.011, 1.338], (-0.011, 1.338], (-1.299, -0.011]]\nLength: 1000\nCategories (4, interval[float64]): [(-3.746, -1.299] < (-1.299, -0.011] < (-0.011, 1.338] < (1.338, 3.26]]"
     },
     "metadata": {},
     "execution_count": 70
    }
   ],
   "source": [
    "pd.qcut(data, [0, 0.1, 0.5, 0.9, 1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## detecting and filtering outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                 0            1            2            3\ncount  1000.000000  1000.000000  1000.000000  1000.000000\nmean      0.004788    -0.007611    -0.046584    -0.062845\nstd       1.026866     1.001232     0.988780     1.006448\nmin      -3.183867    -3.481593    -3.194414    -3.530912\n25%      -0.688238    -0.692893    -0.752298    -0.755464\n50%       0.001966     0.032160    -0.034938    -0.035628\n75%       0.708299     0.694813     0.617693     0.611504\nmax       3.189940     2.961194     3.023720     2.916153",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>0.004788</td>\n      <td>-0.007611</td>\n      <td>-0.046584</td>\n      <td>-0.062845</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>1.026866</td>\n      <td>1.001232</td>\n      <td>0.988780</td>\n      <td>1.006448</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>-3.183867</td>\n      <td>-3.481593</td>\n      <td>-3.194414</td>\n      <td>-3.530912</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>-0.688238</td>\n      <td>-0.692893</td>\n      <td>-0.752298</td>\n      <td>-0.755464</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>0.001966</td>\n      <td>0.032160</td>\n      <td>-0.034938</td>\n      <td>-0.035628</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.708299</td>\n      <td>0.694813</td>\n      <td>0.617693</td>\n      <td>0.611504</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>3.189940</td>\n      <td>2.961194</td>\n      <td>3.023720</td>\n      <td>2.916153</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 72
    }
   ],
   "source": [
    "data = pd.DataFrame(np.random.randn(1000, 4))\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0     -0.115903\n1     -0.752216\n2      0.452252\n3     -0.016703\n4     -0.826004\n         ...   \n995    0.809087\n996    0.509974\n997    0.675436\n998   -0.978421\n999    0.604569\nName: 2, Length: 1000, dtype: float64"
     },
     "metadata": {},
     "execution_count": 74
    }
   ],
   "source": [
    "col = data[2]\n",
    "col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "224   -3.194414\n254    3.023720\n644   -3.044612\n834   -3.105636\nName: 2, dtype: float64"
     },
     "metadata": {},
     "execution_count": 75
    }
   ],
   "source": [
    "col[np.abs(col) > 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "            0         1         2         3\n12  -3.183867  1.050471 -1.042736  1.680374\n59  -3.140963 -1.509976 -0.389818 -0.273253\n224  1.090038 -0.848098 -3.194414  0.077839\n254  0.003349 -0.011807  3.023720 -1.105312\n283  0.452649 -3.481593  0.789944  1.737746\n452  3.082067 -0.516982  0.251909 -0.029354\n480  3.189940  0.070978  0.516982 -0.805171\n644 -0.436479  0.901529 -3.044612 -1.193980\n678 -1.148738 -3.170292 -1.017073 -1.147658\n834 -1.202724 -0.286215 -3.105636 -0.369009\n846 -3.024110 -1.168413 -0.888664  0.111410\n906  0.126636  1.666106  0.691626 -3.530912",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>12</th>\n      <td>-3.183867</td>\n      <td>1.050471</td>\n      <td>-1.042736</td>\n      <td>1.680374</td>\n    </tr>\n    <tr>\n      <th>59</th>\n      <td>-3.140963</td>\n      <td>-1.509976</td>\n      <td>-0.389818</td>\n      <td>-0.273253</td>\n    </tr>\n    <tr>\n      <th>224</th>\n      <td>1.090038</td>\n      <td>-0.848098</td>\n      <td>-3.194414</td>\n      <td>0.077839</td>\n    </tr>\n    <tr>\n      <th>254</th>\n      <td>0.003349</td>\n      <td>-0.011807</td>\n      <td>3.023720</td>\n      <td>-1.105312</td>\n    </tr>\n    <tr>\n      <th>283</th>\n      <td>0.452649</td>\n      <td>-3.481593</td>\n      <td>0.789944</td>\n      <td>1.737746</td>\n    </tr>\n    <tr>\n      <th>452</th>\n      <td>3.082067</td>\n      <td>-0.516982</td>\n      <td>0.251909</td>\n      <td>-0.029354</td>\n    </tr>\n    <tr>\n      <th>480</th>\n      <td>3.189940</td>\n      <td>0.070978</td>\n      <td>0.516982</td>\n      <td>-0.805171</td>\n    </tr>\n    <tr>\n      <th>644</th>\n      <td>-0.436479</td>\n      <td>0.901529</td>\n      <td>-3.044612</td>\n      <td>-1.193980</td>\n    </tr>\n    <tr>\n      <th>678</th>\n      <td>-1.148738</td>\n      <td>-3.170292</td>\n      <td>-1.017073</td>\n      <td>-1.147658</td>\n    </tr>\n    <tr>\n      <th>834</th>\n      <td>-1.202724</td>\n      <td>-0.286215</td>\n      <td>-3.105636</td>\n      <td>-0.369009</td>\n    </tr>\n    <tr>\n      <th>846</th>\n      <td>-3.024110</td>\n      <td>-1.168413</td>\n      <td>-0.888664</td>\n      <td>0.111410</td>\n    </tr>\n    <tr>\n      <th>906</th>\n      <td>0.126636</td>\n      <td>1.666106</td>\n      <td>0.691626</td>\n      <td>-3.530912</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 76
    }
   ],
   "source": [
    "data[(np.abs(data) >3).any(1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                 0            1            2            3\ncount  1000.000000  1000.000000  1000.000000  1000.000000\nmean      0.004865    -0.006960    -0.046264    -0.062314\nstd       1.025001     0.999148     0.987650     1.004755\nmin      -3.000000    -3.000000    -3.000000    -3.000000\n25%      -0.688238    -0.692893    -0.752298    -0.755464\n50%       0.001966     0.032160    -0.034938    -0.035628\n75%       0.708299     0.694813     0.617693     0.611504\nmax       3.000000     2.961194     3.000000     2.916153",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>0.004865</td>\n      <td>-0.006960</td>\n      <td>-0.046264</td>\n      <td>-0.062314</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>1.025001</td>\n      <td>0.999148</td>\n      <td>0.987650</td>\n      <td>1.004755</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>-0.688238</td>\n      <td>-0.692893</td>\n      <td>-0.752298</td>\n      <td>-0.755464</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>0.001966</td>\n      <td>0.032160</td>\n      <td>-0.034938</td>\n      <td>-0.035628</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.708299</td>\n      <td>0.694813</td>\n      <td>0.617693</td>\n      <td>0.611504</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>3.000000</td>\n      <td>2.961194</td>\n      <td>3.000000</td>\n      <td>2.916153</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 77
    }
   ],
   "source": [
    "data[np.abs(data) >3] = np.sign(data) *3\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     0    1    2    3\n0 -1.0  1.0 -1.0 -1.0\n1  1.0 -1.0 -1.0 -1.0\n2 -1.0  1.0  1.0 -1.0\n3 -1.0  1.0 -1.0 -1.0\n4 -1.0 -1.0 -1.0  1.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>-1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.0</td>\n      <td>-1.0</td>\n      <td>-1.0</td>\n      <td>1.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 78
    }
   ],
   "source": [
    "np.sign(data).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## permutation and random sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    0   1   2   3\n0   0   1   2   3\n1   4   5   6   7\n2   8   9  10  11\n3  12  13  14  15\n4  16  17  18  19",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>12</td>\n      <td>13</td>\n      <td>14</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 79
    }
   ],
   "source": [
    "df = pd.DataFrame(np.arange(5 * 4).reshape((5, 4)))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([2, 4, 0, 3, 1])"
     },
     "metadata": {},
     "execution_count": 80
    }
   ],
   "source": [
    "sampler = np.random.permutation(5)\n",
    "sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    0   1   2   3\n2   8   9  10  11\n4  16  17  18  19\n0   0   1   2   3\n3  12  13  14  15\n1   4   5   6   7",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>12</td>\n      <td>13</td>\n      <td>14</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 81
    }
   ],
   "source": [
    "df.take(sampler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    0   1   2   3\n0   0   1   2   3\n4  16  17  18  19\n3  12  13  14  15",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>12</td>\n      <td>13</td>\n      <td>14</td>\n      <td>15</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 82
    }
   ],
   "source": [
    "df.sample(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    0   1   2   3\n1   4   5   6   7\n0   0   1   2   3\n4  16  17  18  19",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 83
    }
   ],
   "source": [
    "df.sample(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0    5\n1    7\n2   -1\n3    6\n4    4\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 84
    }
   ],
   "source": [
    "choices = pd.Series([5, 7, -1, 6, 4])\n",
    "choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "3    6\n3    6\n3    6\n3    6\n3    6\n0    5\n0    5\n2   -1\n0    5\n2   -1\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 86
    }
   ],
   "source": [
    "draws = choices.sample(n=10, replace=True)\n",
    "draws"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## computing indicators / dummy variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37564bit42731fe29a1c4072b88558cb14c14b00",
   "display_name": "Python 3.7.5 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}