Unit-2 Ipynb

{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# Unit-2\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"## Data Loading"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"### Loading the dataset\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### importing the data the mail data\n"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"X-GM-THRID\n",
"X-Gmail-Labels\n",
"MIME-Version\n",
"Date\n",
"Message-ID\n",
"Subject\n",
"From\n",
"To\n",
"Content-Type\n"
]
},
{
"data": {
"text/plain": [
"<mailbox.mbox at 0x12a9cacd490>"
]
},
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import mailbox\n",
"\n",
"mboxfile = \"./All mail Including Spam and Trash.mbox\"\n",
"mbox = mailbox.mbox(mboxfile)\n",
"\n",
"for key in mbox[0].keys():\n",
" print(key)\n",
"mbox"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"### Data transformation"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Data cleansing"
]
},
{
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"\n",
"with open('testdata.csv','w') as outputfile:\n",
" writing = csv.writer(outputfile)\n",
" writing.writerow(['subject', 'from', 'date', 'to', 'label','Thread'])\n",
"\n",
" for message in mbox:\n",
" writing.writerow([\n",
" message['subject'],\n",
" message['from'],\n",
" message['date'],\n",
" message['to'],\n",
" message['X-Gmail-Labels'],\n",
" message['X-GM-THRID']\n",
" ])\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Loading the csv file\n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>Kadarla Sai Nithin <sainithink.be25@uceou.edu></td>\n",
" <td>Wed, 11 Jan 2023 10:00:04 +0530</td>\n",
" <td>Suresh Kumar Lokhande <suresh.l@uceou.edu></td>\n",
" <td>Archived,Sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>ThingSpeak Support <support@mail.thingspeak.com></td>\n",
" <td>Sat, 22 Apr 2023 09:56:31 +0000</td>\n",
" <td>sainithink.be25@uceou.edu</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>Gmail Team <mail-noreply@google.com></td>\n",
" <td>Wed, 30 Nov 2022 00:05:59 -0800</td>\n",
" <td>Inbox,Unread</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>Xmind Notifications <notifications@mail.xmind....</td>\n",
" <td>Tue, 16 May 2023 15:37:20 +0000</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>Google <no-reply@accounts.google.com></td>\n",
" <td>Sat, 25 Feb 2023 04:51:44 GMT</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>Verify Email Address</td>\n",
" <td>service@account.mathworks.com</td>\n",
" <td>Sat, 22 Apr 2023 09:55:12 +0000</td>\n",
" <td>1763869787228555223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62</th>\n",
" <td>uceou.edu account email verification code</td>\n",
" <td>msonlineservicesteam@microsoftonline.com</td>\n",
" <td>Sun, 19 Mar 2023 23:55:52 -0700</td>\n",
" <td>Sainithink.be25@uceou.edu</td>\n",
" <td>1760868809297254026</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>Mon, 13 Mar 2023 23:40:25 -0700</td>\n",
" <td>1760324258667829753</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64</th>\n",
" <td>Fri, 19 May 2023 10:03:42 GMT</td>\n",
" <td>Inbox,Opened,Category Updates</td>\n",
" <td>1766316440594475561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
" <td>[Request received]</td>\n",
" <td>\"Tech (Support)\" <agents@xmind.zendesk.com></td>\n",
" <td>Wed, 08 Feb 2023 14:48:12 +0000</td>\n",
" <td>\"sainithink.be25\" <sainithink.be25@uceou.edu></td>\n",
" <td>1757274643381080149</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>66 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
"0 Conceptual Physics book file \n",
"1 Thank you for signing up for ThingSpeak! \n",
"2 Get the official Gmail app \n",
"3 Reset Your Password \n",
"4 Security alert \n",
".. ... \n",
"61 Verify Email Address \n",
"62 uceou.edu account email verification code \n",
"65 [Request received] \n",
"\n",
" from \\\n",
"0 Kadarla Sai Nithin <sainithink.be25@uceou.edu> \n",
"1 ThingSpeak Support <support@mail.thingspeak.com> \n",
"2 Gmail Team <mail-noreply@google.com> \n",
"3 Xmind Notifications <notifications@mail.xmind.... \n",
"4 Google <no-reply@accounts.google.com> \n",
".. ... \n",
"61 service@account.mathworks.com \n",
"62 msonlineservicesteam@microsoftonline.com \n",
"65 \"Tech (Support)\" <agents@xmind.zendesk.com> \n",
"\n",
" date \\\n",
"0 Wed, 11 Jan 2023 10:00:04 +0530 \n",
"1 Sat, 22 Apr 2023 09:56:31 +0000 \n",
"2 Wed, 30 Nov 2022 00:05:59 -0800 \n",
"3 Tue, 16 May 2023 15:37:20 +0000 \n",
"4 Sat, 25 Feb 2023 04:51:44 GMT \n",
".. ... \n",
"61 Sat, 22 Apr 2023 09:55:12 +0000 \n",
"62 Sun, 19 Mar 2023 23:55:52 -0700 \n",
"63 Mon, 13 Mar 2023 23:40:25 -0700 \n",
"64 Fri, 19 May 2023 10:03:42 GMT \n",
"65 Wed, 08 Feb 2023 14:48:12 +0000 \n",
"\n",
" to \\\n",
"0 Suresh Kumar Lokhande <suresh.l@uceou.edu> \n",
"1 sainithink.be25@uceou.edu \n",
".. ... \n",
"62 Sainithink.be25@uceou.edu \n",
"63 Sainithink.be25@uceou.edu \n",
"65 \"sainithink.be25\" <sainithink.be25@uceou.edu> \n",
"\n",
" label Thread \n",
"0 Archived,Sent 1754524927945493361 \n",
"1 Inbox,Category Updates,Unread 1763869869345248847 \n",
"2 Inbox,Unread 1750907548775931784 \n",
"3 Inbox,Important,Opened,Category Updates 1766065641328425275 \n",
".. ... ... \n",
"64 Inbox,Opened,Category Updates 1766316440594475561 \n",
"\n",
"[66 rows x 6 columns]"
]
},
"metadata": {},
}
],
"source": [
"dfs = pd.read_csv('testdata.csv')\n",
"dfs"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Converting the data"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"subject object\n",
"from object\n",
"date object\n",
"to object\n",
"label object\n",
"Thread int64\n",
"dtype: object\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>2023-04-22 09:55:12+00:00</td>\n",
" <td>1763869787228555223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62</th>\n",
" <td>2023-03-20 06:55:52+00:00</td>\n",
" <td>1760868809297254026</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>2023-03-14 06:40:25+00:00</td>\n",
" <td>1760324258667829753</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64</th>\n",
" <td>2023-05-19 10:03:42+00:00</td>\n",
" <td>Inbox,Opened,Category Updates</td>\n",
" <td>1766316440594475561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
" <td>\"Tech (Support)\" <agents@xmind.zendesk.com></td>\n",
" <td>2023-02-08 14:48:12+00:00</td>\n",
" <td>1757274643381080149</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
".. ... \n",
"\n",
" from \\\n",
".. ... \n",
"61 service@account.mathworks.com \n",
"65 \"Tech (Support)\" <agents@xmind.zendesk.com> \n",
"\n",
" date
to \\\n",
"0 2023-01-11 04:30:04+00:00 Suresh Kumar Lokhande
<suresh.l@uceou.edu> \n",
"1 2023-04-22 09:56:31+00:00
sainithink.be25@uceou.edu \n",
"2 2022-11-30 08:05:59+00:00 Kadarla Sai Nithin
<sainithink.be25@uceou.edu> \n",
"3 2023-05-16 15:37:20+00:00
"4 2023-02-25 04:51:44+00:00
".. ... ... \

n",
"61 2023-04-22 09:55:12+00:00
"62 2023-03-20 06:55:52+00:00
Sainithink.be25@uceou.edu \n",
"63 2023-03-14 06:40:25+00:00
Sainithink.be25@uceou.edu \n",
"64 2023-05-19 10:03:42+00:00
"65 2023-02-08 14:48:12+00:00 \"sainithink.be25\"
<sainithink.be25@uceou.edu> \n",
"\n",
" label Thread \n",
"0 Archived,Sent 1754524927945493361 \n",
"2 Inbox,Unread 1750907548775931784 \n",
".. ... ... \n",
"64 Inbox,Opened,Category Updates 1766316440594475561 \n",
"\n",
]
},
"metadata": {},
}
],
"source": [
"print(dfs.dtypes)\n",
"dfs['date'] = dfs['date'].apply(lambda x: pd.to_datetime(x,errors='coerce',
utc=True))\n",
"dfs"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Removing NaN values"
]
},
{
"metadata": {},
"outputs": [],
"source": [
"dfs= dfs[dfs['date'].notna()]\n",
"dfs.to_csv(\"gmaildata.csv\")\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Applying descriptive statistics"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"Int64Index: 66 entries, 0 to 65\n",
"Data columns (total 6 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 subject 66 non-null object \n",
" 1 from 66 non-null object \n",
" 2 date 66 non-null datetime64[ns, UTC]\n",
" 3 to 66 non-null object \n",
" 4 label 66 non-null object \n",
" 5 Thread 66 non-null int64 \n",
"dtypes: datetime64[ns, UTC](1), int64(1), object(4)\n",
"memory usage: 3.6+ KB\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
"\n",
" from \\\n",
"\n",
" date to
\\\n",
"0 2023-01-11 04:30:04+00:00 Suresh Kumar Lokhande <suresh.l@uceou.edu>
\n",
"1 2023-04-22 09:56:31+00:00 sainithink.be25@uceou.edu
\n",
"2 2022-11-30 08:05:59+00:00 Kadarla Sai Nithin <sainithink.be25@uceou.edu>
\n",
\n",
\n",
"\n",
" label Thread \n",
"0 Archived,Sent 1754524927945493361 \n",
"2 Inbox,Unread 1750907548775931784 \n",
"4 Inbox,Category Updates,Unread 1758777265723926184 "
]
},
"metadata": {},
}
],
"source": [
"dfs.info()\n",
"dfs.head()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"In the above output 'from' field has name and email at one column\n",
"Let's refactor it"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Data Refactoring"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>support@mail.thingspeak.com</td>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>inbox</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>mail-noreply@google.com</td>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>notifications@mail.xmind.net</td>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>no-reply@accounts.google.com</td>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>2023-04-22 09:55:12+00:00</td>\n",
" <td>1763869787228555223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62</th>\n",
" <td>2023-03-20 06:55:52+00:00</td>\n",
" <td>1760868809297254026</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>2023-03-14 06:40:25+00:00</td>\n",
" <td>1760324258667829753</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64</th>\n",
" <td>2023-05-19 10:03:42+00:00</td>\n",
" <td>1766316440594475561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
" <td>agents@xmind.zendesk.com</td>\n",
" <td>2023-02-08 14:48:12+00:00</td>\n",
" <td>1757274643381080149</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
".. ... \n",
"\n",
" from date \\\
n",
"0 sainithink.be25@uceou.edu 2023-01-11 04:30:04+00:00 \
n",
"1 support@mail.thingspeak.com 2023-04-22 09:56:31+00:00 \
n",
"2 mail-noreply@google.com 2022-11-30 08:05:59+00:00 \
n",
"3 notifications@mail.xmind.net 2023-05-16 15:37:20+00:00 \
n",
"4 no-reply@accounts.google.com 2023-02-25 04:51:44+00:00 \
n",
".. ... ... \
n",
"61 service@account.mathworks.com 2023-04-22 09:55:12+00:00 \
n",
"62 msonlineservicesteam@microsoftonline.com 2023-03-20 06:55:52+00:00 \
n",
"63 msonlineservicesteam@microsoftonline.com 2023-03-14 06:40:25+00:00 \
n",
"64 no-reply@accounts.google.com 2023-05-19 10:03:42+00:00 \
n",
"65 agents@xmind.zendesk.com 2023-02-08 14:48:12+00:00 \
n",
"\n",
" to label
Thread \n",
"0 Suresh Kumar Lokhande <suresh.l@uceou.edu> sent
1754524927945493361 \n",
"1 sainithink.be25@uceou.edu inbox
1763869869345248847 \n",
"2 Kadarla Sai Nithin <sainithink.be25@uceou.edu> inbox
1750907548775931784 \n",
1766065641328425275 \n",
1758777265723926184 \n",
".. ... ... ... \

n",
1763869787228555223 \n",
"62 Sainithink.be25@uceou.edu inbox
1760868809297254026 \n",
"63 Sainithink.be25@uceou.edu inbox
1760324258667829753 \n",
1766316440594475561 \n",
"65 \"sainithink.be25\" <sainithink.be25@uceou.edu> inbox
1757274643381080149 \n",
"\n",
]
},
"metadata": {},
}
],
"source": [
"import re\n",
"\n",
"def extract_email(string):\n",
" email = re.findall(r'<(.+?)>', string)\n",
" if not email:\n",
" email = list(filter(lambda y: '@' in y, string.split()))\n",
" return email[0] if email else np.nan\n",
"\n",
"dfs['from'] = dfs['from'].apply(lambda x: extract_email(x))\n",
"\n",
"dfs\n",
"\n",
"myemail = 'sainithink.be25@uceou.edu'\n",
"dfs['label'] = dfs['from'].apply(lambda x: 'sent' if x==myemail else 'inbox')\
n",
"\n",
"dfs"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Dropping columns\n",
"\n",
"We remove the 'to' coulmn as it contains our email only. So, we removed it\n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject
from \\\n",
"0 Conceptual Physics book file sainithink.be25@uceou.edu
\n",
"1 Thank you for signing up for ThingSpeak! support@mail.thingspeak.com
\n",
"2 Get the official Gmail app mail-noreply@google.com
\n",
"3 Reset Your Password notifications@mail.xmind.net
\n",
"4 Security alert no-reply@accounts.google.com
\n",
"\n",
" date label Thread \n",
"0 2023-01-11 04:30:04+00:00 sent 1754524927945493361 \n",
"1 2023-04-22 09:56:31+00:00 inbox 1763869869345248847 \n",
"2 2022-11-30 08:05:59+00:00 inbox 1750907548775931784 \n",
"3 2023-05-16 15:37:20+00:00 inbox 1766065641328425275 \n",
"4 2023-02-25 04:51:44+00:00 inbox 1758777265723926184 "
]
},
"metadata": {},
}
],
"source": [
"dfs.drop(columns='to',inplace=True)\n",
"dfs.head()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Refactoring timezones\n"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"datetime64[ns, US/Eastern]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>dayofweek</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2023-01-10 23:30:04-05:00</td>\n",
" <td>sent</td>\n",
" <td>1754524927945493361</td>\n",
" <td>Tuesday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2023-04-22 05:56:31-04:00</td>\n",
" <td>1763869869345248847</td>\n",
" <td>Saturday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2022-11-30 03:05:59-05:00</td>\n",
" <td>1750907548775931784</td>\n",
" <td>Wednesday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2023-05-16 11:37:20-04:00</td>\n",
" <td>1766065641328425275</td>\n",
" <td>Tuesday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2023-02-24 23:51:44-05:00</td>\n",
" <td>1758777265723926184</td>\n",
" <td>Friday</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject
from \\\n",
"0 Conceptual Physics book file sainithink.be25@uceou.edu
\n",
"1 Thank you for signing up for ThingSpeak! support@mail.thingspeak.com
\n",
"2 Get the official Gmail app mail-noreply@google.com
\n",
"3 Reset Your Password notifications@mail.xmind.net
\n",
"4 Security alert no-reply@accounts.google.com
\n",
"\n",
" date label Thread dayofweek \n",
"0 2023-01-10 23:30:04-05:00 sent 1754524927945493361 Tuesday \n",
"1 2023-04-22 05:56:31-04:00 inbox 1763869869345248847 Saturday \n",
"2 2022-11-30 03:05:59-05:00 inbox 1750907548775931784 Wednesday \n",
"3 2023-05-16 11:37:20-04:00 inbox 1766065641328425275 Tuesday \n",
"4 2023-02-24 23:51:44-05:00 inbox 1758777265723926184 Friday "
]
},
"metadata": {},
}
],
"source": [
"import datetime\n",
"import pytz\n",
"\n",
"def refactor_timezone(x):\n",
" est=pytz.timezone('US/Eastern')\n",
" return x.astimezone(est)\n",
"\n",
"dfs['date'] = dfs['date'].apply(lambda x: refactor_timezone(x))\n",
"\n",
"print(dfs['date'].dtype)\n",
"\n",
"dfs['dayofweek'] = dfs['date'].dt.strftime('%A')\n",
"\n",
"dfs.head()"
]
},
{
"metadata": {},
"outputs": [],
"source": [
"dfs.index = dfs['date']\n",
"#del dfs['date']"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"### Data Analysis\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Number of emails"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"Wed, 30 Nov 2022 03:05 AM\n",
"Wed, 24 May 2023 06:31 PM\n",
"inbox 65\n",
"sent 1\n",
"Name: label, dtype: int64\n"
]
}
],
"source": [
"print(dfs.index.min().strftime('%a, %d %b %Y %I:%M %p'))\n",
"print(dfs.index.max().strftime('%a, %d %b %Y %I:%M %p'))\n",
"\n",
"print(dfs['label'].value_counts())"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Time of the day "
]
},
{
"metadata": {},
"outputs": [],
"source": [
"sent = dfs[dfs['label']==\"sent\"]\n",
"recieved = dfs[dfs['label']==\"inbox\"]"
]
},
{
"metadata": {},
"outputs": [],
"source": [
"from matplotlib.ticker import MaxNLocator\n",
"from scipy import ndimage\n",
"import matplotlib.gridspec as gridspec\n",
"import matplotlib.patches as mpatches"
]
},
{
"metadata": {},
"outputs": [],
"source": [
"def plot_todo_vs_year(df, ax, color='CO', s=0.5, title=''):\n",
" ind = np.zeros(len(df), dtype='bool')\n",
" est = pytz.timezone('US/Eastern')\n",
" df[~ind].plot.scatter()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"## Data Cleaning\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"### Exploring data"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"Importing the dataset\n"
]
},
{
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"data = pd.read_csv('employee.csv')"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"Printing the first 5 records"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"0 Allen Smith 45.0 NaN NaN Operations G3
723\n",
"1 S Kumar NaN 16000.0 F Finance G0
520\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2
674\n",
"3 Ying Chin 45.0 65000.0 F Sales G3
556\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711"
]
},
"metadata": {},
}
],
"source": [
"data.head()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"Printing the last 5 records"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
711\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3
649\n",
"6 James Authur 54.0 NaN F Operations G3
53\n",
"7 Josh Wills 54.0 52000.0 F Finance G3
901\n",
"8 Leo Duck 23.0 98000.0 M Sales G4
709"
]
},
"metadata": {},
}
],
"source": [
"data.tail()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"listing the columns\n"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"Index(['name', 'age', 'income', 'gender', 'department', 'grade',\n",
" 'performance_score'],\n",
" dtype='object')\n"
]
}
],
"source": [
"print(data.columns)"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"Printing the shape of the dataframe\n",
"\n",
"Output indicates --> 9 rows and 7 columns"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"(9, 7)\n"
]
}
],
"source": [
"print(data.shape)"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"To check the table schema, its columns, rows, data types, and missing values
in the\n",
"DataFrame\n",
"\n",
"There 7 columns, 9 rows(entries) and their names, datatypes of the column,
missing values, no. of datatypes present in the dataframe"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"RangeIndex: 9 entries, 0 to 8\n",
"--- ------ -------------- ----- \n",
" 0 name 9 non-null object \n",
" 1 age 7 non-null float64\n",
" 2 income 7 non-null float64\n",
" 3 gender 7 non-null object \n",
" 4 department 9 non-null object \n",
" 5 grade 9 non-null object \n",
" 6 performance_score 9 non-null int64 \n",
"dtypes: float64(2), int64(1), object(4)\n",
"memory usage: 632.0+ bytes\n"
]
}
],
"source": [
"data.info()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"To check descriptive statistics of the dataframe"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>7.000000</td>\n",
" <td>7.000000</td>\n",
" <td>9.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>40.428571</td>\n",
" <td>52857.142857</td>\n",
" <td>610.666667</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>12.204605</td>\n",
" <td>26028.372797</td>\n",
" <td>235.671912</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>23.000000</td>\n",
" <td>16000.000000</td>\n",
" <td>53.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>31.000000</td>\n",
" <td>38500.000000</td>\n",
" <td>556.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>45.000000</td>\n",
" <td>52000.000000</td>\n",
" <td>674.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>49.500000</td>\n",
" <td>63500.000000</td>\n",
" <td>711.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>54.000000</td>\n",
" <td>98000.000000</td>\n",
" <td>901.000000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" age income performance_score\n",
"count 7.000000 7.000000 9.000000\n",
"mean 40.428571 52857.142857 610.666667\n",
"std 12.204605 26028.372797 235.671912\n",
"min 23.000000 16000.000000 53.000000\n",
"25% 31.000000 38500.000000 556.000000\n",
"50% 45.000000 52000.000000 674.000000\n",
"75% 49.500000 63500.000000 711.000000\n",
"max 54.000000 98000.000000 901.000000"
]
},
"metadata": {},
}
],
"source": [
"data.describe()"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"The above table indicates that range of age, average age, max age and min age"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"### Filtering data to weed out the noise"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Column-wise filtration"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"To filter columns, use filter() method\n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name department\n",
"0 Allen Smith Operations\n",
"1 S Kumar Finance\n",
"2 Jack Morgan Finance\n",
"3 Ying Chin Sales\n",
"4 Dheeraj Patel Operations\n",
"5 Satyam Sharma Sales\n",
"6 James Authur Operations\n",
"7 Josh Wills Finance\n",
"8 Leo Duck Sales"
]
},
"metadata": {},
}
],
"source": [
"data.filter(['name', 'department'])"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"We can also filter coulmns by slicing\n",
"\n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0 Allen Smith\n",
"1 S Kumar\n",
"2 Jack Morgan\n",
"3 Ying Chin\n",
"4 Dheeraj Patel\n",
"5 Satyam Sharma\n",
"6 James Authur\n",
"7 Josh Wills\n",
"8 Leo Duck\n",
"Name: name, dtype: object"
]
},
"metadata": {},
}
],
"source": [
"data['name']"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"To filter multiple columns, we need a list whereas single column list is not
needed "
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name department\n",
"0 Allen Smith Operations\n",
"1 S Kumar Finance\n",
"2 Jack Morgan Finance\n",
"3 Ying Chin Sales\n",
"4 Dheeraj Patel Operations\n",
"5 Satyam Sharma Sales\n",
"6 James Authur Operations\n",
"7 Josh Wills Finance\n",
"8 Leo Duck Sales"
]
},
"metadata": {},
}
],
"source": [
"data[['name', 'department']]"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"We can also select a single column using python list"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name\n",
"0 Allen Smith\n",
"1 S Kumar\n",
"2 Jack Morgan\n",
"3 Ying Chin\n",
"4 Dheeraj Patel\n",
"5 Satyam Sharma\n",
"6 James Authur\n",
"7 Josh Wills\n",
"8 Leo Duck"
]
},
"metadata": {},
}
],
"source": [
"data[['name']]"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Row-wise filtration"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" We can filter by using indices, slicing and conditions\n",
"\n",
" Here, axis = 0 for specifying the indices and axis = 1 for columns"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Indices \n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556"
]
},
"metadata": {},
}
],
"source": [
"data.filter([1,2,3], axis=0)"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Slicing "
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
711\n",
649\n",
53"
]
},
"metadata": {},
}
],
"source": [
"data[-5:-2]"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Condition-based filtration\n",
"\n",
" For single value, '==' operator \n",
" syntax: DataFrame[DataFrame.column_name == 'Data']\n",
"\n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709"
]
},
"metadata": {},
}
],
"source": [
"data[data.department=='Sales']"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" For multiple values, isin() method\n",
" syntax: DataFrame[DataFrame.column_name.isin(['value_1', 'value_2'])]"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
723\n",
556\n",
711\n",
649\n",
53\n",
709"
]
},
"metadata": {},
}
],
"source": [
"data[data.department.isin(['Sales', 'Operations'])]"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" For continuous variables, We can also use conditions like '<=' and '>='\
n",
" To Specifiy single condition:\n",
" Syntax: DataFrame[(DataFrame.column_name <= ''Value')]"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
723\n",
711\n",
901\n",
709"
]
},
"metadata": {},
}
],
"source": [
"data[(data.performance_score >=700)]"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" To specify multiple conditions\n",
" Syntax: DataFrame[(DataFrame.column_name <= ''Value') &
(DataFrame.column_name <= ''Value')]"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
n",
n",
n",
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649"
]
},
"metadata": {},
}
],
"source": [
"data[(data.performance_score <=700) & (data.performance_score >500)]"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" We can also use query method where we specify boolean expressions\n",
" Syntax: DataFrame.query('boolean expression')"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
n",
n",
n",
"6 James Authur 54.0 NaN F Operations G3 53"
]
},
"metadata": {},
}
],
"source": [
"data.query('performance_score < 600')"
]
},
{
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"attachments": {},
"metadata": {},
"source": [
"### Handling missing values"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Dropping missing values\n",
" Using dropna(), we can remove the rows that containing the missing values\
n",
"\n",
" This method has 'how' argument which takes two values 'any' and 'all'\n"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"2 Jack Morgan 32.0 35000.0 M Finance G2
674\n",
556\n",
711\n",
901\n",
709"
]
},
"metadata": {},
}
],
"source": [
"data2 = pd.read_csv('employee.csv')\n",
"\n",
"data2 = data2.dropna()\n",
"data2\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Filling in a missing value\n",
" \n",
" To fill a missing value, we use mean, median and mode\n",
"\n",
"\n",
" Now, Let's fill the mising values in 'age' column with mean\n"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723\
n",
n",
n",
n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711\
n",
n",
"6 James Authur 54.0 NaN F Operations G3 53\
n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>40.428571</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.000000</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.000000</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.000000</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>40.428571</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.000000</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.000000</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade \\\n",
"0 Allen Smith 45.000000 NaN NaN Operations G3 \n",
"1 S Kumar 40.428571 16000.0 F Finance G0 \n",
"2 Jack Morgan 32.000000 35000.0 M Finance G2 \n",
"3 Ying Chin 45.000000 65000.0 F Sales G3 \n",
"4 Dheeraj Patel 30.000000 42000.0 F Operations G2 \n",
"5 Satyam Sharma 40.428571 62000.0 NaN Sales G3 \n",
"6 James Authur 54.000000 NaN F Operations G3 \n",
"7 Josh Wills 54.000000 52000.0 F Finance G3 \n",
"8 Leo Duck 23.000000 98000.0 M Sales G4 \n",
"\n",
" performance_score \n",
"0 723 \n",
"1 520 \n",
"2 674 \n",
"3 556 \n",
"4 711 \n",
"5 649 \n",
"6 53 \n",
"7 901 \n",
"8 709 "
]
},
"metadata": {},
}
],
"source": [
"fillingsdf = pd.read_csv('employee.csv')\n",
"print(fillingsdf)\n",
"\n",
"fillingsdf['age'] = fillingsdf.age.fillna(fillingsdf.age.mean())\n",
"fillingsdf"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Now, Let's fill the mising values in 'income' column with median"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.000000</td>\n",
" <td>52000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>40.428571</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.000000</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.000000</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.000000</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>40.428571</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.000000</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"0 Allen Smith 45.000000 52000.0 NaN Operations G3 \n",
"5 Satyam Sharma 40.428571 62000.0 NaN Sales G3 \n",
"6 James Authur 54.000000 52000.0 F Operations G3 \n",
"\n",
"0 723 \n",
"1 520 \n",
"2 674 \n",
"3 556 \n",
"4 711 \n",
"5 649 \n",
"6 53 \n",
"7 901 \n",
"8 709 "
]
},
"metadata": {},
}
],
"source": [
"fillingsdf['income'] = fillingsdf.income.fillna(fillingsdf.income.median())\
n",
"fillingsdf"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Now, Let's fill the missing values in 'gender' column with mode"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>40.428571</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.000000</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.000000</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.000000</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>40.428571</td>\n",
" <td>62000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.000000</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"0 Allen Smith 45.000000 52000.0 F Operations G3 \n",
"5 Satyam Sharma 40.428571 62000.0 F Sales G3 \n",
"6 James Authur 54.000000 52000.0 F Operations G3 \n",
"\n",
"0 723 \n",
"1 520 \n",
"2 674 \n",
"3 556 \n",
"4 711 \n",
"5 649 \n",
"6 53 \n",
"7 901 \n",
"8 709 "
]
},
"metadata": {},
}
],
"source": [
"fillingsdf['gender'] = fillingsdf['gender'].fillna(fillingsdf['gender'].mode()
[0])\n",
"fillingsdf"
]
},
{
"metadata": {},
"outputs": [],
"source": []
},
{
"attachments": {},
"metadata": {},
"source": [
"### Handling outliers\n",
"\n",
" "
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Outliers are those points that are distinct from most of the similar
points"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Dropping the outliers using the standard deviation"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
n",
n",
n",
n",
n",
n",
n",
n",
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n",
"\n",
n",
n",
n",
n",
n",
n",
n",
n",
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n"
]
}
],
"source": [
"data = pd.read_csv('employee.csv')\n",
"\n",
"# Dropping the outliers using the standard deviation \n",
"upper_limit = data['performance_score'].mean() + 3 *
data['performance_score'].std()\n",
"lower_limit = data['performance_score'].mean() - 3 *
data['performance_score'].std()\n",
"\n",
"# Upper limit = mean + 3*std\n",
"# Lower limit = mean - 3*std\n",
"\n",
"data_filtered = data[ (data['performance_score'] < upper_limit) &
(data['performance_score'] > lower_limit) ]\n",
"\n",
"print(data)\n",
"print()\n",
"print(data_filtered)"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Now, Let's remove the outliers using the percentile values\n",
" "
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
n",
n",
n",
n",
n",
n",
n",
n",
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n",
"\n",
n",
n",
n",
n",
n",
n",
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n"
]
}
],
"source": [
"\n",
"# Using the percentile values to filter outliers of the data \n",
"\n",
"upper_limit = data['performance_score'].quantile(0.99)\n",
"lower_limit = data['performance_score'].quantile(0.01)\n",
"\n",
"data_filtered = data[(data['performance_score'] < upper_limit) &
(data['performance_score'] > lower_limit)]\n",
"\n",
"print(data)\n",
"print()\n",
"print(data_filtered)\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"### Feature encoding techniques"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### One-hot encoding\n",
" One-hot encoding transforms the categorical column into labels and splits
the column into multiple columns"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" To perform one-hot encoding, use get_dummies() method"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>F</th>\n",
" <th>M</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score
\\\n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723
\n",
"1 S Kumar NaN 16000.0 F Finance G0 520
\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674
\n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556
\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711
\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649
\n",
"6 James Authur 54.0 NaN F Operations G3 53
\n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901
\n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709
\n",
"\n",
" F M \n",
"0 0 0 \n",
"1 1 0 \n",
"2 0 1 \n",
"3 1 0 \n",
"4 1 0 \n",
"5 0 0 \n",
"6 1 0 \n",
"7 1 0 \n",
"8 0 1 "
]
},
"metadata": {},
}
],
"source": [
"\n",
"encoded_data = pd.get_dummies(data['gender'])\n",
"\n",
"data = data.join(encoded_data)\n",
"\n",
"data"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Previously, we have used geet_dummies() method\n",
" \n",
" Now, Let's use OneHotEncoder from scikit-learn module"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"[[1. 0.]\n",
" [1. 0.]\n",
" [0. 1.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [0. 1.]]\n"
]
}
],
"source": [
"from sklearn.preprocessing import OneHotEncoder\n",
"\n",
"onehotencoder = OneHotEncoder()\n",
"\n",
"data['gender'] = data['gender'].fillna(data['gender'].mode()[0])\n",
"\n",
"print(onehotencoder.fit_transform(data[['gender']]).toarray())\n",
"\n"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Label encoding\n",
" Label encoding is also known as integer encoding. Integer encoding
replaces categorical values with numeric values\n",
" "
]
},
{
"attachments": {},
"metadata": {},
"source": [
" Here, the unique values in variables are replaced with a sequence of
integer values"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"[1 0 0 2 1 2 1 0 2]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>Encoded department</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
\\\n",
\n",
\n",
\n",
\n",
\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649
\n",
"6 James Authur 54.0 NaN F Operations G3 53
\n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901
\n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709
\n",
"\n",
" Encoded department \n",
"0 1 \n",
"1 0 \n",
"2 0 \n",
"3 2 \n",
"4 1 \n",
"5 2 \n",
"6 1 \n",
"7 0 \n",
"8 2 "
]
},
"metadata": {},
}
],
"source": [
"from sklearn.preprocessing import LabelEncoder\n",
"\n",
"\n",
"labelencoder = LabelEncoder()\n",
"\n",
"encoded_data = labelencoder.fit_transform(data['department'])\n",
"\n",
"print(encoded_data)\n",
"\n",
"data['Encoded department'] = encoded_data\n",
"\n",
"data"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" We can also inversely transform categorical values i.e integer values to
categorical values"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"['Finance' 'Sales' 'Operations']\n"
]
}
],
"source": [
"inverse_encoded_data = labelencoder.inverse_transform([0,2,1])\n",
"print(inverse_encoded_data)"
]
},
{
"attachments": {},
"metadata": {},
"source": [
"#### Ordinal encoding\n",
" It except there's an order to the encoding. The output encoding starts
from 0 to N-1"
]
},
{
"attachments": {},
"metadata": {},
"source": [
" We can define the order of the values as a list and pass it to the
category parameter\n",
" The ordinal encoder uses the integer or numeric values to encode where
they ordinal in nature"
]
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>grade_encoded</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
\\\n",
\n",
\n",
\n",
\n",
\n",
"\n",
" grade_encoded \n",
"0 3.0 \n",
"1 0.0 \n",
"2 2.0 \n",
"3 3.0 \n",
"4 2.0 "
]
},
"metadata": {},
}
],
"source": [
"from sklearn.preprocessing import OrdinalEncoder\n",
"\n",
" \n",
"ordinalencoder = OrdinalEncoder(categories=[['G0', 'G1', 'G2', 'G3', 'G4']])\
n",
"data['grade_encoded'] = ordinalencoder.fit_transform(data[['grade']])\n",
"\n",
"data.head()"
]
},
{
"metadata": {},
"outputs": [],
"source": []
},
{
"attachments": {},
"metadata": {},
"source": [
"### Missing values\n"
]
},
{
"metadata": {},
"source": []
},
{
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" }\n",
"\n",
" }\n",
"\n",
" }\n",
"</style>\n",
" <thead>\n",
" <th></th>\n",
" <th>DateTime</th>\n",
" <th>Temperature</th>\n",
" <th>Humidity</th>\n",
" <th>Wind_Speed</th>\n",
" <th>Wind_Direction</th>\n",
" <th>NO2_Location_A</th>\n",
" <th>NO2_Location_B</th>\n",
" <th>NO2_Location_C</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1/1/2020 0:00</td>\n",
" <td>2.180529</td>\n",
" <td>87</td>\n",
" <td>1.484318</td>\n",
" <td>75.963760</td>\n",
" <td>39.23</td>\n",
" <td>52.88</td>\n",
" <td>62.37</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1/1/2020 1:00</td>\n",
" <td>1.490529</td>\n",
" <td>89</td>\n",
" <td>2.741678</td>\n",
" <td>113.198590</td>\n",
" <td>38.30</td>\n",
" <td>44.95</td>\n",
" <td>52.53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1/1/2020 2:00</td>\n",
" <td>1.690529</td>\n",
" <td>85</td>\n",
" <td>3.563818</td>\n",
" <td>135.000000</td>\n",
" <td>NaN</td>\n",
" <td>48.40</td>\n",
" <td>54.80</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1/1/2020 3:00</td>\n",
" <td>1.430529</td>\n",
" <td>84</td>\n",
" <td>2.811690</td>\n",
" <td>129.805570</td>\n",
" <td>37.28</td>\n",
" <td>47.18</td>\n",
" <td>50.80</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1/1/2020 4:00</td>\n",
" <td>0.840529</td>\n",
" <td>86</td>\n",
" <td>1.800000</td>\n",
" <td>126.869896</td>\n",
" <td>29.97</td>\n",
" <td>41.48</td>\n",
" <td>54.43</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8779</th>\n",
" <td>12/31/2020 19:00</td>\n",
" <td>4.920528</td>\n",
" <td>72</td>\n",
" <td>4.553680</td>\n",
" <td>251.565060</td>\n",
" <td>53.44</td>\n",
" <td>32.65</td>\n",
" <td>66.82</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8780</th>\n",
" <td>12/31/2020 20:00</td>\n",
" <td>4.990529</td>\n",
" <td>74</td>\n",
" <td>3.259939</td>\n",
" <td>186.340200</td>\n",
" <td>49.80</td>\n",
" <td>34.03</td>\n",
" <td>67.31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8781</th>\n",
" <td>12/31/2020 21:00</td>\n",
" <td>4.360529</td>\n",
" <td>84</td>\n",
" <td>10.587917</td>\n",
" <td>252.181120</td>\n",
" <td>43.32</td>\n",
" <td>25.55</td>\n",
" <td>59.33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8782</th>\n",
" <td>12/31/2020 22:00</td>\n",
" <td>3.820528</td>\n",
" <td>88</td>\n",
" <td>8.435069</td>\n",
" <td>219.805570</td>\n",
" <td>39.88</td>\n",
" <td>19.90</td>\n",
" <td>43.70</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8783</th>\n",
" <td>12/31/2020 23:00</td>\n",
" <td>3.170529</td>\n",
" <td>89</td>\n",
" <td>6.792466</td>\n",
" <td>212.005390</td>\n",
" <td>39.04</td>\n",
" <td>30.33</td>\n",
" <td>46.87</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" DateTime Temperature Humidity Wind_Speed
Wind_Direction \\\n",
"0 1/1/2020 0:00 2.180529 87 1.484318 75.963760
\n",
"1 1/1/2020 1:00 1.490529 89 2.741678 113.198590
\n",
"2 1/1/2020 2:00 1.690529 85 3.563818 135.000000
\n",
"3 1/1/2020 3:00 1.430529 84 2.811690 129.805570
\n",
"4 1/1/2020 4:00 0.840529 86 1.800000 126.869896
\n",
"... ... ... ... ... ...
\n",
"8779 12/31/2020 19:00 4.920528 72 4.553680 251.565060
\n",
"8780 12/31/2020 20:00 4.990529 74 3.259939 186.340200
\n",
"8781 12/31/2020 21:00 4.360529 84 10.587917 252.181120
\n",
"8782 12/31/2020 22:00 3.820528 88 8.435069 219.805570
\n",
"8783 12/31/2020 23:00 3.170529 89 6.792466 212.005390
\n",
"\n",
" NO2_Location_A NO2_Location_B NO2_Location_C \n",
"0 39.23 52.88 62.37 \n",
"1 38.30 44.95 52.53 \n",
"2 NaN 48.40 54.80 \n",
"3 37.28 47.18 50.80 \n",
"4 29.97 41.48 54.43 \n",
"... ... ... ... \n",
"8779 53.44 32.65 66.82 \n",
"8780 49.80 34.03 67.31 \n",
"8781 43.32 25.55 59.33 \n",
"8782 39.88 19.90 43.70 \n",
"8783 39.04 30.33 46.87 \n",
"\n",
]
},
"metadata": {},
}
],
"source": [
"air_data = pd.read_csv('Airdata.csv')\n",
"air_data"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"RangeIndex: 8784 entries, 0 to 8783\n",
"--- ------ -------------- ----- \n",
" 0 DateTime 8784 non-null object \n",
" 1 Temperature 8784 non-null float64\n",
" 2 Humidity 8784 non-null int64 \n",
" 3 Wind_Speed 8784 non-null float64\n",
" 4 Wind_Direction 8784 non-null float64\n",
" 5 NO2_Location_A 8664 non-null float64\n",
" 6 NO2_Location_B 8204 non-null float64\n",
" 7 NO2_Location_C 8652 non-null float64\n",
"dtypes: float64(6), int64(1), object(1)\n",
"memory usage: 549.1+ KB\n"
]
}
],
"source": [
"air_data.info()"
]
},
{
"metadata": {},
"outputs": [
{
"name": "stdout",
"text": [
"missing values in the given dataset\n",
"--------------------------------\n",
"DateTime: 0\n",
"Temperature: 0\n",
"Humidity: 0\n",
"Wind_Speed: 0\n",
"Wind_Direction: 0\n",
"NO2_Location_A: 120\n",
"NO2_Location_B: 580\n",
"NO2_Location_C: 132\n"
]
}
],
"source": [
"print('missing values in the given dataset\\
n--------------------------------')\n",
"for col in air_data.columns:\n",
" nvm = sum(air_data[col].isna())\n",
" print('{}: {}'.format(col,nvm))\n"
]
},
{
"metadata": {},
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}

Unit-2 Ipynb

Uploaded by

Document Information

Original Description:

Original Title

Copyright

Available Formats

Share this document

Share or Embed Document

Sharing Options

Did you find this document useful?

Is this content inappropriate?

Copyright:

Available Formats

Unit-2 Ipynb

Uploaded by

Copyright:

Available Formats

{

".. ... ... \

".. ... ... ... \

You might also like