{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "411169cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c95cf542",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Staff_Id</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Years_Of_Service</th>\n",
       "      <th>Residence</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "      <th>Resigned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>SA63171</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Depok</td>\n",
       "      <td>4</td>\n",
       "      <td>5582218</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>SP10211</td>\n",
       "      <td>43</td>\n",
       "      <td>3</td>\n",
       "      <td>Jakarta</td>\n",
       "      <td>1</td>\n",
       "      <td>9213443</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>SA79627</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>Bekasi</td>\n",
       "      <td>3</td>\n",
       "      <td>5836455</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>SA02310</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>Depok</td>\n",
       "      <td>4</td>\n",
       "      <td>6035466</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>SA98565</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>Jakarta</td>\n",
       "      <td>1</td>\n",
       "      <td>5568101</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "0                  3                  4                          5   \n",
       "1                  2                  3                          1   \n",
       "2                  3                  3                          2   \n",
       "3                  3                  3                          4   \n",
       "4                  3                  2                          4   \n",
       "\n",
       "   Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "0                   2                  2                           3   \n",
       "1                   2                  4                           3   \n",
       "2                   2                  2                           5   \n",
       "3                   4                  3                           1   \n",
       "4                   3                  3                           2   \n",
       "\n",
       "   Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "0                3                       3                            2   \n",
       "1                4                       3                            2   \n",
       "2                4                       4                            3   \n",
       "3                4                       4                            4   \n",
       "4                3                       4                            2   \n",
       "\n",
       "   Diversity & Inclusion  Work Environment  Compensation Staff_Id  \\\n",
       "0                      3                 4             4  SA63171   \n",
       "1                      2                 2             4  SP10211   \n",
       "2                      2                 4             5  SA79627   \n",
       "3                      5                 3             5  SA02310   \n",
       "4                      3                 2             3  SA98565   \n",
       "\n",
       "   Month_Of_Service  Years_Of_Service Residence  Residence_Code  Net_Salary  \\\n",
       "0                 1                 0     Depok               4     5582218   \n",
       "1                43                 3   Jakarta               1     9213443   \n",
       "2                10                 0    Bekasi               3     5836455   \n",
       "3                17                 1     Depok               4     6035466   \n",
       "4                17                 1   Jakarta               1     5568101   \n",
       "\n",
       "   Resigned  \n",
       "0         0  \n",
       "1         0  \n",
       "2         0  \n",
       "3         0  \n",
       "4         0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grab Data\n",
    "df = pd.read_excel(\"https://raw.githubusercontent.com/youronlydimwit/Data_ScienceUse_Cases/main/Classification/Data/HRD_Survey_50.xlsx\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c5d95f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Role Satisfaction               int64\n",
       "Skill Utilization               int64\n",
       "Career Growth Opportunity       int64\n",
       "Supervisor Support              int64\n",
       "Work-Life Balance               int64\n",
       "Recognition & Appreciation      int64\n",
       "Company Culture                 int64\n",
       "Training & Development          int64\n",
       "Communication Effectiveness     int64\n",
       "Diversity & Inclusion           int64\n",
       "Work Environment                int64\n",
       "Compensation                    int64\n",
       "Staff_Id                       object\n",
       "Month_Of_Service                int64\n",
       "Years_Of_Service                int64\n",
       "Residence                      object\n",
       "Residence_Code                  int64\n",
       "Net_Salary                      int64\n",
       "Resigned                        int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check data types\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c386bd3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Years_Of_Service</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "      <th>Resigned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5582218</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>43</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9213443</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5836455</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6035466</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5568101</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "0                  3                  4                          5   \n",
       "1                  2                  3                          1   \n",
       "2                  3                  3                          2   \n",
       "3                  3                  3                          4   \n",
       "4                  3                  2                          4   \n",
       "\n",
       "   Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "0                   2                  2                           3   \n",
       "1                   2                  4                           3   \n",
       "2                   2                  2                           5   \n",
       "3                   4                  3                           1   \n",
       "4                   3                  3                           2   \n",
       "\n",
       "   Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "0                3                       3                            2   \n",
       "1                4                       3                            2   \n",
       "2                4                       4                            3   \n",
       "3                4                       4                            4   \n",
       "4                3                       4                            2   \n",
       "\n",
       "   Diversity & Inclusion  Work Environment  Compensation  Month_Of_Service  \\\n",
       "0                      3                 4             4                 1   \n",
       "1                      2                 2             4                43   \n",
       "2                      2                 4             5                10   \n",
       "3                      5                 3             5                17   \n",
       "4                      3                 2             3                17   \n",
       "\n",
       "   Years_Of_Service  Residence_Code  Net_Salary  Resigned  \n",
       "0                 0               4     5582218         0  \n",
       "1                 3               1     9213443         0  \n",
       "2                 0               3     5836455         0  \n",
       "3                 1               4     6035466         0  \n",
       "4                 1               1     5568101         0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Making a copy of df, but with only numerical information\n",
    "# Removing unnecessary columns\n",
    "pred_df = df.drop(columns=['Staff_Id','Residence'])\n",
    "pred_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2489f593",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Years_Of_Service</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "      <th>Resigned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.00000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.00000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>5.000000e+02</td>\n",
       "      <td>500.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.006000</td>\n",
       "      <td>3.038000</td>\n",
       "      <td>3.004000</td>\n",
       "      <td>2.948000</td>\n",
       "      <td>2.950000</td>\n",
       "      <td>3.01600</td>\n",
       "      <td>2.946000</td>\n",
       "      <td>3.016000</td>\n",
       "      <td>3.024000</td>\n",
       "      <td>3.014000</td>\n",
       "      <td>2.990000</td>\n",
       "      <td>4.022000</td>\n",
       "      <td>34.510000</td>\n",
       "      <td>2.30400</td>\n",
       "      <td>3.018000</td>\n",
       "      <td>7.908836e+06</td>\n",
       "      <td>0.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.989911</td>\n",
       "      <td>1.046301</td>\n",
       "      <td>1.020817</td>\n",
       "      <td>0.991596</td>\n",
       "      <td>0.980522</td>\n",
       "      <td>1.00487</td>\n",
       "      <td>1.000543</td>\n",
       "      <td>1.042073</td>\n",
       "      <td>1.016608</td>\n",
       "      <td>1.048811</td>\n",
       "      <td>0.977656</td>\n",
       "      <td>1.021569</td>\n",
       "      <td>19.267391</td>\n",
       "      <td>1.43935</td>\n",
       "      <td>1.444246</td>\n",
       "      <td>4.383144e+06</td>\n",
       "      <td>0.3003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.070036e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>18.750000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.688296e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.341903e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.342422e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.443630e+07</td>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "count         500.000000         500.000000                 500.000000   \n",
       "mean            3.006000           3.038000                   3.004000   \n",
       "std             0.989911           1.046301                   1.020817   \n",
       "min             1.000000           1.000000                   1.000000   \n",
       "25%             2.000000           2.000000                   2.000000   \n",
       "50%             3.000000           3.000000                   3.000000   \n",
       "75%             4.000000           4.000000                   4.000000   \n",
       "max             5.000000           5.000000                   5.000000   \n",
       "\n",
       "       Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "count          500.000000         500.000000                   500.00000   \n",
       "mean             2.948000           2.950000                     3.01600   \n",
       "std              0.991596           0.980522                     1.00487   \n",
       "min              1.000000           1.000000                     1.00000   \n",
       "25%              2.000000           2.000000                     2.00000   \n",
       "50%              3.000000           3.000000                     3.00000   \n",
       "75%              4.000000           4.000000                     4.00000   \n",
       "max              5.000000           5.000000                     5.00000   \n",
       "\n",
       "       Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "count       500.000000              500.000000                   500.000000   \n",
       "mean          2.946000                3.016000                     3.024000   \n",
       "std           1.000543                1.042073                     1.016608   \n",
       "min           1.000000                1.000000                     1.000000   \n",
       "25%           2.000000                2.000000                     2.000000   \n",
       "50%           3.000000                3.000000                     3.000000   \n",
       "75%           4.000000                4.000000                     4.000000   \n",
       "max           5.000000                5.000000                     5.000000   \n",
       "\n",
       "       Diversity & Inclusion  Work Environment  Compensation  \\\n",
       "count             500.000000        500.000000    500.000000   \n",
       "mean                3.014000          2.990000      4.022000   \n",
       "std                 1.048811          0.977656      1.021569   \n",
       "min                 1.000000          1.000000      2.000000   \n",
       "25%                 2.000000          2.000000      3.000000   \n",
       "50%                 3.000000          3.000000      4.000000   \n",
       "75%                 4.000000          4.000000      5.000000   \n",
       "max                 5.000000          5.000000      6.000000   \n",
       "\n",
       "       Month_Of_Service  Years_Of_Service  Residence_Code    Net_Salary  \\\n",
       "count        500.000000         500.00000      500.000000  5.000000e+02   \n",
       "mean          34.510000           2.30400        3.018000  7.908836e+06   \n",
       "std           19.267391           1.43935        1.444246  4.383144e+06   \n",
       "min            0.000000           0.00000        1.000000  5.070036e+06   \n",
       "25%           18.750000           1.00000        2.000000  5.688296e+06   \n",
       "50%           35.000000           2.00000        3.000000  6.341903e+06   \n",
       "75%           50.000000           4.00000        4.000000  8.342422e+06   \n",
       "max           66.000000           4.00000        5.000000  5.443630e+07   \n",
       "\n",
       "       Resigned  \n",
       "count  500.0000  \n",
       "mean     0.1000  \n",
       "std      0.3003  \n",
       "min      0.0000  \n",
       "25%      0.0000  \n",
       "50%      0.0000  \n",
       "75%      0.0000  \n",
       "max      1.0000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Describe the new df\n",
    "pred_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee3b8be1",
   "metadata": {},
   "source": [
    "## First Takeaway\n",
    "- min 1 max 5\n",
    "- mean stayed close to 3 indicating normality in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fccbbca9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 71304 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 18 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Making BOXPLOTS to see if there's outlier\n",
    "# Create a figure and axes for subplots\n",
    "fig, axes = plt.subplots(nrows=3, ncols=len(pred_df.columns) // 3 + (len(pred_df.columns) % 3 > 0), figsize=(15, 10))\n",
    "\n",
    "# Flatten the axes array to easily iterate over\n",
    "axes = axes.flatten()\n",
    "\n",
    "# Loop through each column and create a boxplot\n",
    "for i, col in enumerate(pred_df.columns):\n",
    "    pred_df.boxplot(column=col, ax=axes[i])\n",
    "    axes[i].set_title(col)\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "45fb2a76",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from scipy.stats import norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "178492fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 121456 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 12 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Making HISTOGRAM and comparing with normal distirbution's bell curve\n",
    "# Create a figure and axes for subplots\n",
    "fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(15, 10))\n",
    "\n",
    "# Flatten the axes array to easily iterate over\n",
    "axes = axes.flatten()\n",
    "\n",
    "# Loop through each column and create a histogram\n",
    "for i, col in enumerate(pred_df.columns[:12]):\n",
    "    # Plot histogram\n",
    "    pred_df[col].hist(ax=axes[i], bins=[1, 2, 3, 4, 5, 6], alpha=0.5, edgecolor='black', density=True)  # Adjust the number of bins as needed\n",
    "    axes[i].set_title(col)\n",
    "    axes[i].set_xlabel('Value')\n",
    "    axes[i].set_ylabel('Density')\n",
    "\n",
    "    # Add normal distribution curve\n",
    "    mu, std = norm.fit(pred_df[col])\n",
    "    xmin, xmax = axes[i].get_xlim()\n",
    "    x = np.linspace(xmin, xmax, 100)\n",
    "    p = norm.pdf(x, mu, std)\n",
    "    axes[i].plot(x, p, 'k', linewidth=2, alpha=0.3)\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6496f677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 60784 characters]\n",
      "text/plain": [
       "<Figure size 1080x360 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Making HISTOGRAM and comparing with normal distirbution's bell curve,\n",
    "# But this time for the remaining columns\n",
    "# Create a figure and axes for subplots\n",
    "fig, axes = plt.subplots(nrows=1, ncols=5, figsize=(15, 5))\n",
    "\n",
    "# Loop through each column and create a histogram\n",
    "for i, col in enumerate(['Month_Of_Service', 'Years_Of_Service', 'Residence_Code', 'Net_Salary', 'Resigned']):\n",
    "    # Plot histogram\n",
    "    pred_df[col].hist(ax=axes[i], bins=5, alpha=0.5, edgecolor='black', density=True)  # Adjust the number of bins as needed\n",
    "    axes[i].set_title(col)\n",
    "    axes[i].set_xlabel('Value')\n",
    "    axes[i].set_ylabel('Density')\n",
    "\n",
    "    # Add normal distribution curve\n",
    "    mu, std = norm.fit(pred_df[col])\n",
    "    xmin, xmax = axes[i].get_xlim()\n",
    "    x = np.linspace(xmin, xmax, 100)\n",
    "    p = norm.pdf(x, mu, std)\n",
    "    axes[i].plot(x, p, 'k', linewidth=2, alpha=0.5)  # Adjust opacity here\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6d937b21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 185364 characters]\n",
      "text/plain": [
       "<Figure size 864x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix = pred_df.corr()\n",
    "\n",
    "# Create the heatmap\n",
    "plt.figure(figsize=(12, 10))\n",
    "sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\n",
    "plt.title('Correlation Heatmap of All Columns')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db3aa707",
   "metadata": {},
   "source": [
    "## Second takeaway\n",
    "- drop years_of_service since it has autocorrelation\n",
    "- This is an imbalanced dataset, where 1 is way smaller than 0. So needs methods to predict like SMOTE or undersampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "134bbf58",
   "metadata": {},
   "outputs": [],
   "source": [
    "Bar graph resigned column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ba06ed3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Role Satisfaction              int64\n",
       "Skill Utilization              int64\n",
       "Career Growth Opportunity      int64\n",
       "Supervisor Support             int64\n",
       "Work-Life Balance              int64\n",
       "Recognition & Appreciation     int64\n",
       "Company Culture                int64\n",
       "Training & Development         int64\n",
       "Communication Effectiveness    int64\n",
       "Diversity & Inclusion          int64\n",
       "Work Environment               int64\n",
       "Compensation                   int64\n",
       "Month_Of_Service               int64\n",
       "Residence_Code                 int64\n",
       "Net_Salary                     int64\n",
       "Resigned                       int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df = pred_df.drop(columns=[\"Years_Of_Service\"])\n",
    "pred_df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ffccfd1",
   "metadata": {},
   "source": [
    "# Training the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "88eead04",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "90beb683",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7d6b6698",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into features (X) and labels (y)\n",
    "X = pred_df.drop(columns=['Resigned'])\n",
    "y = pred_df['Resigned']\n",
    "\n",
    "# Split the data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Define a dictionary to store results\n",
    "results = {'Model': [], 'F1_score': [], 'Accuracy': [], 'Precision': [], 'Recall': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "17ff7433",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# Train Random Forest model\n",
    "rf_model = RandomForestClassifier(random_state=42)\n",
    "rf_model.fit(X_train, y_train)\n",
    "\n",
    "# Train Decision Tree model\n",
    "dt_model = DecisionTreeClassifier(random_state=42)\n",
    "dt_model.fit(X_train, y_train)\n",
    "\n",
    "# Train Logistic Regression model\n",
    "lr_model = LogisticRegression(random_state=42)\n",
    "lr_model.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate models\n",
    "models = {\"Random Forest\": rf_model, \"Decision Tree\": dt_model, \"Logistic Regression\": lr_model}\n",
    "metrics = {\"Accuracy\": accuracy_score, \"Precision\": precision_score, \"Recall\": recall_score, \"F1 Score\": f1_score}\n",
    "results = {}\n",
    "\n",
    "for name, model in models.items():\n",
    "    y_pred = model.predict(X_test)\n",
    "    result = {}\n",
    "    for metric_name, metric_func in metrics.items():\n",
    "        result[metric_name] = metric_func(y_test, y_pred)\n",
    "    results[name] = result\n",
    "\n",
    "# Convert results to DataFrame for easier plotting\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "36bd8c97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Logistic Regression</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.7</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Random Forest  Decision Tree  Logistic Regression\n",
       "Accuracy             0.7       0.680000                  0.7\n",
       "Precision            0.0       0.444444                  0.0\n",
       "Recall               0.0       0.266667                  0.0\n",
       "F1 Score             0.0       0.333333                  0.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Plot training results\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ce6e7f8",
   "metadata": {},
   "source": [
    "# Third Takeaway\n",
    "- score interpretations\n",
    "- whys and could be's"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "dffda208",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38132 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Assuming dt_model is your trained Decision Tree model\n",
    "# Train Decision Tree model\n",
    "dt_model = DecisionTreeClassifier(random_state=42)\n",
    "dt_model.fit(X_train, y_train)\n",
    "\n",
    "# Get feature importances\n",
    "feature_importances = dt_model.feature_importances_\n",
    "\n",
    "# Create a DataFrame to store feature names and their importance scores\n",
    "feature_importance_df = pd.DataFrame({'Feature': X_train.columns, 'Importance': feature_importances})\n",
    "\n",
    "# Sort the DataFrame by importance scores in descending order\n",
    "feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n",
    "\n",
    "# Plot feature importance with decimal value labels\n",
    "plt.figure(figsize=(10, 6))\n",
    "bars = plt.barh(feature_importance_df['Feature'], feature_importance_df['Importance'], color='lightblue')\n",
    "\n",
    "plt.xlabel('Importance Score')\n",
    "plt.ylabel('Feature')\n",
    "plt.title('Decision Tree Feature Importance')\n",
    "plt.gca().invert_yaxis()  # Invert y-axis to display the most important features at the top\n",
    "\n",
    "# Add decimal value labels to bars\n",
    "for bar, importance in zip(bars, feature_importance_df['Importance']):\n",
    "    plt.text(importance, bar.get_y() + bar.get_height()/2, f'{importance:.2f}', ha='left', va='center')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "2cd6c72b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31608 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Assuming lr_model is your trained Logistic Regression model\n",
    "# Train Logistic Regression model\n",
    "lr_model = LogisticRegression(random_state=42)\n",
    "lr_model.fit(X_train, y_train)\n",
    "\n",
    "# Get feature coefficients (weights)\n",
    "feature_coefficients = lr_model.coef_[0]\n",
    "\n",
    "# Create a DataFrame to store feature names and their coefficients\n",
    "feature_importance_df = pd.DataFrame({'Feature': X_train.columns, 'Coefficient': feature_coefficients})\n",
    "\n",
    "# Sort the DataFrame by coefficient magnitudes in descending order\n",
    "feature_importance_df['Abs_Coefficient'] = feature_importance_df['Coefficient'].abs()  # Absolute value for sorting\n",
    "feature_importance_df = feature_importance_df.sort_values(by='Abs_Coefficient', ascending=False)\n",
    "\n",
    "# Plot feature importance\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.barh(feature_importance_df['Feature'], feature_importance_df['Abs_Coefficient'], color='lightcoral')\n",
    "plt.xlabel('Absolute Coefficient')\n",
    "plt.ylabel('Feature')\n",
    "plt.title('Logistic Regression Feature Importance (Absolute Coefficient)')\n",
    "plt.gca().invert_yaxis()  # Invert y-axis to display the most important features at the top\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "2e96e4ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 39088 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Assuming rf_model is your trained Random Forest model\n",
    "# Train Random Forest model\n",
    "rf_model = RandomForestClassifier(random_state=42)\n",
    "rf_model.fit(X_train, y_train)\n",
    "\n",
    "# Get feature importances\n",
    "feature_importances = rf_model.feature_importances_\n",
    "\n",
    "# Create a DataFrame to store feature names and their importance scores\n",
    "feature_importance_df = pd.DataFrame({'Feature': X_train.columns, 'Importance': feature_importances})\n",
    "\n",
    "# Sort the DataFrame by importance scores in descending order\n",
    "feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n",
    "\n",
    "# Plot feature importance with decimal value labels\n",
    "plt.figure(figsize=(10, 6))\n",
    "bars = plt.barh(feature_importance_df['Feature'], feature_importance_df['Importance'], color='lightgreen')\n",
    "\n",
    "plt.xlabel('Importance Score')\n",
    "plt.ylabel('Feature')\n",
    "plt.title('Random Forest Feature Importance')\n",
    "plt.gca().invert_yaxis()  # Invert y-axis to display the most important features at the top\n",
    "\n",
    "# Add decimal value labels to bars\n",
    "for bar, importance in zip(bars, feature_importance_df['Importance']):\n",
    "    plt.text(importance, bar.get_y() + bar.get_height()/2, f'{importance:.2f}', ha='left', va='center')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d0c810c",
   "metadata": {},
   "outputs": [],
   "source": [
    "========="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9eefd756",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8bb7eec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.utils import class_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e676bc53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.13\n",
      "Precision: 0.13\n",
      "Recall: 1.0\n",
      "F1 Score: 0.23008849557522126\n"
     ]
    }
   ],
   "source": [
    "# Initialize RandomUnderSampler\n",
    "rus = RandomUnderSampler(random_state=42)\n",
    "\n",
    "# Undersample the majority class\n",
    "X_train_resampled, y_train_resampled = rus.fit_resample(X_train, y_train)\n",
    "\n",
    "# Train the logistic regression model\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Predict on the test set\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Calculate evaluation metrics\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "precision = precision_score(y_test, y_pred)\n",
    "recall = recall_score(y_test, y_pred)\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "\n",
    "# Store results\n",
    "results['Model'].append('Logistic Regression with Random Under Sampling')\n",
    "results['F1_score'].append(f1)\n",
    "results['Accuracy'].append(accuracy)\n",
    "results['Precision'].append(precision)\n",
    "results['Recall'].append(recall)\n",
    "\n",
    "# Print evaluation metrics\n",
    "print(\"Accuracy:\", accuracy)\n",
    "print(\"Precision:\", precision)\n",
    "print(\"Recall:\", recall)\n",
    "print(\"F1 Score:\", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f2cd285f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.13\n",
      "Precision: 0.13\n",
      "Recall: 1.0\n",
      "F1 Score: 0.23008849557522126\n"
     ]
    }
   ],
   "source": [
    "# Initialize SMOTE\n",
    "smote = SMOTE(random_state=42)\n",
    "\n",
    "# Oversample the minority class\n",
    "X_train_resampled, y_train_resampled = smote.fit_resample(X_train, y_train)\n",
    "\n",
    "# Train the logistic regression model\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Predict on the test set\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Calculate evaluation metrics\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "precision = precision_score(y_test, y_pred)\n",
    "recall = recall_score(y_test, y_pred)\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "\n",
    "# Store results\n",
    "results['Model'].append('Logistic Regression with SMOTE Over Sampling')\n",
    "results['F1_score'].append(f1)\n",
    "results['Accuracy'].append(accuracy)\n",
    "results['Precision'].append(precision)\n",
    "results['Recall'].append(recall)\n",
    "\n",
    "# Print evaluation metrics\n",
    "print(\"Accuracy:\", accuracy)\n",
    "print(\"Precision:\", precision)\n",
    "print(\"Recall:\", recall)\n",
    "print(\"F1 Score:\", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "58868377",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Metrics:\n",
      "Accuracy: 0.87\n",
      "Precision: 0.0\n",
      "Recall: 0.0\n",
      "F1 Score: 0.0\n",
      "\n",
      "Decision Tree Metrics:\n",
      "Accuracy: 0.78\n",
      "Precision: 0.0\n",
      "Recall: 0.0\n",
      "F1 Score: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# Train the Random Forest model with class weights\n",
    "rf_model = RandomForestClassifier(class_weight='balanced')\n",
    "rf_model.fit(X_train, y_train)\n",
    "\n",
    "# Predict on the test set using Random Forest\n",
    "y_pred_rf = rf_model.predict(X_test)\n",
    "\n",
    "# Calculate evaluation metrics for Random Forest\n",
    "accuracy_rf = accuracy_score(y_test, y_pred_rf)\n",
    "precision_rf = precision_score(y_test, y_pred_rf)\n",
    "recall_rf = recall_score(y_test, y_pred_rf)\n",
    "f1_rf = f1_score(y_test, y_pred_rf)\n",
    "\n",
    "# Store results for Random Forest\n",
    "results['Model'].append('Random Forest with Class Weights')\n",
    "results['F1_score'].append(f1_rf)\n",
    "results['Accuracy'].append(accuracy_rf)\n",
    "results['Precision'].append(precision_rf)\n",
    "results['Recall'].append(recall_rf)\n",
    "\n",
    "# Print evaluation metrics for Random Forest\n",
    "print(\"Random Forest Metrics:\")\n",
    "print(\"Accuracy:\", accuracy_rf)\n",
    "print(\"Precision:\", precision_rf)\n",
    "print(\"Recall:\", recall_rf)\n",
    "print(\"F1 Score:\", f1_rf)\n",
    "\n",
    "# Train the Decision Tree model with class weights\n",
    "dt_model = DecisionTreeClassifier(class_weight='balanced')\n",
    "dt_model.fit(X_train, y_train)\n",
    "\n",
    "# Predict on the test set using Decision Tree\n",
    "y_pred_dt = dt_model.predict(X_test)\n",
    "\n",
    "# Calculate evaluation metrics for Decision Tree\n",
    "accuracy_dt = accuracy_score(y_test, y_pred_dt)\n",
    "precision_dt = precision_score(y_test, y_pred_dt)\n",
    "recall_dt = recall_score(y_test, y_pred_dt)\n",
    "f1_dt = f1_score(y_test, y_pred_dt)\n",
    "\n",
    "# Store results for Decision Tree\n",
    "results['Model'].append('Decision Tree with Class Weights')\n",
    "results['F1_score'].append(f1_dt)\n",
    "results['Accuracy'].append(accuracy_dt)\n",
    "results['Precision'].append(precision_dt)\n",
    "results['Recall'].append(recall_dt)\n",
    "\n",
    "# Print evaluation metrics for Decision Tree\n",
    "print(\"\\nDecision Tree Metrics:\")\n",
    "print(\"Accuracy:\", accuracy_dt)\n",
    "print(\"Precision:\", precision_dt)\n",
    "print(\"Recall:\", recall_dt)\n",
    "print(\"F1 Score:\", f1_dt)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c9f8ee78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes Accuracy: 0.85\n",
      "Naive Bayes Precision: 0.0\n",
      "Naive Bayes Recall: 0.0\n",
      "Naive Bayes F1 Score: 0.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "# Train the Gaussian Naive Bayes model\n",
    "nb_model = GaussianNB()\n",
    "nb_model.fit(X_train, y_train)\n",
    "\n",
    "# Predict on the test set\n",
    "nb_y_pred = nb_model.predict(X_test)\n",
    "\n",
    "# Calculate evaluation metrics for Naive Bayes\n",
    "nb_accuracy = accuracy_score(y_test, nb_y_pred)\n",
    "nb_precision = precision_score(y_test, nb_y_pred)\n",
    "nb_recall = recall_score(y_test, nb_y_pred)\n",
    "nb_f1 = f1_score(y_test, nb_y_pred)\n",
    "\n",
    "# Store results for Naive Bayes\n",
    "results['Model'].append('Gaussian Naive Bayes')\n",
    "results['F1_score'].append(nb_f1)\n",
    "results['Accuracy'].append(nb_accuracy)\n",
    "results['Precision'].append(nb_precision)\n",
    "results['Recall'].append(nb_recall)\n",
    "\n",
    "# Print evaluation metrics for Naive Bayes\n",
    "print(\"Naive Bayes Accuracy:\", nb_accuracy)\n",
    "print(\"Naive Bayes Precision:\", nb_precision)\n",
    "print(\"Naive Bayes Recall:\", nb_recall)\n",
    "print(\"Naive Bayes F1 Score:\", nb_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ce9ad2f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 11580 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 10680 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot class distribution before undersampling\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.countplot(x=y_train, palette='viridis')\n",
    "plt.title('Class Distribution Before Random Under Sampling')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Count')\n",
    "plt.show()\n",
    "\n",
    "# Initialize RandomUnderSampler\n",
    "rus = RandomUnderSampler(random_state=42)\n",
    "\n",
    "# Undersample the majority class\n",
    "X_train_resampled, y_train_resampled = rus.fit_resample(X_train, y_train)\n",
    "\n",
    "# Plot class distribution after undersampling\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.countplot(x=y_train_resampled, palette='viridis')\n",
    "plt.title('Class Distribution After Random Under Sampling')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Count')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f62a7ca",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
