{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b1913dbb",
   "metadata": {},
   "source": [
    "# Table of Contents <a id=\"top\"></a>\n",
    "- [Reading The Data](#readData)\n",
    "- [First Takeaway](#firstTake)\n",
    "- [Second Takeaway](#secondTake)\n",
    "- [Training The Data](#trainData)\n",
    " - [Third Takeaway](#thirdTake)\n",
    " - [Class Weights](#weightCtrl)\n",
    " - [Oversampling](#trainOver)\n",
    " - [Undersampling](#trainUnder)\n",
    "- [Feature Importance](#dissect)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71195f0a",
   "metadata": {},
   "source": [
    "## Reading The Data <a id=\"readData\"></a>\n",
    "For starters, lets read the data and its properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "411169cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c95cf542",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Staff_Id</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Years_Of_Service</th>\n",
       "      <th>Residence</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "      <th>Resigned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>SA63171</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Depok</td>\n",
       "      <td>4</td>\n",
       "      <td>5582218</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>SP10211</td>\n",
       "      <td>43</td>\n",
       "      <td>3</td>\n",
       "      <td>Jakarta</td>\n",
       "      <td>1</td>\n",
       "      <td>9213443</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>SA79627</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>Bekasi</td>\n",
       "      <td>3</td>\n",
       "      <td>5836455</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>SA02310</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>Depok</td>\n",
       "      <td>4</td>\n",
       "      <td>6035466</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>SA98565</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>Jakarta</td>\n",
       "      <td>1</td>\n",
       "      <td>5568101</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "0                  3                  4                          5   \n",
       "1                  2                  3                          1   \n",
       "2                  3                  3                          2   \n",
       "3                  3                  3                          4   \n",
       "4                  3                  2                          4   \n",
       "\n",
       "   Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "0                   2                  2                           3   \n",
       "1                   2                  4                           3   \n",
       "2                   2                  2                           5   \n",
       "3                   4                  3                           1   \n",
       "4                   3                  3                           2   \n",
       "\n",
       "   Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "0                3                       3                            2   \n",
       "1                4                       3                            2   \n",
       "2                4                       4                            3   \n",
       "3                4                       4                            4   \n",
       "4                3                       4                            2   \n",
       "\n",
       "   Diversity & Inclusion  Work Environment  Compensation Staff_Id  \\\n",
       "0                      3                 4             3  SA63171   \n",
       "1                      2                 2             3  SP10211   \n",
       "2                      2                 4             4  SA79627   \n",
       "3                      5                 3             4  SA02310   \n",
       "4                      3                 2             2  SA98565   \n",
       "\n",
       "   Month_Of_Service  Years_Of_Service Residence  Residence_Code  Net_Salary  \\\n",
       "0                 1                 0     Depok               4     5582218   \n",
       "1                43                 3   Jakarta               1     9213443   \n",
       "2                10                 0    Bekasi               3     5836455   \n",
       "3                17                 1     Depok               4     6035466   \n",
       "4                17                 1   Jakarta               1     5568101   \n",
       "\n",
       "   Resigned  \n",
       "0         0  \n",
       "1         0  \n",
       "2         0  \n",
       "3         0  \n",
       "4         0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grab Data\n",
    "df = pd.read_excel(\"https://raw.githubusercontent.com/youronlydimwit/Data_ScienceUse_Cases/main/Classification/Data/HRD_Survey_50.xlsx\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c5d95f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Role Satisfaction               int64\n",
       "Skill Utilization               int64\n",
       "Career Growth Opportunity       int64\n",
       "Supervisor Support              int64\n",
       "Work-Life Balance               int64\n",
       "Recognition & Appreciation      int64\n",
       "Company Culture                 int64\n",
       "Training & Development          int64\n",
       "Communication Effectiveness     int64\n",
       "Diversity & Inclusion           int64\n",
       "Work Environment                int64\n",
       "Compensation                    int64\n",
       "Staff_Id                       object\n",
       "Month_Of_Service                int64\n",
       "Years_Of_Service                int64\n",
       "Residence                      object\n",
       "Residence_Code                  int64\n",
       "Net_Salary                      int64\n",
       "Resigned                        int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check data types\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c386bd3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Years_Of_Service</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "      <th>Resigned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5582218</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>43</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9213443</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5836455</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6035466</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5568101</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "0                  3                  4                          5   \n",
       "1                  2                  3                          1   \n",
       "2                  3                  3                          2   \n",
       "3                  3                  3                          4   \n",
       "4                  3                  2                          4   \n",
       "\n",
       "   Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "0                   2                  2                           3   \n",
       "1                   2                  4                           3   \n",
       "2                   2                  2                           5   \n",
       "3                   4                  3                           1   \n",
       "4                   3                  3                           2   \n",
       "\n",
       "   Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "0                3                       3                            2   \n",
       "1                4                       3                            2   \n",
       "2                4                       4                            3   \n",
       "3                4                       4                            4   \n",
       "4                3                       4                            2   \n",
       "\n",
       "   Diversity & Inclusion  Work Environment  Compensation  Month_Of_Service  \\\n",
       "0                      3                 4             3                 1   \n",
       "1                      2                 2             3                43   \n",
       "2                      2                 4             4                10   \n",
       "3                      5                 3             4                17   \n",
       "4                      3                 2             2                17   \n",
       "\n",
       "   Years_Of_Service  Residence_Code  Net_Salary  Resigned  \n",
       "0                 0               4     5582218         0  \n",
       "1                 3               1     9213443         0  \n",
       "2                 0               3     5836455         0  \n",
       "3                 1               4     6035466         0  \n",
       "4                 1               1     5568101         0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Making a copy of df, but with only numerical information\n",
    "# Removing unnecessary columns\n",
    "pred_df = df.drop(columns=['Staff_Id','Residence'])\n",
    "pred_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2489f593",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Years_Of_Service</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "      <th>Resigned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.00000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>500.00000</td>\n",
       "      <td>500.000000</td>\n",
       "      <td>5.000000e+02</td>\n",
       "      <td>500.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.006000</td>\n",
       "      <td>3.038000</td>\n",
       "      <td>3.004000</td>\n",
       "      <td>2.948000</td>\n",
       "      <td>2.950000</td>\n",
       "      <td>3.01600</td>\n",
       "      <td>2.946000</td>\n",
       "      <td>3.016000</td>\n",
       "      <td>3.024000</td>\n",
       "      <td>3.014000</td>\n",
       "      <td>2.990000</td>\n",
       "      <td>3.022000</td>\n",
       "      <td>34.510000</td>\n",
       "      <td>2.30400</td>\n",
       "      <td>3.018000</td>\n",
       "      <td>7.908836e+06</td>\n",
       "      <td>0.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.989911</td>\n",
       "      <td>1.046301</td>\n",
       "      <td>1.020817</td>\n",
       "      <td>0.991596</td>\n",
       "      <td>0.980522</td>\n",
       "      <td>1.00487</td>\n",
       "      <td>1.000543</td>\n",
       "      <td>1.042073</td>\n",
       "      <td>1.016608</td>\n",
       "      <td>1.048811</td>\n",
       "      <td>0.977656</td>\n",
       "      <td>1.021569</td>\n",
       "      <td>19.267391</td>\n",
       "      <td>1.43935</td>\n",
       "      <td>1.444246</td>\n",
       "      <td>4.383144e+06</td>\n",
       "      <td>0.3003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.070036e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>18.750000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.688296e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.341903e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.342422e+06</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.443630e+07</td>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "count         500.000000         500.000000                 500.000000   \n",
       "mean            3.006000           3.038000                   3.004000   \n",
       "std             0.989911           1.046301                   1.020817   \n",
       "min             1.000000           1.000000                   1.000000   \n",
       "25%             2.000000           2.000000                   2.000000   \n",
       "50%             3.000000           3.000000                   3.000000   \n",
       "75%             4.000000           4.000000                   4.000000   \n",
       "max             5.000000           5.000000                   5.000000   \n",
       "\n",
       "       Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "count          500.000000         500.000000                   500.00000   \n",
       "mean             2.948000           2.950000                     3.01600   \n",
       "std              0.991596           0.980522                     1.00487   \n",
       "min              1.000000           1.000000                     1.00000   \n",
       "25%              2.000000           2.000000                     2.00000   \n",
       "50%              3.000000           3.000000                     3.00000   \n",
       "75%              4.000000           4.000000                     4.00000   \n",
       "max              5.000000           5.000000                     5.00000   \n",
       "\n",
       "       Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "count       500.000000              500.000000                   500.000000   \n",
       "mean          2.946000                3.016000                     3.024000   \n",
       "std           1.000543                1.042073                     1.016608   \n",
       "min           1.000000                1.000000                     1.000000   \n",
       "25%           2.000000                2.000000                     2.000000   \n",
       "50%           3.000000                3.000000                     3.000000   \n",
       "75%           4.000000                4.000000                     4.000000   \n",
       "max           5.000000                5.000000                     5.000000   \n",
       "\n",
       "       Diversity & Inclusion  Work Environment  Compensation  \\\n",
       "count             500.000000        500.000000    500.000000   \n",
       "mean                3.014000          2.990000      3.022000   \n",
       "std                 1.048811          0.977656      1.021569   \n",
       "min                 1.000000          1.000000      1.000000   \n",
       "25%                 2.000000          2.000000      2.000000   \n",
       "50%                 3.000000          3.000000      3.000000   \n",
       "75%                 4.000000          4.000000      4.000000   \n",
       "max                 5.000000          5.000000      5.000000   \n",
       "\n",
       "       Month_Of_Service  Years_Of_Service  Residence_Code    Net_Salary  \\\n",
       "count        500.000000         500.00000      500.000000  5.000000e+02   \n",
       "mean          34.510000           2.30400        3.018000  7.908836e+06   \n",
       "std           19.267391           1.43935        1.444246  4.383144e+06   \n",
       "min            0.000000           0.00000        1.000000  5.070036e+06   \n",
       "25%           18.750000           1.00000        2.000000  5.688296e+06   \n",
       "50%           35.000000           2.00000        3.000000  6.341903e+06   \n",
       "75%           50.000000           4.00000        4.000000  8.342422e+06   \n",
       "max           66.000000           4.00000        5.000000  5.443630e+07   \n",
       "\n",
       "       Resigned  \n",
       "count  500.0000  \n",
       "mean     0.1000  \n",
       "std      0.3003  \n",
       "min      0.0000  \n",
       "25%      0.0000  \n",
       "50%      0.0000  \n",
       "75%      0.0000  \n",
       "max      1.0000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Describe the new df\n",
    "pred_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee3b8be1",
   "metadata": {},
   "source": [
    "## First Takeaway <a id=\"firstTake\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "- Most features have a minimum value of 1 and a maximum value of 5, except for `Month_of_service` to `Resigned` features.\n",
    "- The mean value from `Role Satisfaction` to `Compensation` consistently hovers around 3, suggesting normality in the data distribution, which aligns with the observed minimum and maximum values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fccbbca9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 70112 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 18 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Making BOXPLOTS to see if there's outlier\n",
    "# Create a figure and axes for subplots\n",
    "fig, axes = plt.subplots(nrows=3, ncols=len(pred_df.columns) // 3 + (len(pred_df.columns) % 3 > 0), figsize=(15, 10))\n",
    "\n",
    "# Flatten the axes array to easily iterate over\n",
    "axes = axes.flatten()\n",
    "\n",
    "# Loop through each column and create a boxplot\n",
    "for i, col in enumerate(pred_df.columns):\n",
    "    pred_df.boxplot(column=col, ax=axes[i])\n",
    "    axes[i].set_title(col)\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "45fb2a76",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from scipy.stats import norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "178492fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 121824 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 12 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Making HISTOGRAM and comparing with normal distirbution's bell curve\n",
    "# Create a figure and axes for subplots\n",
    "fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(15, 10))\n",
    "\n",
    "# Flatten the axes array to easily iterate over\n",
    "axes = axes.flatten()\n",
    "\n",
    "# Loop through each column and create a histogram\n",
    "for i, col in enumerate(pred_df.columns[:12]):\n",
    "    # Plot histogram\n",
    "    pred_df[col].hist(ax=axes[i], bins=[1, 2, 3, 4, 5, 6], alpha=0.5, edgecolor='black', density=True)  # Adjust the number of bins as needed\n",
    "    axes[i].set_title(col)\n",
    "    axes[i].set_xlabel('Value')\n",
    "    axes[i].set_ylabel('Frequency')\n",
    "\n",
    "    # Add normal distribution curve\n",
    "    mu, std = norm.fit(pred_df[col])\n",
    "    xmin, xmax = axes[i].get_xlim()\n",
    "    x = np.linspace(xmin, xmax, 100)\n",
    "    p = norm.pdf(x, mu, std)\n",
    "    axes[i].plot(x, p, 'k', linewidth=2, alpha=0.3)\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6496f677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 61308 characters]\n",
      "text/plain": [
       "<Figure size 1080x360 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Making HISTOGRAM and comparing with normal distirbution's bell curve,\n",
    "# But this time for the remaining columns\n",
    "# Create a figure and axes for subplots\n",
    "fig, axes = plt.subplots(nrows=1, ncols=5, figsize=(15, 5))\n",
    "\n",
    "# Loop through each column and create a histogram\n",
    "for i, col in enumerate(['Month_Of_Service', 'Years_Of_Service', 'Residence_Code', 'Net_Salary', 'Resigned']):\n",
    "    # Plot histogram\n",
    "    pred_df[col].hist(ax=axes[i], bins=5, alpha=0.5, edgecolor='black', density=True)  # Adjust the number of bins as needed\n",
    "    axes[i].set_title(col)\n",
    "    axes[i].set_xlabel('Value')\n",
    "    axes[i].set_ylabel('Frequency')\n",
    "\n",
    "    # Add normal distribution curve\n",
    "    mu, std = norm.fit(pred_df[col])\n",
    "    xmin, xmax = axes[i].get_xlim()\n",
    "    x = np.linspace(xmin, xmax, 100)\n",
    "    p = norm.pdf(x, mu, std)\n",
    "    axes[i].plot(x, p, 'k', linewidth=2, alpha=0.5)  # Adjust opacity here\n",
    "\n",
    "# Adjust layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6d937b21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 184320 characters]\n",
      "text/plain": [
       "<Figure size 864x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix = pred_df.corr()\n",
    "\n",
    "# Create the heatmap\n",
    "plt.figure(figsize=(12, 10))\n",
    "sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\n",
    "plt.title('Correlation Heatmap of All Columns')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db3aa707",
   "metadata": {},
   "source": [
    "## Second takeaway <a id=\"secondTake\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "- Identified autocorrelation between the columns `Years_Of_Service` and `Month_Of_Service`. It's necessary to remove one of them. In this scenario, `Years_Of_Service` is chosen for removal.\n",
    "- After analyzing the histogram of the `Resigned` column, it's evident that the dataset is imbalanced, with a significantly smaller number of instances labeled as 1 compared to 0. This requires a specialized approach to train the data effectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "134bbf58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 7844 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Confirm the Resigned Column's Labels\n",
    "plt.figure(figsize=(8, 4))\n",
    "sns.countplot(x='Resigned', data=pred_df)\n",
    "plt.title('Resigned Column Countplot')\n",
    "plt.xlabel('Resigned')\n",
    "plt.ylabel('Count')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dfd0a80",
   "metadata": {},
   "source": [
    "An **\"Imbalanced\"** Dataset, where in a feature, the disparity between minority and majority class is very evident."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ba06ed3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Role Satisfaction              int64\n",
       "Skill Utilization              int64\n",
       "Career Growth Opportunity      int64\n",
       "Supervisor Support             int64\n",
       "Work-Life Balance              int64\n",
       "Recognition & Appreciation     int64\n",
       "Company Culture                int64\n",
       "Training & Development         int64\n",
       "Communication Effectiveness    int64\n",
       "Diversity & Inclusion          int64\n",
       "Work Environment               int64\n",
       "Compensation                   int64\n",
       "Month_Of_Service               int64\n",
       "Residence_Code                 int64\n",
       "Net_Salary                     int64\n",
       "Resigned                       int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the column Years_Of_Service\n",
    "pred_df = pred_df.drop(columns=[\"Years_Of_Service\"])\n",
    "pred_df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ffccfd1",
   "metadata": {},
   "source": [
    "## Training the data <a id=\"trainData\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "Now, we will attempt to train the dataset. Firstly, with normal settings / no special technique."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "88eead04",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "90beb683",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7d6b6698",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into features (X) and labels (y)\n",
    "X = pred_df.drop(columns=['Resigned'])\n",
    "y = pred_df['Resigned']\n",
    "\n",
    "# Split the data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Define a dictionary to store results\n",
    "results = {'Model': [], 'F1_score': [], 'Accuracy': [], 'Precision': [], 'Recall': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "17ff7433",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Naive Bayes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.87</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Random Forest  Decision Tree  Naive Bayes\n",
       "Accuracy            0.87       0.820000         0.85\n",
       "Precision           0.00       0.272727         0.00\n",
       "Recall              0.00       0.230769         0.00\n",
       "F1 Score            0.00       0.250000         0.00"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train Random Forest model\n",
    "rf_model = RandomForestClassifier(random_state=42)\n",
    "rf_model.fit(X_train, y_train)\n",
    "\n",
    "# Train Decision Tree model\n",
    "dt_model = DecisionTreeClassifier(random_state=42)\n",
    "dt_model.fit(X_train, y_train)\n",
    "\n",
    "# Train Naive Bayes model\n",
    "nb_model = GaussianNB()\n",
    "nb_model.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate models\n",
    "models = {\"Random Forest\": rf_model, \"Decision Tree\": dt_model, \"Naive Bayes\": nb_model}\n",
    "metrics = {\"Accuracy\": accuracy_score, \"Precision\": precision_score, \"Recall\": recall_score, \"F1 Score\": f1_score}\n",
    "results = {}\n",
    "\n",
    "for name, model in models.items():\n",
    "    y_pred = model.predict(X_test)\n",
    "    result = {}\n",
    "    for metric_name, metric_func in metrics.items():\n",
    "        result[metric_name] = metric_func(y_test, y_pred)\n",
    "    results[name] = result\n",
    "\n",
    "# Convert results to DataFrame for easier plotting\n",
    "results_df = pd.DataFrame(results)\n",
    "\n",
    "# print results\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "745713e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10552 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate class frequencies for y_train and y_test\n",
    "unique_train, counts_train = np.unique(y_train, return_counts=True)\n",
    "unique_test, counts_test = np.unique(y_test, return_counts=True)\n",
    "\n",
    "# Plot y_train and y_test\n",
    "plt.figure(figsize=(8, 4))\n",
    "\n",
    "# Plot for y_train\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.bar(unique_train, counts_train, color='red')\n",
    "plt.title('Distribution of y_train')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xticks(unique_train)\n",
    "plt.ylim(0, max(max(counts_train), max(counts_train)) + 10)\n",
    "\n",
    "# Plot for y_train\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.bar(unique_test, counts_test, color='blue')\n",
    "plt.title('Distribution of y_test')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xticks(unique_test)\n",
    "plt.ylim(0, max(max(counts_test), max(counts_test)) + 10)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ce6e7f8",
   "metadata": {},
   "source": [
    "## Third Takeaway <a id=\"thirdTake\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "- Despite achieving relatively high accuracy, the Random Forest and Naive Bayes models exhibit a precision of 0, suggesting poor performance specifically on the minority class (Resigned = 1).\n",
    "- To enhance model performance, we will explore additional data transformation techniques. These will include strategies such as adjusting class weights and implementing over/undersampling methods."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e11c086",
   "metadata": {},
   "source": [
    "## Class Weights <a id=\"weightCtrl\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "We will start by adjusting the class weights of the `Resigned` Feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "635dee3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5d8c44c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class Weights: {0: 0.5555555555555556, 1: 5.0}\n"
     ]
    }
   ],
   "source": [
    "# Firstly, we want to apply stratified splitting method, and containing the value on a new variable.\n",
    "# X and y are already instantiated in previous code above.\n",
    "\n",
    "# Run Stratified train-test split\n",
    "X_train_strat, X_test_strat, y_train_strat, y_test_strat = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n",
    "\n",
    "# Then, compute class weights\n",
    "class_labels = np.unique(y_train_strat)\n",
    "class_weights = compute_class_weight(class_weight='balanced', classes=class_labels, y=y_train_strat)\n",
    "\n",
    "# Convert class weights to dictionary format\n",
    "class_weights_dict = dict(zip(class_labels, class_weights))\n",
    "\n",
    "# Print class weights\n",
    "print(\"Class Weights:\", class_weights_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffefd4b5",
   "metadata": {},
   "source": [
    "- Class 0 has a weight of approximately **0.5.** This suggests that misclassifications of class 0 will be penalized less heavily during model training compared to class 1. Since this weight is less than 1, it implies that **class 0 is relatively well-represented in the training data.**\n",
    "- Class 1 has a weight of **5.0.** This indicates that misclassifications of class 1 will be penalized more heavily during model training compared to class 0. Since this weight is greater than 1, it suggests that **class 1 is underrepresented in the training data.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2c8cf18b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Naive Bayes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.173913</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Random Forest  Decision Tree  Naive Bayes\n",
       "Accuracy             0.9       0.810000          0.9\n",
       "Precision            0.0       0.153846          0.0\n",
       "Recall               0.0       0.200000          0.0\n",
       "F1 Score             0.0       0.173913          0.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Force the new class weights\n",
    "# Play around with this value\n",
    "class_weights_new = {0: 1, 1: 1.5}\n",
    "\n",
    "# Train Random Forest model\n",
    "rf_model = RandomForestClassifier(random_state=42)\n",
    "rf_model.fit(X_train_strat, y_train_strat)\n",
    "\n",
    "# Train Decision Tree model\n",
    "dt_model = DecisionTreeClassifier(random_state=42)\n",
    "dt_model.fit(X_train_strat, y_train_strat)\n",
    "\n",
    "# Train Naive Bayes model\n",
    "nb_model = GaussianNB()\n",
    "nb_model.fit(X_train_strat, y_train_strat)\n",
    "\n",
    "# Evaluate models\n",
    "models = {\"Random Forest\": rf_model, \"Decision Tree\": dt_model, \"Naive Bayes\": nb_model}\n",
    "metrics = {\"Accuracy\": accuracy_score, \"Precision\": precision_score, \"Recall\": recall_score, \"F1 Score\": f1_score}\n",
    "results = {}\n",
    "\n",
    "for name, model in models.items():\n",
    "    y_pred = model.predict(X_test_strat)\n",
    "    result = {}\n",
    "    for metric_name, metric_func in metrics.items():\n",
    "        result[metric_name] = metric_func(y_test_strat, y_pred)\n",
    "    results[name] = result\n",
    "\n",
    "# Convert results to DataFrame for easier plotting\n",
    "results_df = pd.DataFrame(results)\n",
    "\n",
    "# print results\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65dbd45b",
   "metadata": {},
   "source": [
    "Just like before, the `Random Forest` and `Naive Bayes` model did not pick up the minority class. We will move on to the next technique."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a56b0c5",
   "metadata": {},
   "source": [
    "# Oversampling <a id=\"trainOver\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "The goal of oversampling is to balance the class distribution by artificially increasing the number of instances in the minority class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a242b66f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1f8756c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate oversampler\n",
    "oversampler = RandomOverSampler(random_state=42)\n",
    "\n",
    "# Apply Random Oversampling to balance the training data\n",
    "X_train_resampled, y_train_resampled = oversampler.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ff610390",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10128 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate class frequencies for y_train_resampled and y_train\n",
    "unique_train_resampled, counts_train_resampled = np.unique(y_train_resampled, return_counts=True)\n",
    "unique_train, counts_train = np.unique(y_train, return_counts=True)\n",
    "\n",
    "# Plot bar plots\n",
    "plt.figure(figsize=(8, 4))\n",
    "\n",
    "# Plot for y_train_resampled\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.bar(unique_train_resampled, counts_train_resampled, color='red')\n",
    "plt.title('Distribution of y_train_resampled')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xticks(unique_train_resampled)\n",
    "plt.ylim(0, max(max(counts_train_resampled), max(counts_train)) + 10)\n",
    "\n",
    "# Plot for y_train\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.bar(unique_train, counts_train, color='blue')\n",
    "plt.title('Distribution of y_train')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xticks(unique_train)\n",
    "plt.ylim(0, max(max(counts_train_resampled), max(counts_train)) + 10)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eac0bdd9",
   "metadata": {},
   "source": [
    "We can see the oversampling algorithm makes synthetic samples to the label 1, to match the frequency of label 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "05f6901e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Asus\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Naive Bayes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.87</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.780000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Random Forest  Decision Tree  Naive Bayes\n",
       "Accuracy            0.87           0.78     0.780000\n",
       "Precision           0.00           0.00     0.090909\n",
       "Recall              0.00           0.00     0.076923\n",
       "F1 Score            0.00           0.00     0.083333"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train Random Forest model\n",
    "rf_model = RandomForestClassifier(random_state=42)\n",
    "rf_model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Train Decision Tree model\n",
    "dt_model = DecisionTreeClassifier(random_state=42)\n",
    "dt_model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Train Naive Bayes model\n",
    "nb_model = GaussianNB()\n",
    "nb_model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Evaluate models\n",
    "models = {\"Random Forest\": rf_model, \"Decision Tree\": dt_model, \"Naive Bayes\": nb_model}\n",
    "metrics = {\"Accuracy\": accuracy_score, \"Precision\": precision_score, \"Recall\": recall_score, \"F1 Score\": f1_score}\n",
    "results = {}\n",
    "\n",
    "for name, model in models.items():\n",
    "    y_pred = model.predict(X_test)\n",
    "    result = {}\n",
    "    for metric_name, metric_func in metrics.items():\n",
    "        result[metric_name] = metric_func(y_test, y_pred)\n",
    "    results[name] = result\n",
    "\n",
    "# Convert results to DataFrame for easier plotting\n",
    "results_df_resampled = pd.DataFrame(results)\n",
    "\n",
    "results_df_resampled"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5833dc09",
   "metadata": {},
   "source": [
    "Using Oversampling, the `Naive Bayes` did pick up the minority class. But, the others are still very poorly performed. We will try the next technique."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bba635d",
   "metadata": {},
   "source": [
    "## Undersampling <a id=\"trainUnder\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "Instead of using the entire dataset, undersampling involves reducing the size of the majority class to balance it with the minority class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b4d95ba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8843cc81",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate undersampler\n",
    "undersampler = RandomUnderSampler(random_state=42)\n",
    "\n",
    "# Apply Random Undersampling to balance the training data\n",
    "X_train_resampled, y_train_resampled = undersampler.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "987956b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10104 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate class frequencies for y_train_resampled and y_train\n",
    "unique_train_resampled, counts_train_resampled = np.unique(y_train_resampled, return_counts=True)\n",
    "unique_train, counts_train = np.unique(y_train, return_counts=True)\n",
    "\n",
    "# Plot bar plots\n",
    "plt.figure(figsize=(8, 4))\n",
    "\n",
    "# Plot for y_train_resampled\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.bar(unique_train_resampled, counts_train_resampled, color='red')\n",
    "plt.title('Distribution of y_train_resampled')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xticks(unique_train_resampled)\n",
    "plt.ylim(0, max(max(counts_train_resampled), max(counts_train)) + 10)\n",
    "\n",
    "# Plot for y_train\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.bar(unique_train, counts_train, color='blue')\n",
    "plt.title('Distribution of y_train')\n",
    "plt.xlabel('Class')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xticks(unique_train)\n",
    "plt.ylim(0, max(max(counts_train_resampled), max(counts_train)) + 10)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c6f27ab",
   "metadata": {},
   "source": [
    "We can see the undersampling algorithm makes \"shrinks\" the data of label 0, to match the frequency of label 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d0fb7411",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Naive Bayes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.560000</td>\n",
       "      <td>0.490000</td>\n",
       "      <td>0.780000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.155556</td>\n",
       "      <td>0.183333</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.241379</td>\n",
       "      <td>0.301370</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Random Forest  Decision Tree  Naive Bayes\n",
       "Accuracy        0.560000       0.490000     0.780000\n",
       "Precision       0.155556       0.183333     0.090909\n",
       "Recall          0.538462       0.846154     0.076923\n",
       "F1 Score        0.241379       0.301370     0.083333"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train Random Forest model\n",
    "rf_model = RandomForestClassifier(random_state=42)\n",
    "rf_model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Train Decision Tree model\n",
    "dt_model = DecisionTreeClassifier(random_state=42)\n",
    "dt_model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Train Naive Bayes model\n",
    "nb_model = GaussianNB()\n",
    "nb_model.fit(X_train_resampled, y_train_resampled)\n",
    "\n",
    "# Evaluate models\n",
    "models = {\"Random Forest\": rf_model, \"Decision Tree\": dt_model, \"Naive Bayes\": nb_model}\n",
    "metrics = {\"Accuracy\": accuracy_score, \"Precision\": precision_score, \"Recall\": recall_score, \"F1 Score\": f1_score}\n",
    "results = {}\n",
    "\n",
    "for name, model in models.items():\n",
    "    y_pred = model.predict(X_test)\n",
    "    result = {}\n",
    "    for metric_name, metric_func in metrics.items():\n",
    "        result[metric_name] = metric_func(y_test, y_pred)\n",
    "    results[name] = result\n",
    "\n",
    "# Convert results to DataFrame for easier plotting\n",
    "results_df_resampled = pd.DataFrame(results)\n",
    "\n",
    "results_df_resampled"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e574b3c",
   "metadata": {},
   "source": [
    "While sacrificing some accuracy, our adjustments have notably improved the recall and F1 Score of the models. This indicates that the model is now capable of effectively capturing instances belonging to both label 1 and label 0, enhancing its overall performance."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc20a26b",
   "metadata": {},
   "source": [
    "## Feature Importance <a id=\"dissect\"></a>\n",
    "[Back to Top](#top)<br>\n",
    "From the results of the undersampling method, it produces the best result so far. And we can attempt to dissect the feature importances from each model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "dffda208",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35916 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get feature importances for Decision Tree model\n",
    "feature_importances = dt_model.feature_importances_\n",
    "\n",
    "# Create a DataFrame to store feature names and their importance scores\n",
    "feature_importance_df = pd.DataFrame({'Feature': X_train_resampled.columns, 'Importance': feature_importances})\n",
    "\n",
    "# Sort the DataFrame by importance scores in descending order\n",
    "feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n",
    "\n",
    "# Plot feature importance with decimal value labels\n",
    "plt.figure(figsize=(8, 4))\n",
    "bars = plt.barh(feature_importance_df['Feature'], feature_importance_df['Importance'], color='lightblue')\n",
    "\n",
    "plt.xlabel('Importance Score')\n",
    "plt.ylabel('Feature')\n",
    "plt.title('Decision Tree Feature Importance')\n",
    "plt.gca().invert_yaxis()  # Invert y-axis to display the most important features at the top\n",
    "\n",
    "# Add decimal value labels to bars\n",
    "for bar, importance in zip(bars, feature_importance_df['Importance']):\n",
    "    plt.text(importance, bar.get_y() + bar.get_height()/2, f'{importance:.2f}', ha='left', va='center')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2cd6c72b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 36540 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get feature importances for Random Forest model\n",
    "feature_importances_rf = rf_model.feature_importances_\n",
    "\n",
    "# Create a DataFrame to store feature names and their importance scores\n",
    "feature_importance_df_rf = pd.DataFrame({'Feature': X_train_resampled.columns, 'Importance': feature_importances_rf})\n",
    "\n",
    "# Sort the DataFrame by importance scores in descending order\n",
    "feature_importance_df_rf = feature_importance_df_rf.sort_values(by='Importance', ascending=False)\n",
    "\n",
    "# Plot feature importance with decimal value labels\n",
    "plt.figure(figsize=(8, 4))\n",
    "bars_rf = plt.barh(feature_importance_df_rf['Feature'], feature_importance_df_rf['Importance'], color='lightgreen')\n",
    "\n",
    "plt.xlabel('Importance Score')\n",
    "plt.ylabel('Feature')\n",
    "plt.title('Random Forest Feature Importance')\n",
    "plt.gca().invert_yaxis()  # Invert y-axis to display the most important features at the top\n",
    "\n",
    "# Add decimal value labels to bars\n",
    "for bar, importance in zip(bars_rf, feature_importance_df_rf['Importance']):\n",
    "    plt.text(importance, bar.get_y() + bar.get_height()/2, f'{importance:.2f}', ha='left', va='center')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dddadd89",
   "metadata": {},
   "source": [
    "In traditional Naive Bayes models, such as Gaussian Naive Bayes, there isn't a direct concept of \"feature importance\" as there is in models like Random Forest or Logistic Regression.\n",
    "\n",
    "Naive Bayes models make predictions based on the probability distribution of features given the class and the prior probability of each class. These models assume that features are conditionally independent given the class, which means that the presence of one feature does not directly influence the presence of another feature.\n",
    "\n",
    "As a result, Naive Bayes models do not provide feature importance values in the same way as decision tree-based or linear models. Instead, they focus on probability calculations based on the input features and class labels."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4495be5",
   "metadata": {},
   "source": [
    "## Let's Test it Out!\n",
    "Let's first make a dummy dataframe with 3 rows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "a961eeef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Staff_Id</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>SA63171</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5582218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>SA63172</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>6200012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>SA63173</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>7305551</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "0                  3                  2                          3   \n",
       "1                  4                  3                          4   \n",
       "2                  5                  4                          4   \n",
       "\n",
       "   Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "0                   3                  4                           2   \n",
       "1                   3                  3                           4   \n",
       "2                   3                  2                           4   \n",
       "\n",
       "   Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "0                3                       3                            3   \n",
       "1                4                       3                            3   \n",
       "2                5                       4                            4   \n",
       "\n",
       "   Diversity & Inclusion  Work Environment  Compensation Staff_Id  \\\n",
       "0                      5                 4             3  SA63171   \n",
       "1                      5                 3             3  SA63172   \n",
       "2                      5                 4             4  SA63173   \n",
       "\n",
       "   Month_Of_Service  Residence_Code  Net_Salary  \n",
       "0                 1               4     5582218  \n",
       "1                 2               1     6200012  \n",
       "2                 3               2     7305551  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a dictionary with custom values for each row\n",
    "data = {\n",
    "    'Role Satisfaction': [3, 4, 5],\n",
    "    'Skill Utilization': [2, 3, 4],\n",
    "    'Career Growth Opportunity': [3, 4, 4],\n",
    "    'Supervisor Support': [3, 3, 3],\n",
    "    'Work-Life Balance': [4, 3, 2],\n",
    "    'Recognition & Appreciation': [2, 4, 4],\n",
    "    'Company Culture': [3, 4, 5],\n",
    "    'Training & Development': [3, 3, 4],\n",
    "    'Communication Effectiveness': [3, 3, 4],\n",
    "    'Diversity & Inclusion': [5, 5, 5],\n",
    "    'Work Environment': [4, 3, 4],\n",
    "    'Compensation': [3, 3, 4],\n",
    "    'Staff_Id': ['SA63171', 'SA63172', 'SA63173'],\n",
    "    'Month_Of_Service': [1, 2, 3],\n",
    "    'Residence_Code': [4, 1, 2],\n",
    "    'Net_Salary': [5582218, 6200012, 7305551]\n",
    "}\n",
    "\n",
    "# Create the DataFrame\n",
    "test_df = pd.DataFrame(data)\n",
    "\n",
    "# Display the DataFrame\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ea84b693",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Staff_Id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SA63171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SA63172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SA63173</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Staff_Id\n",
       "0  SA63171\n",
       "1  SA63172\n",
       "2  SA63173"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a DataFrame with only the 'Staff_Id' column for later \n",
    "staff_id_df = test_df[['Staff_Id']].copy()\n",
    "\n",
    "# Display the DataFrame\n",
    "staff_id_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b1b66185",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Role Satisfaction</th>\n",
       "      <th>Skill Utilization</th>\n",
       "      <th>Career Growth Opportunity</th>\n",
       "      <th>Supervisor Support</th>\n",
       "      <th>Work-Life Balance</th>\n",
       "      <th>Recognition &amp; Appreciation</th>\n",
       "      <th>Company Culture</th>\n",
       "      <th>Training &amp; Development</th>\n",
       "      <th>Communication Effectiveness</th>\n",
       "      <th>Diversity &amp; Inclusion</th>\n",
       "      <th>Work Environment</th>\n",
       "      <th>Compensation</th>\n",
       "      <th>Month_Of_Service</th>\n",
       "      <th>Residence_Code</th>\n",
       "      <th>Net_Salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5582218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>6200012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>7305551</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Role Satisfaction  Skill Utilization  Career Growth Opportunity  \\\n",
       "0                  3                  2                          3   \n",
       "1                  4                  3                          4   \n",
       "2                  5                  4                          4   \n",
       "\n",
       "   Supervisor Support  Work-Life Balance  Recognition & Appreciation  \\\n",
       "0                   3                  4                           2   \n",
       "1                   3                  3                           4   \n",
       "2                   3                  2                           4   \n",
       "\n",
       "   Company Culture  Training & Development  Communication Effectiveness  \\\n",
       "0                3                       3                            3   \n",
       "1                4                       3                            3   \n",
       "2                5                       4                            4   \n",
       "\n",
       "   Diversity & Inclusion  Work Environment  Compensation  Month_Of_Service  \\\n",
       "0                      5                 4             3                 1   \n",
       "1                      5                 3             3                 2   \n",
       "2                      5                 4             4                 3   \n",
       "\n",
       "   Residence_Code  Net_Salary  \n",
       "0               4     5582218  \n",
       "1               1     6200012  \n",
       "2               2     7305551  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the Staff_Id\n",
    "test_df = test_df.drop('Staff_Id', axis=1)\n",
    "test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95545726",
   "metadata": {},
   "source": [
    "Since we trained 3 models, we will use all of them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d66c5fc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions using Decision Tree:\n",
      "  Staff_Id  Prediction\n",
      "0  SA63171           0\n",
      "1  SA63172           0\n",
      "2  SA63173           0\n",
      "\n",
      "\n",
      "Predictions using Random Forest:\n",
      "  Staff_Id  Prediction\n",
      "0  SA63171           0\n",
      "1  SA63172           0\n",
      "2  SA63173           1\n",
      "\n",
      "\n",
      "Predictions using Naive Bayes:\n",
      "  Staff_Id  Prediction\n",
      "0  SA63171           0\n",
      "1  SA63172           0\n",
      "2  SA63173           0\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Initialize the models\n",
    "model_to_test = {\n",
    "    \"Decision Tree\": dt_model,\n",
    "    \"Random Forest\": rf_model,\n",
    "    \"Naive Bayes\": nb_model\n",
    "}\n",
    "\n",
    "# Iterate over the models\n",
    "for model_name, model in model_to_test.items():\n",
    "    # Make predictions\n",
    "    predictions = model.predict(test_df)\n",
    "    \n",
    "    # Create a DataFrame to store predictions\n",
    "    results_df = pd.DataFrame({\n",
    "        \"Prediction\": predictions\n",
    "    })\n",
    "    \n",
    "    # Merge predictions with Staff_Id\n",
    "    results_with_id_df = pd.concat([staff_id_df, results_df], axis=1)\n",
    "    \n",
    "    # Print the title\n",
    "    print(f\"Predictions using {model_name}:\")\n",
    "    \n",
    "    # Print the resulting DataFrame\n",
    "    print(results_with_id_df)\n",
    "    print(\"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6793d27b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
