{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "pvZF9BMSzAQ7"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## 1. Setup and Data Connection"
      ],
      "metadata": {
        "id": "3y-QjCv8KJoo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.1 Database Connection Setup"
      ],
      "metadata": {
        "id": "pvZF9BMSzAQ7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XuJLa6e8BKn5",
        "outputId": "209a2064-b797-4ccb-bbb0-8cc14dcb8eb1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install sqlalchemy psycopg2-binary -qq"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sqlalchemy import create_engine, text\n",
        "\n",
        "# database credentials\n",
        "DB_USER = \"salaryloan\"\n",
        "DB_PASSWORD = \"salaryloan\"\n",
        "DB_HOST = \"dev-data.simbrellang.net\"\n",
        "DB_PORT = \"10532\"\n",
        "DB_NAME = \"salaryloan\"\n",
        "\n",
        "# SQLAlchemy connection URL\n",
        "DATABASE_URL = f\"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}\"\n",
        "\n",
        "# Create the database engine\n",
        "engine = create_engine(DATABASE_URL)\n",
        "\n",
        "# Test the connection\n",
        "try:\n",
        "    with engine.connect() as conn:\n",
        "        result = conn.execute(text(\"SELECT version();\"))\n",
        "        print(\"Connected successfully!\")\n",
        "        print(result.fetchone())  # Print PostgreSQL version\n",
        "except Exception as e:\n",
        "    print(\"Error connecting to database:\", e)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B9tkEDTDBNQW",
        "outputId": "a7986644-80e4-488f-97ae-0d9d9dee3eb5"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Connected successfully!\n",
            "('PostgreSQL 16.8 (Ubuntu 16.8-0ubuntu0.24.04.1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0, 64-bit',)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.2 Data Loading\n"
      ],
      "metadata": {
        "id": "gWncpI3TzU1t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "import calendar\n",
        "from datetime import datetime\n",
        "\n",
        "table_name = \"customer_account_transaction_hx\"\n",
        "\n",
        "# Load data into a Pandas DataFrame\n",
        "df = pd.read_sql(f\"SELECT * FROM {table_name}\", engine)\n",
        "\n",
        "# Display the first few rows\n",
        "display(df.head(20))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "XTThey6EB0U1",
        "outputId": "fc3db25e-e988-4e42-fe6a-ceca293b8bfa"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "         id   accountid trx_start_date trx_end_date   amount d1  d2 d3  \\\n",
              "0   1335300  1945204013     2024-06-12   2024-06-12     54.0  T  BI  D   \n",
              "1   1335301  1945204013     2024-06-16   2024-06-16  52500.0  T  BI  C   \n",
              "2   1335302  1945204013     2024-06-15   2024-06-15    505.0  T  BI  D   \n",
              "3   1335303  1945204013     2024-06-17   2024-06-17   5350.0  T  CI  C   \n",
              "4   1335304  1945204013     2024-06-13   2024-06-13  10000.0  T  CI  D   \n",
              "5   1335305  2184939014     2024-06-17   2024-06-17  45150.0  T  CI  C   \n",
              "6   1335306  2184939014     2024-06-16   2024-06-16   2160.0  T  CI  C   \n",
              "7   1335307  2184939014     2024-06-13   2024-06-13  55640.0  T  BI  C   \n",
              "8   1335308  2184939014     2024-06-12   2024-06-12     53.5  T  BI  D   \n",
              "9   1335309  2184939014     2024-06-12   2024-06-12  10100.0  T  CI  D   \n",
              "10  1335310  2184939014     2024-06-14   2024-06-14  20800.0  T  CI  D   \n",
              "11  1335311  2184939014     2024-06-12   2024-06-12  21600.0  T  CI  D   \n",
              "12  1335312  2184939014     2024-06-15   2024-06-15  21000.0  T  CI  D   \n",
              "13  1335313  1757258011     2024-06-13   2024-06-13    104.0  T  CI  D   \n",
              "14  1335314  1757258011     2024-06-15   2024-06-15    330.0  T  CI  D   \n",
              "15  1335315  1757258011     2024-06-16   2024-06-16    107.0  T  CI  D   \n",
              "16  1335316  1757258011     2024-06-13   2024-06-13    105.0  T  CI  D   \n",
              "17  1335317  1757258011     2024-06-15   2024-06-15    108.0  T  CI  D   \n",
              "18  1335318  1757258011     2024-06-13   2024-06-13    109.0  T  CI  D   \n",
              "19  1335319  1757258011     2024-06-15   2024-06-15    106.0  T  CI  D   \n",
              "\n",
              "                                          description       d4  \n",
              "0                               CBN Stamp Duty Charge  1945204  \n",
              "1                                MTHLY ALLO MAR 2018.  1945204  \n",
              "2                                   WELFARE MAR. 2018  1945204  \n",
              "3                                  Online : blessfyne  1945204  \n",
              "4   ATM WD @ FCMB Trans-Amadi 2 ATM1PortHarcourt RVNG  1945204  \n",
              "5                                         web:abdulai  2184939  \n",
              "6                   CSH DEPOSIT BY : ROTIMI OLUWATOBI  2184939  \n",
              "7                                MTHLY ALLO MAR 2018.  2184939  \n",
              "8                               CBN Stamp Duty Charge  2184939  \n",
              "9   ATM WD @ FCMB ONIPANU-2 IKORODU RD, LAGOS    LANG  2184939  \n",
              "10           368837107 BILL PAY @ 12141451/2184939014  2184939  \n",
              "11  ATM WD @ FCMB ONIPANU-3 IKORODU RD, LAGOS    LANG  2184939  \n",
              "12  ATM WD @ FCMB ONIPANU-3 IKORODU RD, LAGOS    LANG  2184939  \n",
              "13       QR/Q10009078250/Airtime Top-up/2347037279970  1757258  \n",
              "14  1019323435/WEB PURCHASE @ 3IWPDNRB/ww.nairabet...  1757258  \n",
              "15       QR/Q10009079949/Airtime Top-up/2347037279970  1757258  \n",
              "16       QR/Q10009036640/Airtime Top-up/2347037279970  1757258  \n",
              "17       QR/Q10008928860/Airtime Top-up/2347037279970  1757258  \n",
              "18  1013759550/WEB PURCHASE @ 3IWPDNRB/ww.nairabet...  1757258  \n",
              "19       QR/Q10008860487/Airtime Top-up/2347037279970  1757258  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-992038e2-6493-4d84-bee4-239371ba6926\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>accountid</th>\n",
              "      <th>trx_start_date</th>\n",
              "      <th>trx_end_date</th>\n",
              "      <th>amount</th>\n",
              "      <th>d1</th>\n",
              "      <th>d2</th>\n",
              "      <th>d3</th>\n",
              "      <th>description</th>\n",
              "      <th>d4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1335300</td>\n",
              "      <td>1945204013</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>54.0</td>\n",
              "      <td>T</td>\n",
              "      <td>BI</td>\n",
              "      <td>D</td>\n",
              "      <td>CBN Stamp Duty Charge</td>\n",
              "      <td>1945204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1335301</td>\n",
              "      <td>1945204013</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>52500.0</td>\n",
              "      <td>T</td>\n",
              "      <td>BI</td>\n",
              "      <td>C</td>\n",
              "      <td>MTHLY ALLO MAR 2018.</td>\n",
              "      <td>1945204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1335302</td>\n",
              "      <td>1945204013</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>505.0</td>\n",
              "      <td>T</td>\n",
              "      <td>BI</td>\n",
              "      <td>D</td>\n",
              "      <td>WELFARE MAR. 2018</td>\n",
              "      <td>1945204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1335303</td>\n",
              "      <td>1945204013</td>\n",
              "      <td>2024-06-17</td>\n",
              "      <td>2024-06-17</td>\n",
              "      <td>5350.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>Online : blessfyne</td>\n",
              "      <td>1945204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1335304</td>\n",
              "      <td>1945204013</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>10000.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>ATM WD @ FCMB Trans-Amadi 2 ATM1PortHarcourt RVNG</td>\n",
              "      <td>1945204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1335305</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-17</td>\n",
              "      <td>2024-06-17</td>\n",
              "      <td>45150.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>web:abdulai</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1335306</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>2160.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>CSH DEPOSIT BY : ROTIMI OLUWATOBI</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1335307</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>55640.0</td>\n",
              "      <td>T</td>\n",
              "      <td>BI</td>\n",
              "      <td>C</td>\n",
              "      <td>MTHLY ALLO MAR 2018.</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1335308</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>53.5</td>\n",
              "      <td>T</td>\n",
              "      <td>BI</td>\n",
              "      <td>D</td>\n",
              "      <td>CBN Stamp Duty Charge</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1335309</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>10100.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>ATM WD @ FCMB ONIPANU-2 IKORODU RD, LAGOS    LANG</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>1335310</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>20800.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>368837107 BILL PAY @ 12141451/2184939014</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>1335311</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>21600.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>ATM WD @ FCMB ONIPANU-3 IKORODU RD, LAGOS    LANG</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>1335312</td>\n",
              "      <td>2184939014</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>21000.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>ATM WD @ FCMB ONIPANU-3 IKORODU RD, LAGOS    LANG</td>\n",
              "      <td>2184939</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>1335313</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>104.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>QR/Q10009078250/Airtime Top-up/2347037279970</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>1335314</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>330.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>1019323435/WEB PURCHASE @ 3IWPDNRB/ww.nairabet...</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>1335315</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>107.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>QR/Q10009079949/Airtime Top-up/2347037279970</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>1335316</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>105.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>QR/Q10009036640/Airtime Top-up/2347037279970</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>1335317</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>108.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>QR/Q10008928860/Airtime Top-up/2347037279970</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>1335318</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>109.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>1013759550/WEB PURCHASE @ 3IWPDNRB/ww.nairabet...</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>1335319</td>\n",
              "      <td>1757258011</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>106.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>D</td>\n",
              "      <td>QR/Q10008860487/Airtime Top-up/2347037279970</td>\n",
              "      <td>1757258</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-992038e2-6493-4d84-bee4-239371ba6926')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-992038e2-6493-4d84-bee4-239371ba6926 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-992038e2-6493-4d84-bee4-239371ba6926');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fbf7dd3b-e862-467f-bf36-0ac42636d149\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fbf7dd3b-e862-467f-bf36-0ac42636d149')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fbf7dd3b-e862-467f-bf36-0ac42636d149 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"display(df\",\n  \"rows\": 20,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5,\n        \"min\": 1335300,\n        \"max\": 1335319,\n        \"num_unique_values\": 20,\n        \"samples\": [\n          1335300,\n          1335317,\n          1335315\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"accountid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"1945204013\",\n          \"2184939014\",\n          \"1757258011\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"trx_start_date\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2024-06-12 00:00:00\",\n        \"max\": \"2024-06-17 00:00:00\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"2024-06-12 00:00:00\",\n          \"2024-06-16 00:00:00\",\n          \"2024-06-14 00:00:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"trx_end_date\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2024-06-12 00:00:00\",\n        \"max\": \"2024-06-17 00:00:00\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"2024-06-12 00:00:00\",\n          \"2024-06-16 00:00:00\",\n          \"2024-06-14 00:00:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"amount\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 18415.714645431864,\n        \"min\": 53.5,\n        \"max\": 55640.0,\n        \"num_unique_values\": 20,\n        \"samples\": [\n          54.0,\n          108.0,\n          107.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d1\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"T\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d2\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"CI\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d3\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"C\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"description\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 17,\n        \"samples\": [\n          \"CBN Stamp Duty Charge\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d4\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"1945204\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.3  Date Features Extraction"
      ],
      "metadata": {
        "id": "Ej3ING67xp_M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"trx_start_date\"] = pd.to_datetime(df[\"trx_start_date\"])\n",
        "df[\"year\"] = df[\"trx_start_date\"].dt.year\n",
        "df[\"month\"] = df[\"trx_start_date\"].dt.month\n",
        "df[\"day_of_month\"] = df[\"trx_start_date\"].dt.day"
      ],
      "metadata": {
        "id": "ldpyVrNkB5mu"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2. Salary Transactions Identification Heuristics\n",
        "**Heuristic Design Principles**\n",
        "\n",
        "**Meta Criteria:** Salary should be a credit, denoted as `\"C\"` in the `d3` column.\n",
        "1. Mention of salary in the description. **Keywords**:\"salary\" \"payroll\" \"income\"\n",
        "2. Investigate patterns in the confirmed salary  transactions.\n",
        "\n",
        "  **a. Time Period:** Significant difference in money received in \"salary_periods\" and remainder of the months.\n",
        "\n",
        "  **b. Transaction types:** See if there are patterns in transaction type for verified salary transactions agianst others.\n",
        "\n",
        "3. Consistently recieve an outlier(large) but similar credit amount of money every month.\n",
        "\n"
      ],
      "metadata": {
        "id": "HMsJe_x6qEjK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### H1: Salary in Description"
      ],
      "metadata": {
        "id": "J2yi-nDm2TJJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# look for transactions with salary in the description\n",
        "df[\"H1\"] =0\n",
        "\n",
        "df.loc[((df[\"description\"].str.contains(\"salary\", case=False)) & (df[\"d3\"]== 'C')), \"H1\"] = 1\n",
        "\n",
        "salary_description_transactions = df[df[\"H1\"]==1]\n",
        "\n",
        "display(salary_description_transactions.head(20))\n",
        "\n",
        "print(f\"Identified {len(salary_description_transactions)} salary-like transactions\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 766
        },
        "id": "knQcLweriJEQ",
        "outputId": "2a651385-7e60-44aa-f692-a4baf1032544"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "            id   accountid trx_start_date trx_end_date       amount d1  d2 d3  \\\n",
              "1434   1336731  2083803018     2024-06-11   2024-06-11   39844.3416  T  CI  C   \n",
              "3939   1525237  2241150015     2024-06-14   2024-06-14   45900.0000  T  BI  C   \n",
              "6195   1341485  1411215013     2024-06-15   2024-06-15  111116.0052  T  CI  C   \n",
              "8416   1343700  1269970016     2024-06-16   2024-06-16   43261.0632  T  CI  C   \n",
              "10151  1345432  1947900010     2024-06-15   2024-06-15    9048.0000  T  CI  C   \n",
              "10831  1346111  1218765016     2024-06-12   2024-06-12   40457.1054  T  CI  C   \n",
              "10977  1346257  1275044015     2024-06-13   2024-06-13  135318.8824  T  CI  C   \n",
              "11764  1347043  2253348011     2024-06-14   2024-06-14   25721.5286  T  CI  C   \n",
              "11805  1347084  1957221017     2024-06-12   2024-06-12   11391.1928  T  CI  C   \n",
              "12956  1348237  1155907018     2024-06-12   2024-06-12   81146.8245  T  CI  C   \n",
              "14929  1350209  1543039017     2024-06-07   2024-06-07   47321.8595  T   I  C   \n",
              "15470  1350750  2131439013     2024-06-12   2024-06-12  118615.8665  T  CI  C   \n",
              "15473  1350753  2131439013     2024-06-14   2024-06-14  115290.1880  T  CI  C   \n",
              "15543  1350823  1516029010     2024-06-16   2024-06-16  146963.9806  T  CI  C   \n",
              "16794  1352075  1989415019     2024-06-14   2024-06-14  126280.1904  T  CI  C   \n",
              "17771  1353052  1998110015     2024-06-12   2024-06-12   13851.0000  T  CI  C   \n",
              "18041  1353322  1259062019     2024-06-10   2024-06-10   27510.5322  T  CI  C   \n",
              "19668  1354949  2020313017     2024-06-10   2024-06-10    5500.0000  T  CI  C   \n",
              "22009  1357281  2117066019     2024-06-13   2024-06-13   42059.3670  T  CI  C   \n",
              "25213  1360480  0897678011     2024-06-10   2024-06-10   55141.2663  T  CI  C   \n",
              "\n",
              "                                             description       d4  year  \\\n",
              "1434                                   IMSG MARCH SALARY  2083803  2024   \n",
              "3939                                     SALARY PAYMENTS  2241150  2024   \n",
              "6195                                   IMSG MARCH SALARY  1411215  2024   \n",
              "8416                              IMSG MARCH 2018 SALARY  1269970  2024   \n",
              "10151                                   SALARY MAR. 2018  1947900  2024   \n",
              "10831                             IMSG MARCH 2018 SALARY  1218765  2024   \n",
              "10977           106/MAR2018 Salary/3152861/         LANG  1275044  2024   \n",
              "11764     N-10022682239/OYIGBO LOC-FEB 2018 SALARY - OYI  2253348  2024   \n",
              "11805              MARCH 2018 SALARY BAPTIST HIGH SCH 11  1957221  2024   \n",
              "12956                                  IMSG MARCH SALARY  1155907  2024   \n",
              "14929                               FEB SALARY=DESOPADEC  1543039  2024   \n",
              "15470     R-215105337/UNIVE:JANUARY2018NETSALARY:CBN:215  2131439  2024   \n",
              "15473     R-215063626/UNIVE:DECEMBER2017NETSALARY:CBN:21  2131439  2024   \n",
              "15543                 N-10022787984/MAR 18 Salary:CITI::  1516029  2024   \n",
              "16794     R-214329984/TRANS:SALARY:CBN:88829762-65070-:2  1989415  2024   \n",
              "17771                              STAFF SALARY MAR 2018  1998110  2024   \n",
              "18041  COP08DPSS960249;54469074;NSUK FEB 2018 SALARY-NSU  1259062  2024   \n",
              "19668                         web:Upfront salary payment  2020313  2024   \n",
              "22009                             IMSG MARCH 2018 SALARY  2117066  2024   \n",
              "25213                                  IMSG MARCH SALARY  0897678  2024   \n",
              "\n",
              "       month  day_of_month  H1  \n",
              "1434       6            11   1  \n",
              "3939       6            14   1  \n",
              "6195       6            15   1  \n",
              "8416       6            16   1  \n",
              "10151      6            15   1  \n",
              "10831      6            12   1  \n",
              "10977      6            13   1  \n",
              "11764      6            14   1  \n",
              "11805      6            12   1  \n",
              "12956      6            12   1  \n",
              "14929      6             7   1  \n",
              "15470      6            12   1  \n",
              "15473      6            14   1  \n",
              "15543      6            16   1  \n",
              "16794      6            14   1  \n",
              "17771      6            12   1  \n",
              "18041      6            10   1  \n",
              "19668      6            10   1  \n",
              "22009      6            13   1  \n",
              "25213      6            10   1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2529fa90-9eb3-4ac8-a693-b50749b35ca3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>accountid</th>\n",
              "      <th>trx_start_date</th>\n",
              "      <th>trx_end_date</th>\n",
              "      <th>amount</th>\n",
              "      <th>d1</th>\n",
              "      <th>d2</th>\n",
              "      <th>d3</th>\n",
              "      <th>description</th>\n",
              "      <th>d4</th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day_of_month</th>\n",
              "      <th>H1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1434</th>\n",
              "      <td>1336731</td>\n",
              "      <td>2083803018</td>\n",
              "      <td>2024-06-11</td>\n",
              "      <td>2024-06-11</td>\n",
              "      <td>39844.3416</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH SALARY</td>\n",
              "      <td>2083803</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>11</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3939</th>\n",
              "      <td>1525237</td>\n",
              "      <td>2241150015</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>45900.0000</td>\n",
              "      <td>T</td>\n",
              "      <td>BI</td>\n",
              "      <td>C</td>\n",
              "      <td>SALARY PAYMENTS</td>\n",
              "      <td>2241150</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6195</th>\n",
              "      <td>1341485</td>\n",
              "      <td>1411215013</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>111116.0052</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH SALARY</td>\n",
              "      <td>1411215</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8416</th>\n",
              "      <td>1343700</td>\n",
              "      <td>1269970016</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>43261.0632</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH 2018 SALARY</td>\n",
              "      <td>1269970</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>16</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10151</th>\n",
              "      <td>1345432</td>\n",
              "      <td>1947900010</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>2024-06-15</td>\n",
              "      <td>9048.0000</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>SALARY MAR. 2018</td>\n",
              "      <td>1947900</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10831</th>\n",
              "      <td>1346111</td>\n",
              "      <td>1218765016</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>40457.1054</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH 2018 SALARY</td>\n",
              "      <td>1218765</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10977</th>\n",
              "      <td>1346257</td>\n",
              "      <td>1275044015</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>135318.8824</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>106/MAR2018 Salary/3152861/         LANG</td>\n",
              "      <td>1275044</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11764</th>\n",
              "      <td>1347043</td>\n",
              "      <td>2253348011</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>25721.5286</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>N-10022682239/OYIGBO LOC-FEB 2018 SALARY - OYI</td>\n",
              "      <td>2253348</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11805</th>\n",
              "      <td>1347084</td>\n",
              "      <td>1957221017</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>11391.1928</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>MARCH 2018 SALARY BAPTIST HIGH SCH 11</td>\n",
              "      <td>1957221</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12956</th>\n",
              "      <td>1348237</td>\n",
              "      <td>1155907018</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>81146.8245</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH SALARY</td>\n",
              "      <td>1155907</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14929</th>\n",
              "      <td>1350209</td>\n",
              "      <td>1543039017</td>\n",
              "      <td>2024-06-07</td>\n",
              "      <td>2024-06-07</td>\n",
              "      <td>47321.8595</td>\n",
              "      <td>T</td>\n",
              "      <td>I</td>\n",
              "      <td>C</td>\n",
              "      <td>FEB SALARY=DESOPADEC</td>\n",
              "      <td>1543039</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15470</th>\n",
              "      <td>1350750</td>\n",
              "      <td>2131439013</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>118615.8665</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>R-215105337/UNIVE:JANUARY2018NETSALARY:CBN:215</td>\n",
              "      <td>2131439</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15473</th>\n",
              "      <td>1350753</td>\n",
              "      <td>2131439013</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>115290.1880</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>R-215063626/UNIVE:DECEMBER2017NETSALARY:CBN:21</td>\n",
              "      <td>2131439</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15543</th>\n",
              "      <td>1350823</td>\n",
              "      <td>1516029010</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>2024-06-16</td>\n",
              "      <td>146963.9806</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>N-10022787984/MAR 18 Salary:CITI::</td>\n",
              "      <td>1516029</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>16</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16794</th>\n",
              "      <td>1352075</td>\n",
              "      <td>1989415019</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>2024-06-14</td>\n",
              "      <td>126280.1904</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>R-214329984/TRANS:SALARY:CBN:88829762-65070-:2</td>\n",
              "      <td>1989415</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17771</th>\n",
              "      <td>1353052</td>\n",
              "      <td>1998110015</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>2024-06-12</td>\n",
              "      <td>13851.0000</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>STAFF SALARY MAR 2018</td>\n",
              "      <td>1998110</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18041</th>\n",
              "      <td>1353322</td>\n",
              "      <td>1259062019</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>27510.5322</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>COP08DPSS960249;54469074;NSUK FEB 2018 SALARY-NSU</td>\n",
              "      <td>1259062</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19668</th>\n",
              "      <td>1354949</td>\n",
              "      <td>2020313017</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>5500.0000</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>web:Upfront salary payment</td>\n",
              "      <td>2020313</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22009</th>\n",
              "      <td>1357281</td>\n",
              "      <td>2117066019</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>2024-06-13</td>\n",
              "      <td>42059.3670</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH 2018 SALARY</td>\n",
              "      <td>2117066</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25213</th>\n",
              "      <td>1360480</td>\n",
              "      <td>0897678011</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>55141.2663</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>IMSG MARCH SALARY</td>\n",
              "      <td>0897678</td>\n",
              "      <td>2024</td>\n",
              "      <td>6</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2529fa90-9eb3-4ac8-a693-b50749b35ca3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2529fa90-9eb3-4ac8-a693-b50749b35ca3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2529fa90-9eb3-4ac8-a693-b50749b35ca3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a524dc01-5f38-4566-b974-1708b533b0e4\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a524dc01-5f38-4566-b974-1708b533b0e4')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a524dc01-5f38-4566-b974-1708b533b0e4 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"print(f\\\"Identified {len(salary_description_transactions)} salary-like transactions\\\")\",\n  \"rows\": 20,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 39725,\n        \"min\": 1336731,\n        \"max\": 1525237,\n        \"num_unique_values\": 20,\n        \"samples\": [\n          1336731,\n          1354949,\n          1353052\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"accountid\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 19,\n        \"samples\": [\n          \"2083803018\",\n          \"1218765016\",\n          \"2131439013\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"trx_start_date\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2024-06-07 00:00:00\",\n        \"max\": \"2024-06-16 00:00:00\",\n        \"num_unique_values\": 8,\n        \"samples\": [\n          \"2024-06-14 00:00:00\",\n          \"2024-06-13 00:00:00\",\n          \"2024-06-11 00:00:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"trx_end_date\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2024-06-07 00:00:00\",\n        \"max\": \"2024-06-16 00:00:00\",\n        \"num_unique_values\": 8,\n        \"samples\": [\n          \"2024-06-14 00:00:00\",\n          \"2024-06-13 00:00:00\",\n          \"2024-06-11 00:00:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"amount\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 46542.19951227044,\n        \"min\": 5500.0,\n        \"max\": 146963.9806,\n        \"num_unique_values\": 20,\n        \"samples\": [\n          39844.3416,\n          5500.0,\n          13851.000000000002\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d1\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"T\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d2\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"CI\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d3\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"C\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"description\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 15,\n        \"samples\": [\n          \"R-215063626/UNIVE:DECEMBER2017NETSALARY:CBN:21\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"d4\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 19,\n        \"samples\": [\n          \"2083803\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"year\",\n      \"properties\": {\n        \"dtype\": \"int32\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          2024\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"month\",\n      \"properties\": {\n        \"dtype\": \"int32\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          6\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"day_of_month\",\n      \"properties\": {\n        \"dtype\": \"int32\",\n        \"num_unique_values\": 8,\n        \"samples\": [\n          14\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"H1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 1,\n        \"max\": 1,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Identified 4228 salary-like transactions\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "NB: Some accounts receive multiple salaries in the same month. e.g accountid, \"0551610021\" which received 28 salary described transactions within the space of 12 months. The amounts were not always the same, meaning it wasn't duplicate/bounced transactions, it was infact multi-salaries. Current fix: Use max salary received in a month, to get the monthly received salary for the month.\n",
        "Alternative: Sum all salries in a month."
      ],
      "metadata": {
        "id": "_6UsJNKUaGZL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### H2: Investigate patterns in transaction types and dates for known salary transactions."
      ],
      "metadata": {
        "id": "Jw4vszdwdc78"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Days of Month"
      ],
      "metadata": {
        "id": "BT7H2p8p4gLr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate and display frequency of salary payments by day of month\n",
        "print(\"Salary Payment Day Frequency:\")\n",
        "print(salary_description_transactions.day_of_month.value_counts())\n",
        "\n",
        "# Create histogram to visualize distribution of payment days\n",
        "ax = salary_description_transactions.day_of_month.plot(\n",
        "    kind=\"hist\",\n",
        "    bins=31,\n",
        "    edgecolor='black',\n",
        "    figsize=(10, 6),\n",
        "    title=\"Distribution of Salary Payment Days\\n(Frequency by Day of Month)\",\n",
        "    xlabel=\"Day of Month\",\n",
        "    ylabel=\"Number of Salary Payments\",\n",
        "    color='#1f77b4'\n",
        ")\n",
        "\n",
        "# Add grid for better readability\n",
        "ax.grid(axis='y', alpha=0.3)\n",
        "\n",
        "\n",
        "# Calculate percentage of salaries paid in top 7 most common days\n",
        "top_days_coverage = (\n",
        "    salary_description_transactions.day_of_month\n",
        "    .value_counts()\n",
        "    .nlargest(7)\n",
        "    .sum() / salary_description_transactions.shape[0]\n",
        ") * 100  # Convert to percentage\n",
        "\n",
        "print(f\"\\nTop 7 payment days account for {top_days_coverage:.1f}% of all salary transactions\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "3zTsFcAmn8RU",
        "outputId": "ac4bc872-1bdc-4668-d0e7-25ec2a1dcc73"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Salary Payment Day Frequency:\n",
            "day_of_month\n",
            "20    226\n",
            "21    206\n",
            "19    194\n",
            "18    193\n",
            "22    180\n",
            "17    174\n",
            "23    163\n",
            "16    146\n",
            "25    146\n",
            "24    144\n",
            "15    142\n",
            "13    135\n",
            "12    133\n",
            "29    133\n",
            "11    129\n",
            "28    126\n",
            "2     126\n",
            "14    125\n",
            "6     125\n",
            "3     123\n",
            "30    121\n",
            "7     119\n",
            "27    115\n",
            "10    112\n",
            "5     112\n",
            "1     111\n",
            "9     108\n",
            "26    107\n",
            "4     105\n",
            "8      97\n",
            "31     52\n",
            "Name: count, dtype: int64\n",
            "\n",
            "Top 7 payment days account for 31.6% of all salary transactions\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 36564 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Transaction Types"
      ],
      "metadata": {
        "id": "5IoiWOIH4lbU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "# for all transaction type columns, d1-d3, print difference in proportion between confirmed salaries and unconfirmed.\n",
        "def calculate_proportion_differences(column):\n",
        "    # Calculate proportions for salary vs non-salary\n",
        "    props = df.groupby('H1')[column].value_counts(normalize=True).unstack().fillna(0)\n",
        "\n",
        "    # Calculate absolute percentage point differences\n",
        "    prop_diff = (props.loc[1] - props.loc[0]).abs().sort_values(ascending=False)\n",
        "\n",
        "    # Format results\n",
        "    results = pd.concat([\n",
        "        props.loc[1].rename('Salary%'),\n",
        "        props.loc[0].rename('Non-Salary%'),\n",
        "        prop_diff.rename('Difference (pp)')\n",
        "    ], axis=1)\n",
        "\n",
        "    return results.sort_values('Difference (pp)', ascending=False)\n",
        "\n",
        "# Analyze all transaction type columns\n",
        "transaction_cols = ['d1', 'd2', 'd3']\n",
        "\n",
        "\n",
        "def plot_component_bars(results_df, col, top_n=10):\n",
        "    # Filter top N categories with largest differences\n",
        "    top_categories = results_df.head(top_n).index\n",
        "\n",
        "    # Get proportions for these categories\n",
        "    salary_props = results_df.loc[top_categories, 'Salary%']\n",
        "    non_salary_props = results_df.loc[top_categories, 'Non-Salary%']\n",
        "\n",
        "    # Create figure\n",
        "    fig, ax = plt.subplots(figsize=(12, 6))\n",
        "\n",
        "    # Set positions and width\n",
        "    x = np.arange(len(top_categories))\n",
        "    width = 0.35\n",
        "\n",
        "    # Plot bars\n",
        "    rects1 = ax.bar(x - width/2, salary_props*100, width,\n",
        "                   label='Salary Transactions', color='#1f77b4')\n",
        "    rects2 = ax.bar(x + width/2, non_salary_props*100, width,\n",
        "                   label='Non-Salary Transactions', color='#ff7f0e')\n",
        "\n",
        "    # Add labels and title\n",
        "    ax.set_ylabel('Percentage (%)')\n",
        "    ax.set_title(f'Transaction Type Composition: {col}')\n",
        "    ax.set_xticks(x)\n",
        "    ax.set_xticklabels(top_categories, rotation=45, ha='right')\n",
        "    ax.legend()\n",
        "\n",
        "    # Add value labels\n",
        "    def autolabel(rects):\n",
        "        for rect in rects:\n",
        "            height = rect.get_height()\n",
        "            ax.annotate(f'{height:.1f}%',\n",
        "                        xy=(rect.get_x() + rect.get_width() / 2, height),\n",
        "                        xytext=(0, 3),  # 3 points vertical offset\n",
        "                        textcoords=\"offset points\",\n",
        "                        ha='center', va='bottom', fontsize=8)\n",
        "\n",
        "    autolabel(rects1)\n",
        "    autolabel(rects2)\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "# Analyze all transaction type columns\n",
        "transaction_cols = ['d1', 'd2', 'd3']\n",
        "\n",
        "for col in transaction_cols:\n",
        "    print(f\"\\n\\033[1mTransaction Type Analysis: {col}\\033[0m\")\n",
        "    print(f\"Categories in salary transactions: {df[df['H1']==1][col].nunique()}\")\n",
        "\n",
        "    diff_df = calculate_proportion_differences(col)\n",
        "    display(diff_df.head(10))\n",
        "\n",
        "    # Plot component bar chart\n",
        "    plot_component_bars(diff_df, col, top_n=10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "h20OR7Moz-XB",
        "outputId": "2a1f35dd-e294-4968-8278-c5e25525f165"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\u001b[1mTransaction Type Analysis: d1\u001b[0m\n",
            "Categories in salary transactions: 1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Salary%  Non-Salary%  Difference (pp)\n",
              "d1                                       \n",
              "T       1.0     0.989452         0.010548\n",
              "C       0.0     0.010221         0.010221\n",
              "L       0.0     0.000327         0.000327"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-dcce0bba-744f-4899-a642-6b004f075d2c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Salary%</th>\n",
              "      <th>Non-Salary%</th>\n",
              "      <th>Difference (pp)</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>d1</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>T</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.989452</td>\n",
              "      <td>0.010548</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>C</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.010221</td>\n",
              "      <td>0.010221</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>L</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000327</td>\n",
              "      <td>0.000327</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dcce0bba-744f-4899-a642-6b004f075d2c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-dcce0bba-744f-4899-a642-6b004f075d2c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-dcce0bba-744f-4899-a642-6b004f075d2c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6f56ad91-0c0b-4a85-80dd-6904523c748c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6f56ad91-0c0b-4a85-80dd-6904523c748c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6f56ad91-0c0b-4a85-80dd-6904523c748c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    plot_component_bars(diff_df, col, top_n=10)\",\n  \"rows\": 3,\n  \"fields\": [\n    {\n      \"column\": \"d1\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"T\",\n          \"C\",\n          \"L\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Salary%\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5773502691896258,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0.0,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Non-Salary%\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5682367466566406,\n        \"min\": 0.0003270979736934726,\n        \"max\": 0.9894517445443329,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.9894517445443329,\n          0.010221157481973631\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Difference (pp)\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.005809065715314987,\n        \"min\": 0.0003270979736934726,\n        \"max\": 0.010548255455667066,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.010548255455667066,\n          0.010221157481973631\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 33920 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\u001b[1mTransaction Type Analysis: d2\u001b[0m\n",
            "Categories in salary transactions: 4\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "     Salary%  Non-Salary%  Difference (pp)\n",
              "d2                                        \n",
              "BI  0.132450     0.201219         0.068769\n",
              "I   0.069536     0.003271         0.066265\n",
              "BS  0.052980     0.000076         0.052904\n",
              "CI  0.745033     0.784823         0.039790\n",
              "NR  0.000000     0.007102         0.007102\n",
              "NP  0.000000     0.003119         0.003119\n",
              "O   0.000000     0.000322         0.000322\n",
              "SI  0.000000     0.000044         0.000044\n",
              "IP  0.000000     0.000021         0.000021\n",
              "SC  0.000000     0.000003         0.000003"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-adbdd9dc-3d8f-4ac4-a0c4-133a1d8529e6\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Salary%</th>\n",
              "      <th>Non-Salary%</th>\n",
              "      <th>Difference (pp)</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>d2</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>BI</th>\n",
              "      <td>0.132450</td>\n",
              "      <td>0.201219</td>\n",
              "      <td>0.068769</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>I</th>\n",
              "      <td>0.069536</td>\n",
              "      <td>0.003271</td>\n",
              "      <td>0.066265</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>BS</th>\n",
              "      <td>0.052980</td>\n",
              "      <td>0.000076</td>\n",
              "      <td>0.052904</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CI</th>\n",
              "      <td>0.745033</td>\n",
              "      <td>0.784823</td>\n",
              "      <td>0.039790</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NR</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.007102</td>\n",
              "      <td>0.007102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NP</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.003119</td>\n",
              "      <td>0.003119</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>O</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000322</td>\n",
              "      <td>0.000322</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SI</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000044</td>\n",
              "      <td>0.000044</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>IP</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000021</td>\n",
              "      <td>0.000021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SC</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000003</td>\n",
              "      <td>0.000003</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-adbdd9dc-3d8f-4ac4-a0c4-133a1d8529e6')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-adbdd9dc-3d8f-4ac4-a0c4-133a1d8529e6 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-adbdd9dc-3d8f-4ac4-a0c4-133a1d8529e6');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-30ab93bd-5852-4a60-9270-b2895113e887\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-30ab93bd-5852-4a60-9270-b2895113e887')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-30ab93bd-5852-4a60-9270-b2895113e887 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    plot_component_bars(diff_df, col, top_n=10)\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"d2\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"IP\",\n          \"I\",\n          \"NP\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Salary%\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.23100255836370376,\n        \"min\": 0.0,\n        \"max\": 0.7450331125827815,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.0695364238410596,\n          0.0,\n          0.052980132450331126\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Non-Salary%\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.24866465739362864,\n        \"min\": 2.6167837895477805e-06,\n        \"max\": 0.7848228035511251,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          2.0934270316382244e-05,\n          0.0032709797369347257,\n          0.0031192062771409545\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Difference (pp)\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.029593659578119966,\n        \"min\": 2.6167837895477805e-06,\n        \"max\": 0.06876894058959918,\n        \"num_unique_values\": 10,\n        \"samples\": [\n          2.0934270316382244e-05,\n          0.06626544410412487,\n          0.0031192062771409545\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 45204 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\u001b[1mTransaction Type Analysis: d3\u001b[0m\n",
            "Categories in salary transactions: 1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Salary%  Non-Salary%  Difference (pp)\n",
              "d3                                       \n",
              "D       0.0     0.808003         0.808003\n",
              "C       1.0     0.191997         0.808003"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4644302d-48b0-4d06-a8e5-5a5275f190ff\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Salary%</th>\n",
              "      <th>Non-Salary%</th>\n",
              "      <th>Difference (pp)</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>d3</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>D</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.808003</td>\n",
              "      <td>0.808003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>C</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.191997</td>\n",
              "      <td>0.808003</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4644302d-48b0-4d06-a8e5-5a5275f190ff')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4644302d-48b0-4d06-a8e5-5a5275f190ff button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4644302d-48b0-4d06-a8e5-5a5275f190ff');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ef8e781c-38d9-456a-8a2d-f32b9435b77d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ef8e781c-38d9-456a-8a2d-f32b9435b77d')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ef8e781c-38d9-456a-8a2d-f32b9435b77d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"    plot_component_bars(diff_df, col, top_n=10)\",\n  \"rows\": 2,\n  \"fields\": [\n    {\n      \"column\": \"d3\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"C\",\n          \"D\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Salary%\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7071067811865476,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Non-Salary%\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.43558152231033176,\n        \"min\": 0.19199735181480498,\n        \"max\": 0.8080026481851951,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0.19199735181480498,\n          0.8080026481851951\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Difference (pp)\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1102230246251565e-16,\n        \"min\": 0.808002648185195,\n        \"max\": 0.8080026481851951,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0.808002648185195,\n          0.8080026481851951\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 34696 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Conclusion**:\n",
        "There are no clear distinguishing criterion for salaries using the days of the month and transaction types."
      ],
      "metadata": {
        "id": "GGFCb__L4tbk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### H3: Consistently high credits"
      ],
      "metadata": {
        "id": "qm1FX4WyDR2-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def identify_consistent_high_credits(df, n_months=3, cv_threshold=0.15):\n",
        "    \"\"\"\n",
        "    Identifies salary-like transactions by:\n",
        "    1. Finding monthly max credits per account\n",
        "    2. Calculating consistency (CV) of these max amounts\n",
        "    3. Flagging ONLY the max transactions from consistent accounts\n",
        "    \"\"\"\n",
        "    # Convert to datetime and extract month\n",
        "    df['month'] = pd.to_datetime(df['trx_start_date']).dt.to_period('M')\n",
        "\n",
        "    # Filter only credit transactions\n",
        "    credits = df[df['d3'] == 'C'].copy()\n",
        "\n",
        "    # Get monthly max credits per account (with transaction IDs)\n",
        "    monthly_max = credits.loc[credits.groupby(['accountid', 'month'])['amount'].idxmax()]\n",
        "\n",
        "    # Calculate dispersion metrics for each account\n",
        "    dispersion = monthly_max.groupby('accountid').agg(\n",
        "        mean_amount=('amount', 'mean'),\n",
        "        std_amount=('amount', 'std'),\n",
        "        count_months=('amount', 'count'),\n",
        "        mad=('amount', lambda x: np.mean(np.abs(x - np.mean(x)))),  # Mean Absolute Deviation\n",
        "        cv=('amount', lambda x: np.mean(np.abs(x - np.mean(x)))/np.mean(x) if len(x) > 1 and np.mean(x) != 0 else np.nan)  # MAD/Mean ratio as CV\n",
        "    ).reset_index()\n",
        "\n",
        "    # Identify consistent accounts\n",
        "    consistent_accounts = dispersion[\n",
        "        (dispersion['count_months'] >= n_months) &\n",
        "        (dispersion['cv'] <= cv_threshold)\n",
        "    ]['accountid']\n",
        "\n",
        "    # Create flag column (initialize with 0)\n",
        "    df['H3'] = 0\n",
        "\n",
        "    # Flag ONLY the specific max transactions from consistent accounts\n",
        "    df.loc[\n",
        "        df.index.isin(monthly_max[monthly_max['accountid'].isin(consistent_accounts)].index),\n",
        "        'H3'\n",
        "    ] = 1\n",
        "\n",
        "    return df\n",
        "\n",
        "df = identify_consistent_high_credits(df, n_months=9, cv_threshold=0.15)\n",
        "\n",
        "# View flagged transactions\n",
        "salary_candidates = df[df['H3'] == 1]\n",
        "print(f\"Identified {len(salary_candidates)} salary-like transactions\")\n",
        "display(salary_candidates.sort_values(['accountid', 'trx_start_date']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 894
        },
        "id": "oGI8Esgez9p0",
        "outputId": "84cb5be9-4e25-4fcc-f742-eea291cea4bc"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Identified 468635 salary-like transactions\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "              id   accountid trx_start_date trx_end_date   amount d1  d2 d3  \\\n",
              "4396016   376992  0006538016     2024-03-15   2024-03-15  10300.0  T  CI  C   \n",
              "4778410   759398  0006538016     2024-04-11   2024-04-11  10600.0  T  CI  C   \n",
              "5160885  1141852  0006538016     2024-05-14   2024-05-14  10700.0  T  CI  C   \n",
              "189013   1524306  0006538016     2024-06-10   2024-06-10  11000.0  T  CI  C   \n",
              "571487   1906760  0006538016     2024-07-04   2024-07-04  10700.0  T  CI  C   \n",
              "...          ...         ...            ...          ...      ... ..  .. ..   \n",
              "1913542  3248825  2277851016     2024-11-01   2024-11-01   8800.0  T  CI  C   \n",
              "2678451  4013733  2277851016     2024-12-29   2024-12-29   8000.0  T  CI  C   \n",
              "3060877  4396187  2277851016     2025-01-24   2025-01-24   8480.0  T  CI  C   \n",
              "3443339  4778641  2277851016     2025-02-22   2025-02-22   8400.0  T  CI  C   \n",
              "3825785  5161095  2277851016     2025-03-22   2025-03-22   8720.0  T  CI  C   \n",
              "\n",
              "                                            description       d4  year  \\\n",
              "4396016                                  Online : wayde  0006538  2024   \n",
              "4778410                                  Online : wayde  0006538  2024   \n",
              "5160885                                  Online : wayde  0006538  2024   \n",
              "189013                                   Online : wayde  0006538  2024   \n",
              "571487                                   Online : wayde  0006538  2024   \n",
              "...                                                 ...      ...   ...   \n",
              "1913542  NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA  2277851  2024   \n",
              "2678451  NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA  2277851  2024   \n",
              "3060877  NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA  2277851  2025   \n",
              "3443339  NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA  2277851  2025   \n",
              "3825785  NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA  2277851  2025   \n",
              "\n",
              "           month  day_of_month  H1  H3  \n",
              "4396016  2024-03            15   0   1  \n",
              "4778410  2024-04            11   0   1  \n",
              "5160885  2024-05            14   0   1  \n",
              "189013   2024-06            10   0   1  \n",
              "571487   2024-07             4   0   1  \n",
              "...          ...           ...  ..  ..  \n",
              "1913542  2024-11             1   0   1  \n",
              "2678451  2024-12            29   0   1  \n",
              "3060877  2025-01            24   0   1  \n",
              "3443339  2025-02            22   0   1  \n",
              "3825785  2025-03            22   0   1  \n",
              "\n",
              "[468635 rows x 15 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6259e123-67f8-416c-b49a-b6de7f4086b4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>accountid</th>\n",
              "      <th>trx_start_date</th>\n",
              "      <th>trx_end_date</th>\n",
              "      <th>amount</th>\n",
              "      <th>d1</th>\n",
              "      <th>d2</th>\n",
              "      <th>d3</th>\n",
              "      <th>description</th>\n",
              "      <th>d4</th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day_of_month</th>\n",
              "      <th>H1</th>\n",
              "      <th>H3</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4396016</th>\n",
              "      <td>376992</td>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-03-15</td>\n",
              "      <td>2024-03-15</td>\n",
              "      <td>10300.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>Online : wayde</td>\n",
              "      <td>0006538</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-03</td>\n",
              "      <td>15</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4778410</th>\n",
              "      <td>759398</td>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-04-11</td>\n",
              "      <td>2024-04-11</td>\n",
              "      <td>10600.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>Online : wayde</td>\n",
              "      <td>0006538</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-04</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5160885</th>\n",
              "      <td>1141852</td>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-05-14</td>\n",
              "      <td>2024-05-14</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>Online : wayde</td>\n",
              "      <td>0006538</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-05</td>\n",
              "      <td>14</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189013</th>\n",
              "      <td>1524306</td>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>2024-06-10</td>\n",
              "      <td>11000.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>Online : wayde</td>\n",
              "      <td>0006538</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-06</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>571487</th>\n",
              "      <td>1906760</td>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-07-04</td>\n",
              "      <td>2024-07-04</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>Online : wayde</td>\n",
              "      <td>0006538</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-07</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1913542</th>\n",
              "      <td>3248825</td>\n",
              "      <td>2277851016</td>\n",
              "      <td>2024-11-01</td>\n",
              "      <td>2024-11-01</td>\n",
              "      <td>8800.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA</td>\n",
              "      <td>2277851</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-11</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2678451</th>\n",
              "      <td>4013733</td>\n",
              "      <td>2277851016</td>\n",
              "      <td>2024-12-29</td>\n",
              "      <td>2024-12-29</td>\n",
              "      <td>8000.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA</td>\n",
              "      <td>2277851</td>\n",
              "      <td>2024</td>\n",
              "      <td>2024-12</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3060877</th>\n",
              "      <td>4396187</td>\n",
              "      <td>2277851016</td>\n",
              "      <td>2025-01-24</td>\n",
              "      <td>2025-01-24</td>\n",
              "      <td>8480.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA</td>\n",
              "      <td>2277851</td>\n",
              "      <td>2025</td>\n",
              "      <td>2025-01</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3443339</th>\n",
              "      <td>4778641</td>\n",
              "      <td>2277851016</td>\n",
              "      <td>2025-02-22</td>\n",
              "      <td>2025-02-22</td>\n",
              "      <td>8400.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA</td>\n",
              "      <td>2277851</td>\n",
              "      <td>2025</td>\n",
              "      <td>2025-02</td>\n",
              "      <td>22</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3825785</th>\n",
              "      <td>5161095</td>\n",
              "      <td>2277851016</td>\n",
              "      <td>2025-03-22</td>\n",
              "      <td>2025-03-22</td>\n",
              "      <td>8720.0</td>\n",
              "      <td>T</td>\n",
              "      <td>CI</td>\n",
              "      <td>C</td>\n",
              "      <td>NIP FRM ANYAPA MUHAMMED YAKUBU-FT For self  RA</td>\n",
              "      <td>2277851</td>\n",
              "      <td>2025</td>\n",
              "      <td>2025-03</td>\n",
              "      <td>22</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>468635 rows × 15 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6259e123-67f8-416c-b49a-b6de7f4086b4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6259e123-67f8-416c-b49a-b6de7f4086b4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6259e123-67f8-416c-b49a-b6de7f4086b4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3bb2a9f4-9a4b-48d0-b64d-fcd32db46d5f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3bb2a9f4-9a4b-48d0-b64d-fcd32db46d5f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3bb2a9f4-9a4b-48d0-b64d-fcd32db46d5f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Evaluate H3\n",
        "Evaluate Efficiency of H3 by seeing how many percent of the known salary transactions it captures.\n",
        "\n",
        "\n",
        "**Result**: H3 captures **78%** of known salary transactions, this effectiveness is satisfactory for identifying probable salaries."
      ],
      "metadata": {
        "id": "qBVAe0md6SFM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# check percentage of transactions that passed H1 failed H3\n",
        "df[((df[\"H1\"]==1) & (df[\"H3\"]==1))].shape[0] / df[df[\"H1\"]==1].shape[0] * 100"
      ],
      "metadata": {
        "id": "UatRQ66GQEfm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60f55bf0-20d4-441a-cf2b-b4cb0bf7269e"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "78.28760643330179"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Salary Earning Customers"
      ],
      "metadata": {
        "id": "KTWRsARKuvp6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# define function to extract salary table from transactions table\n",
        "def get_salary_earner_summary(df, min_salary_months=9, col=\"H1\"):\n",
        "    \"\"\"\n",
        "    Returns a monthly aggregated table of salary amounts for confirmed salary earners\n",
        "    Parameters:\n",
        "    - min_salary_months: Minimum number of months with salary transactions (default 9)\n",
        "    - col: Column name for salary indicator (default \"H1\")\n",
        "    Returns:\n",
        "    - DataFrame with columns: [accountid, year_month, salary_max, month_seq, day_of_month]\n",
        "    \"\"\"\n",
        "    # 1. Extract month period and filter salary transactions\n",
        "    df['year_month'] = pd.to_datetime(df['trx_start_date']).dt.to_period('M')\n",
        "    # Extract day of month from transaction date\n",
        "    df['day_of_month'] = pd.to_datetime(df['trx_start_date']).dt.day\n",
        "\n",
        "    salary_txns = df[df[col] == 1].copy()\n",
        "\n",
        "    # 2. Identify salary earners (accounts with salary in >= N months)\n",
        "    salary_months = salary_txns.groupby('accountid')['year_month'].nunique()\n",
        "    salary_earners = salary_months[salary_months >= min_salary_months].index\n",
        "\n",
        "    # 3. Get transactions only for salary earners\n",
        "    salary_earner_txns = salary_txns[salary_txns['accountid'].isin(salary_earners)]\n",
        "\n",
        "    # 4. For each account and month, find the transaction with the maximum amount\n",
        "    # This assumes the largest transaction is the main salary\n",
        "    idx_max_amount = salary_earner_txns.groupby(['accountid', 'year_month'])['amount'].idxmax()\n",
        "\n",
        "    # 5. Create summary with the day of month information\n",
        "    max_txns = salary_earner_txns.loc[idx_max_amount]\n",
        "\n",
        "    # 6. Aggregate and create the summary dataframe\n",
        "    summary = (\n",
        "        max_txns\n",
        "        .groupby(['accountid', 'year_month'])\n",
        "        .agg(\n",
        "            salary_max=('amount', 'max'),\n",
        "            day_of_month=('day_of_month', 'first')  # Get day of month from the max transaction\n",
        "        )\n",
        "        .reset_index()\n",
        "    )\n",
        "\n",
        "    # 7. Add month sequence for trend analysis\n",
        "    summary['month_seq'] = (\n",
        "        summary.groupby('accountid')['year_month']\n",
        "        .rank(method='dense').astype(int)\n",
        "    )\n",
        "\n",
        "    return summary.sort_values(['accountid', 'year_month'])\n",
        "\n",
        "# Generate the summary for verified salary transactions (using H1)\n",
        "verified_salary_earner_summary = get_salary_earner_summary(df, min_salary_months=9)\n",
        "\n",
        "# Display results\n",
        "print(f\"Found {verified_salary_earner_summary['accountid'].nunique()} verified salary earners\")\n",
        "display(verified_salary_earner_summary.head(20))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 694
        },
        "id": "PWVY8QLqu4ud",
        "outputId": "0e617d94-8955-4ff9-867b-f4a4ccbedf03"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found 290 verified salary earners\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "     accountid year_month  salary_max  day_of_month  month_seq\n",
              "0   0532207024    2024-03     59950.0            23          1\n",
              "1   0532207024    2024-04     58850.0            16          2\n",
              "2   0532207024    2024-05     57200.0            15          3\n",
              "3   0532207024    2024-06     56650.0            16          4\n",
              "4   0532207024    2024-07     56650.0             9          5\n",
              "5   0532207024    2024-08     60500.0            10          6\n",
              "6   0532207024    2024-09     57750.0             6          7\n",
              "7   0532207024    2024-10     59950.0            30          8\n",
              "8   0532207024    2024-11     56100.0            26          9\n",
              "9   0532207024    2024-12     58300.0            24         10\n",
              "10  0532207024    2025-01     59950.0            22         11\n",
              "11  0532207024    2025-02     57750.0            20         12\n",
              "12  0532207024    2025-03     56100.0            22         13\n",
              "13  0532452020    2024-03     83200.0            21          1\n",
              "14  0532452020    2024-04     82400.0            16          2\n",
              "15  0532452020    2024-05     81600.0            14          3\n",
              "16  0532452020    2024-06     84000.0            12          4\n",
              "17  0532452020    2024-07     83200.0            12          5\n",
              "18  0532452020    2024-08     86400.0            11          6\n",
              "19  0532452020    2024-09     80000.0             7          7"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fc803e8e-5d6c-4522-9b92-1b4f075e7988\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accountid</th>\n",
              "      <th>year_month</th>\n",
              "      <th>salary_max</th>\n",
              "      <th>day_of_month</th>\n",
              "      <th>month_seq</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-03</td>\n",
              "      <td>59950.0</td>\n",
              "      <td>23</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-04</td>\n",
              "      <td>58850.0</td>\n",
              "      <td>16</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-05</td>\n",
              "      <td>57200.0</td>\n",
              "      <td>15</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-06</td>\n",
              "      <td>56650.0</td>\n",
              "      <td>16</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-07</td>\n",
              "      <td>56650.0</td>\n",
              "      <td>9</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-08</td>\n",
              "      <td>60500.0</td>\n",
              "      <td>10</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-09</td>\n",
              "      <td>57750.0</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-10</td>\n",
              "      <td>59950.0</td>\n",
              "      <td>30</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-11</td>\n",
              "      <td>56100.0</td>\n",
              "      <td>26</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2024-12</td>\n",
              "      <td>58300.0</td>\n",
              "      <td>24</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2025-01</td>\n",
              "      <td>59950.0</td>\n",
              "      <td>22</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2025-02</td>\n",
              "      <td>57750.0</td>\n",
              "      <td>20</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>0532207024</td>\n",
              "      <td>2025-03</td>\n",
              "      <td>56100.0</td>\n",
              "      <td>22</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-03</td>\n",
              "      <td>83200.0</td>\n",
              "      <td>21</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-04</td>\n",
              "      <td>82400.0</td>\n",
              "      <td>16</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-05</td>\n",
              "      <td>81600.0</td>\n",
              "      <td>14</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-06</td>\n",
              "      <td>84000.0</td>\n",
              "      <td>12</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-07</td>\n",
              "      <td>83200.0</td>\n",
              "      <td>12</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-08</td>\n",
              "      <td>86400.0</td>\n",
              "      <td>11</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>0532452020</td>\n",
              "      <td>2024-09</td>\n",
              "      <td>80000.0</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fc803e8e-5d6c-4522-9b92-1b4f075e7988')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fc803e8e-5d6c-4522-9b92-1b4f075e7988 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fc803e8e-5d6c-4522-9b92-1b4f075e7988');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6bd48f70-5543-4483-9d07-21b41b4f7074\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6bd48f70-5543-4483-9d07-21b41b4f7074')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6bd48f70-5543-4483-9d07-21b41b4f7074 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"display(verified_salary_earner_summary\",\n  \"rows\": 20,\n  \"fields\": [\n    {\n      \"column\": \"accountid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"0532452020\",\n          \"0532207024\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"year_month\",\n      \"properties\": {\n        \"dtype\": \"period[M]\",\n        \"num_unique_values\": 13,\n        \"samples\": [\n          \"2025-02\",\n          \"2024-12\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"salary_max\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 12272.021023450048,\n        \"min\": 56100.0,\n        \"max\": 86400.0,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          82400.0,\n          84000.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"day_of_month\",\n      \"properties\": {\n        \"dtype\": \"int32\",\n        \"num_unique_values\": 16,\n        \"samples\": [\n          23,\n          16\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"month_seq\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3,\n        \"min\": 1,\n        \"max\": 13,\n        \"num_unique_values\": 13,\n        \"samples\": [\n          12,\n          10\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# extract for probable salary earners\n",
        "\n",
        "# exclude already verified salary eaners\n",
        "unverified_df = df[~df[\"accountid\"].isin(verified_salary_earner_summary[\"accountid\"].unique())]\n",
        "\n",
        "probable_salary_earner_summary = get_salary_earner_summary(unverified_df, min_salary_months=9,  col = \"H3\")\n",
        "\n",
        "# Display results\n",
        "print(f\"Found {probable_salary_earner_summary['accountid'].nunique()} probable salary earners\")\n",
        "display(probable_salary_earner_summary.head(20))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 694
        },
        "id": "XjlW_AQnn4Uh",
        "outputId": "bb4041d0-e28a-494f-8a4b-8d656760ee02"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found 35867 probable salary earners\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "     accountid year_month  salary_max  day_of_month  month_seq\n",
              "0   0006538016    2024-03     10300.0            15          1\n",
              "1   0006538016    2024-04     10600.0            11          2\n",
              "2   0006538016    2024-05     10700.0            14          3\n",
              "3   0006538016    2024-06     11000.0            10          4\n",
              "4   0006538016    2024-07     10700.0             4          5\n",
              "5   0006538016    2024-08     10700.0             4          6\n",
              "6   0006538016    2024-09     10700.0             2          7\n",
              "7   0006538016    2024-10     10700.0            28          8\n",
              "8   0006538016    2024-11     10400.0            21          9\n",
              "9   0006538016    2024-12     10000.0            19         10\n",
              "10  0006538016    2025-01     10100.0            18         11\n",
              "11  0006538016    2025-02     10200.0            18         12\n",
              "12  0006538016    2025-03     10900.0            14         13\n",
              "13  0010954040    2024-03     74900.0            25          1\n",
              "14  0010954040    2024-04     74200.0            18          2\n",
              "15  0010954040    2024-05     72100.0            16          3\n",
              "16  0010954040    2024-06     72800.0            14          4\n",
              "17  0010954040    2024-07     74200.0            10          5\n",
              "18  0010954040    2024-08     72100.0            12          6\n",
              "19  0010954040    2024-09     74900.0             6          7"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-853b8380-a698-4dd2-8d5c-1704ee9201c4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accountid</th>\n",
              "      <th>year_month</th>\n",
              "      <th>salary_max</th>\n",
              "      <th>day_of_month</th>\n",
              "      <th>month_seq</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-03</td>\n",
              "      <td>10300.0</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-04</td>\n",
              "      <td>10600.0</td>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-05</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>14</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-06</td>\n",
              "      <td>11000.0</td>\n",
              "      <td>10</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-07</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-08</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-09</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-10</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>28</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-11</td>\n",
              "      <td>10400.0</td>\n",
              "      <td>21</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2024-12</td>\n",
              "      <td>10000.0</td>\n",
              "      <td>19</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2025-01</td>\n",
              "      <td>10100.0</td>\n",
              "      <td>18</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2025-02</td>\n",
              "      <td>10200.0</td>\n",
              "      <td>18</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>2025-03</td>\n",
              "      <td>10900.0</td>\n",
              "      <td>14</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-03</td>\n",
              "      <td>74900.0</td>\n",
              "      <td>25</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-04</td>\n",
              "      <td>74200.0</td>\n",
              "      <td>18</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-05</td>\n",
              "      <td>72100.0</td>\n",
              "      <td>16</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-06</td>\n",
              "      <td>72800.0</td>\n",
              "      <td>14</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-07</td>\n",
              "      <td>74200.0</td>\n",
              "      <td>10</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-08</td>\n",
              "      <td>72100.0</td>\n",
              "      <td>12</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>2024-09</td>\n",
              "      <td>74900.0</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-853b8380-a698-4dd2-8d5c-1704ee9201c4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-853b8380-a698-4dd2-8d5c-1704ee9201c4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-853b8380-a698-4dd2-8d5c-1704ee9201c4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9ea4bedb-0025-456e-bc2c-422b311874d2\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9ea4bedb-0025-456e-bc2c-422b311874d2')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9ea4bedb-0025-456e-bc2c-422b311874d2 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"display(probable_salary_earner_summary\",\n  \"rows\": 20,\n  \"fields\": [\n    {\n      \"column\": \"accountid\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"0010954040\",\n          \"0006538016\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"year_month\",\n      \"properties\": {\n        \"dtype\": \"period[M]\",\n        \"num_unique_values\": 13,\n        \"samples\": [\n          \"2025-02\",\n          \"2024-12\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"salary_max\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 30868.68452348636,\n        \"min\": 10000.0,\n        \"max\": 74900.0,\n        \"num_unique_values\": 13,\n        \"samples\": [\n          72100.0,\n          74900.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"day_of_month\",\n      \"properties\": {\n        \"dtype\": \"int32\",\n        \"num_unique_values\": 14,\n        \"samples\": [\n          18,\n          16\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"month_seq\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3,\n        \"min\": 1,\n        \"max\": 13,\n        \"num_unique_values\": 13,\n        \"samples\": [\n          12,\n          10\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# write dfs to csv\n",
        "verified_salary_earner_summary.to_csv(\"verified_salary_earner_summary.csv\", index=False)\n",
        "probable_salary_earner_summary.to_csv(\"probable_salary_earner_summary.csv\", index=False)"
      ],
      "metadata": {
        "id": "x-ciNaS4-vHD"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# now combining the two dataframes\n",
        "salary_earner_summary = pd.concat([verified_salary_earner_summary, probable_salary_earner_summary], ignore_index=True)"
      ],
      "metadata": {
        "id": "joXqE4OHoaVu"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Monthly Salary Averaging and Estimation"
      ],
      "metadata": {
        "id": "21JXE0ja_p8F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Estimating next monthly salary amount and date for each user\n",
        "def predict_next_salary(summary_table):\n",
        "    \"\"\"\n",
        "    Creates a table with salary predictions for each account based on the summary table,\n",
        "    using median of last 6 months for simplicity.\n",
        "\n",
        "    Parameters:\n",
        "    - summary_table: DataFrame from get_salary_earner_summary function\n",
        "\n",
        "    Returns:\n",
        "    - DataFrame with columns: [accountid, avg_monthly_salary, next_salary_amount, next_salary_date]\n",
        "    \"\"\"\n",
        "\n",
        "\n",
        "    # Group by accountid to get account-level statistics\n",
        "    account_predictions = []\n",
        "\n",
        "    for account_id, account_data in summary_table.groupby('accountid'):\n",
        "        # Sort data by year_month in descending order\n",
        "        account_data = account_data.sort_values('year_month', ascending=False)\n",
        "\n",
        "        # Calculate overall median salary (all available months)\n",
        "        median_salary_all = account_data['salary_max'].median()\n",
        "\n",
        "        # Get last 6 months of data or all if less than 6 months available\n",
        "        last_six_months = account_data.head(min(6, len(account_data)))\n",
        "\n",
        "        # Calculate median salary and day of month from last 6 months\n",
        "        median_salary_recent = last_six_months['salary_max'].median()\n",
        "        median_day = int(last_six_months['day_of_month'].median())\n",
        "\n",
        "        # Get the last known payment information\n",
        "        last_payment = account_data.iloc[0]\n",
        "\n",
        "        # Convert Period to timestamp (get first day of month)\n",
        "        # This is safer than direct conversion which might raise errors\n",
        "        last_month_str = str(last_payment['year_month'])\n",
        "        last_month = pd.to_datetime(last_month_str)\n",
        "\n",
        "        # Predict next month\n",
        "        if last_month.month == 12:\n",
        "            next_month_year = last_month.year + 1\n",
        "            next_month_month = 1\n",
        "        else:\n",
        "            next_month_year = last_month.year\n",
        "            next_month_month = last_month.month + 1\n",
        "\n",
        "        # Get the number of days in the next month\n",
        "        next_month_days = calendar.monthrange(next_month_year, next_month_month)[1]\n",
        "\n",
        "        # Adjust day if necessary\n",
        "        adjusted_day = min(median_day, next_month_days)\n",
        "\n",
        "        # Create the predicted date\n",
        "        predicted_date = pd.Timestamp(year=next_month_year,\n",
        "                                     month=next_month_month,\n",
        "                                     day=adjusted_day)\n",
        "\n",
        "        # Append to results\n",
        "        account_predictions.append({\n",
        "            'accountid': account_id,\n",
        "            'avg_monthly_salary': round(median_salary_all, 2),\n",
        "            'next_salary_amount': round(median_salary_recent, 2),\n",
        "            'next_salary_date': predicted_date.strftime('%Y-%m-%d')\n",
        "        })\n",
        "\n",
        "    # Convert to DataFrame\n",
        "    prediction_table = pd.DataFrame(account_predictions)\n",
        "\n",
        "    return prediction_table\n",
        "\n",
        "prediction_table = predict_next_salary(salary_earner_summary)\n",
        "prediction_table.head(20)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 676
        },
        "id": "-xlLGZYGvO2w",
        "outputId": "772724f7-4ac5-46e5-db3d-95185580d3c5"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     accountid  avg_monthly_salary  next_salary_amount next_salary_date\n",
              "0   0006538016             10700.0             10300.0       2025-04-18\n",
              "1   0010954040             74200.0             74200.0       2025-04-21\n",
              "2   0020801031            160500.0            162000.0       2025-04-23\n",
              "3   0022324028             41200.0             41200.0       2025-04-23\n",
              "4   0022848049           1463760.0           1477440.0       2025-04-22\n",
              "5   0025339021            586360.0            589100.0       2025-04-23\n",
              "6   0027643012            106000.0            108000.0       2025-04-23\n",
              "7   0028024021            515000.0            517500.0       2025-04-20\n",
              "8   0034199027            140450.0            141775.0       2025-04-19\n",
              "9   0035106020              2180.0              2190.0       2025-04-20\n",
              "10  0046472024             14700.0             14560.0       2025-04-25\n",
              "11  0050282020             15750.0             15900.0       2025-04-20\n",
              "12  0051390023            159000.0            161250.0       2025-04-20\n",
              "13  0051987023             37100.0             37100.0       2025-04-18\n",
              "14  0053388019            145080.0            144382.5       2025-04-17\n",
              "15  0060225026            156000.0            159750.0       2025-04-25\n",
              "16  0060968040               208.0               214.0       2025-04-21\n",
              "17  0068203024            155040.0            158840.0       2025-04-22\n",
              "18  0103261028             95680.0             95220.0       2025-04-23\n",
              "19  0104926023             42800.0             42000.0       2025-04-12"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f0cf1c43-153d-4351-a1f5-f4aefe9f8170\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accountid</th>\n",
              "      <th>avg_monthly_salary</th>\n",
              "      <th>next_salary_amount</th>\n",
              "      <th>next_salary_date</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0006538016</td>\n",
              "      <td>10700.0</td>\n",
              "      <td>10300.0</td>\n",
              "      <td>2025-04-18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0010954040</td>\n",
              "      <td>74200.0</td>\n",
              "      <td>74200.0</td>\n",
              "      <td>2025-04-21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0020801031</td>\n",
              "      <td>160500.0</td>\n",
              "      <td>162000.0</td>\n",
              "      <td>2025-04-23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0022324028</td>\n",
              "      <td>41200.0</td>\n",
              "      <td>41200.0</td>\n",
              "      <td>2025-04-23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0022848049</td>\n",
              "      <td>1463760.0</td>\n",
              "      <td>1477440.0</td>\n",
              "      <td>2025-04-22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0025339021</td>\n",
              "      <td>586360.0</td>\n",
              "      <td>589100.0</td>\n",
              "      <td>2025-04-23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0027643012</td>\n",
              "      <td>106000.0</td>\n",
              "      <td>108000.0</td>\n",
              "      <td>2025-04-23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0028024021</td>\n",
              "      <td>515000.0</td>\n",
              "      <td>517500.0</td>\n",
              "      <td>2025-04-20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>0034199027</td>\n",
              "      <td>140450.0</td>\n",
              "      <td>141775.0</td>\n",
              "      <td>2025-04-19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>0035106020</td>\n",
              "      <td>2180.0</td>\n",
              "      <td>2190.0</td>\n",
              "      <td>2025-04-20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0046472024</td>\n",
              "      <td>14700.0</td>\n",
              "      <td>14560.0</td>\n",
              "      <td>2025-04-25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0050282020</td>\n",
              "      <td>15750.0</td>\n",
              "      <td>15900.0</td>\n",
              "      <td>2025-04-20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>0051390023</td>\n",
              "      <td>159000.0</td>\n",
              "      <td>161250.0</td>\n",
              "      <td>2025-04-20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>0051987023</td>\n",
              "      <td>37100.0</td>\n",
              "      <td>37100.0</td>\n",
              "      <td>2025-04-18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>0053388019</td>\n",
              "      <td>145080.0</td>\n",
              "      <td>144382.5</td>\n",
              "      <td>2025-04-17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>0060225026</td>\n",
              "      <td>156000.0</td>\n",
              "      <td>159750.0</td>\n",
              "      <td>2025-04-25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>0060968040</td>\n",
              "      <td>208.0</td>\n",
              "      <td>214.0</td>\n",
              "      <td>2025-04-21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>0068203024</td>\n",
              "      <td>155040.0</td>\n",
              "      <td>158840.0</td>\n",
              "      <td>2025-04-22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>0103261028</td>\n",
              "      <td>95680.0</td>\n",
              "      <td>95220.0</td>\n",
              "      <td>2025-04-23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>0104926023</td>\n",
              "      <td>42800.0</td>\n",
              "      <td>42000.0</td>\n",
              "      <td>2025-04-12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f0cf1c43-153d-4351-a1f5-f4aefe9f8170')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f0cf1c43-153d-4351-a1f5-f4aefe9f8170 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f0cf1c43-153d-4351-a1f5-f4aefe9f8170');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-82cc4823-e85b-4765-aa16-6c6f4c166712\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-82cc4823-e85b-4765-aa16-6c6f4c166712')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-82cc4823-e85b-4765-aa16-6c6f4c166712 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "prediction_table",
              "summary": "{\n  \"name\": \"prediction_table\",\n  \"rows\": 36157,\n  \"fields\": [\n    {\n      \"column\": \"accountid\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 36157,\n        \"samples\": [\n          \"1539827015\",\n          \"2185975011\",\n          \"1980514012\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"avg_monthly_salary\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 633779.5101455783,\n        \"min\": 0.19,\n        \"max\": 61285000.0,\n        \"num_unique_values\": 6916,\n        \"samples\": [\n          20790.0,\n          293.08,\n          73736.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"next_salary_amount\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 642057.2400670127,\n        \"min\": 0.19,\n        \"max\": 63962500.0,\n        \"num_unique_values\": 8495,\n        \"samples\": [\n          40000.0,\n          10506.0,\n          79875.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"next_salary_date\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"num_unique_values\": 21,\n        \"samples\": [\n          \"2025-04-18\",\n          \"2025-04-27\",\n          \"2025-04-15\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# get number of customers with at least 10k predicted salary\n",
        "high_earning_customers = prediction_table[prediction_table[\"avg_monthly_salary\"] >= 10000]\n",
        "\n",
        "print(f\"Number of customers with at least 10k predicted salary: {high_earning_customers.shape[0]}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PaZLRoIVySB9",
        "outputId": "5f4fab25-047a-429d-d99c-d83580f291f5"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of customers with at least 10k predicted salary: 23978\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# write table to csv\n",
        "prediction_table.to_csv(\"salary_earner_prediction.csv\", index=False)"
      ],
      "metadata": {
        "id": "xE-vTMY9Atmq"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 4. ML Model Prediction"
      ],
      "metadata": {
        "id": "qSiu_T3X7PYr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "# Filter accounts with at least 12 months data\n",
        "valid_accounts = salary_earner_summary.groupby('accountid').filter(lambda x: x['month_seq'].max() >= 12)\n",
        "\n",
        "def create_safe_features(df):\n",
        "    df = df.copy().sort_values(['accountid', 'month_seq'])\n",
        "\n",
        "    # Lag features (safe - only uses past data)\n",
        "    df['lag_1'] = df.groupby('accountid')['salary_max'].shift(1)\n",
        "    df['lag_2'] = df.groupby('accountid')['salary_max'].shift(2)\n",
        "    df['lag_3'] = df.groupby('accountid')['salary_max'].shift(3)\n",
        "\n",
        "    # Expanding window statistics (safe - only uses past data)\n",
        "    df['expanding_mean'] = df.groupby('accountid')['salary_max'].expanding().mean().reset_index(0, drop=True)\n",
        "    df['expanding_std'] = df.groupby('accountid')['salary_max'].expanding().std().reset_index(0, drop=True)\n",
        "\n",
        "    # Month-over-month change (safe - using lag 1)\n",
        "    df['mom_change'] = df.groupby('accountid')['salary_max'].pct_change()\n",
        "\n",
        "    return df.dropna()\n",
        "\n",
        "safe_featured_data = create_safe_features(valid_accounts)\n",
        "\n",
        "# Split into train/test (first 10 vs last 2 months per account)\n",
        "train_data = safe_featured_data[safe_featured_data['month_seq'] <= 10]\n",
        "test_data = safe_featured_data[safe_featured_data['month_seq'] > 10]\n",
        "\n",
        "# Prepare features\n",
        "safe_features = ['lag_1', 'lag_2', 'lag_3', 'expanding_mean', 'expanding_std', 'mom_change']\n",
        "X_train = train_data[safe_features]\n",
        "y_train = train_data['salary_max']\n",
        "X_test = test_data[safe_features]\n",
        "y_test = test_data['salary_max']\n",
        "\n",
        "# Scale features\n",
        "scaler = StandardScaler()\n",
        "X_train_scaled = scaler.fit_transform(X_train)\n",
        "X_test_scaled = scaler.transform(X_test)"
      ],
      "metadata": {
        "id": "wh4SEaJx79W_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LinearRegression\n",
        "\n",
        "model = LinearRegression()\n",
        "model.fit(X_train_scaled, y_train)\n",
        "\n",
        "#Evaluation\n",
        "from sklearn.metrics import mean_absolute_error, r2_score\n",
        "\n",
        "# Make predictions\n",
        "test_preds = model.predict(X_test_scaled)\n",
        "\n",
        "# Calculate metrics\n",
        "test_mae = mean_absolute_error(y_test, test_preds)\n",
        "\n",
        "test_r2 = r2_score(y_test, test_preds)\n",
        "\n",
        "print(f\"Test MAE: {test_mae:.2f}\")\n",
        "print(f\"Test R²: {test_r2:.3f}\")\n",
        "\n",
        "# Create results dataframe\n",
        "results = test_data[['accountid', 'month_seq', 'salary_max']].copy()\n",
        "results['predicted'] = test_preds"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-MOnchDEEJWd",
        "outputId": "b5f85d12-f818-4405-8a00-501f380ba9ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test MAE: 3537.68\n",
            "Test R²: 0.996\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "# Randomly sample 100 points for clearer visualization\n",
        "sample_results = results.sample(100)\n",
        "\n",
        "plt.figure(figsize=(8, 8))\n",
        "sns.scatterplot(data=sample_results, x='salary_max', y='predicted', alpha=0.7)\n",
        "\n",
        "# Add perfect prediction line\n",
        "max_val = max(sample_results['salary_max'].max(), sample_results['predicted'].max())\n",
        "min_val = min(sample_results['salary_max'].min(), sample_results['predicted'].min())\n",
        "plt.plot([min_val, max_val], [min_val, max_val], 'r--', label='Perfect Prediction')\n",
        "\n",
        "plt.xlabel(\"Actual Salary Amount\", fontsize=12)\n",
        "plt.ylabel(\"Predicted Salary Amount\", fontsize=12)\n",
        "plt.title(f\"Actual vs Predicted Salaries\\n(Random Sample of 100 Points)\", fontsize=14)\n",
        "plt.legend()\n",
        "plt.grid(True, alpha=0.3)\n",
        "\n",
        "# Add RMSE annotation\n",
        "plt.annotate(f'Test MAE: {test_mae:.2f}',\n",
        "             xy=(0.05, 0.95),\n",
        "             xycoords='axes fraction',\n",
        "             fontsize=12,\n",
        "             bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 745
        },
        "id": "h1A0_QYrHhHa",
        "outputId": "7b241810-fa35-4b5f-a489-708a3a12c797"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 800x800 with 1 Axes>"
            ],
            "image/png": "[encoded data, 63928 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Pushing to Git from Colab"
      ],
      "metadata": {
        "id": "7ijJTmw2EkYI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Pushing code\n",
        "# First, install git and set up credentials\n",
        "!apt-get install git -y\n",
        "!git config --global user.name \"makindekayode75\"\n",
        "!git config --global user.email \"makindekayode75@gmail.com\""
      ],
      "metadata": {
        "id": "Ko2zG1JKJfgJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8862a5fd-0a68-478b-9ca2-2412cdf0ce18"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "git is already the newest version (1:2.34.1-1ubuntu1.12).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 29 not upgraded.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Clone the repository (replace with your credentials)\n",
        "!git clone https://makindekayode75:MgK1nDe.F7f22!x@gitlab.chiefsoft.net/DigiFi/digifi-Anayltics.git\n",
        "\n",
        "# Navigate to the repository\n",
        "%cd digifi-Anayltics/"
      ],
      "metadata": {
        "id": "_ZiLmYIlENx8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Copy your files from Colab to the local repository\n",
        "\n",
        "# Add, commit and push the files\n",
        "!git add .\n",
        "!git commit -m \"First commit from Colab\"\n",
        "!git push origin master"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RGKVthOWDX75",
        "outputId": "1e94e2c4-b8b3-4dc1-9838-fea6908fce5e"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[master (root-commit) d7de6ed] First commit from Colab\n",
            " 3 files changed, 504787 insertions(+)\n",
            " create mode 100644 probable_salary_earner_summary.csv\n",
            " create mode 100644 salary_earner_prediction.csv\n",
            " create mode 100644 verified_salary_earner_summary.csv\n",
            "Enumerating objects: 5, done.\n",
            "Counting objects: 100% (5/5), done.\n",
            "Delta compression using up to 2 threads\n",
            "Compressing objects: 100% (5/5), done.\n",
            "Writing objects: 100% (5/5), 3.56 MiB | 1.37 MiB/s, done.\n",
            "Total 5 (delta 0), reused 0 (delta 0), pack-reused 0\n",
            "To https://gitlab.chiefsoft.net/DigiFi/digifi-Anayltics.git\n",
            " * [new branch]      master -> master\n"
          ]
        }
      ]
    }
  ]
}