first commit

.dockerignore

@@ -0,0 +1,45 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Output
+output/
+
+# Logs
+*.log
+
+# Local development
+.env
+.env.local 

.env.example

@@ -0,0 +1,6 @@
+# Database Configuration
+DB_USER=your_username
+DB_PASSWORD=your_password
+DB_NAME=your_database
+DB_PORT=your_port
+DB_HOST=your_host 

.gitignore

@@ -0,0 +1,17 @@
+transaction.csv
+output/csv/final_table.csv
+output/csv/high_earner_details.csv
+output/csv/likely_salary_earner.csv
+output/plots/consistent_earners_predictions.png
+output/plots/hypothesis_overlap.png
+output/plots/inconsistent_earners_predictions.png
+output/models/consistent_model.joblib
+output/models/inconsistent_model.joblib
+output/models/consistent_scaler.joblib
+output/models/inconsistent_scaler.joblib
+.env
+__pycache__
+__pycache__/*
+output/*
+.idea/*
+salary_analytics/__pycache__/*

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y libpq-dev && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY salary_analytics/ ./salary_analytics/
+
+RUN mkdir -p output/csv output/plots output/models
+
+ENV PYTHONPATH=/app
+ENV HOST=0.0.0.0
+ENV PORT=8000
+
+EXPOSE 8000
+
+CMD ["uvicorn", "salary_analytics.api:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] 

README.md

@@ -0,0 +1,201 @@
+# Salary Analytics
+
+A comprehensive salary analytics system that analyzes transaction data to identify salary earners, predict future salaries, and generate detailed reports.
+
+## Features
+
+- **Transaction Analysis**
+  - Keyword-based salary transaction identification
+  - Consistent amount transaction analysis
+  - Transaction type analysis
+  - Hypothesis overlap visualization
+
+- **Salary Earner Classification**
+  - Verified salary earners identification
+  - Likely salary earners identification
+  - High earner detection
+  - Salary pattern analysis
+
+- **Machine Learning**
+  - Salary prediction models
+  - Separate models for consistent and inconsistent earners
+  - Feature engineering
+  - Model evaluation metrics
+  - Model persistence (saved in output/models)
+
+- **Reporting**
+  - CSV reports generation
+  - Visualization plots
+  - High earner details
+  - Salary earner statistics
+
+## Architecture
+
+The project is organized into the following modules:
+
+```
+salary_analytics/
+├── __init__.py
+├── config.py           # Configuration settings
+├── data_loader.py      # Database connection and data loading
+├── keyword_analyzer.py # Keyword-based analysis
+├── consistent_amount_analyzer.py # Consistent amount analysis
+├── transaction_type_analyzer.py  # Transaction type analysis
+├── salary_earner_analyzer.py     # Salary earner analysis
+├── salary_predictor.py # Machine learning models
+├── main.py            # Main pipeline
+└── api.py             # FastAPI endpoints
+```
+
+## Configuration
+
+The system can be configured through environment variables using a `.env` file:
+
+1. Copy the example environment file:
+```bash
+cp .env.example .env
+```
+
+2. Edit the `.env` file with your database credentials:
+```bash
+DB_USER=your_username
+DB_PASSWORD=your_password
+DB_NAME=your_database
+DB_PORT=your_port
+DB_HOST=your_host
+```
+
+## Usage
+
+### Using the API
+
+1. Start the API server:
+```bash
+uvicorn salary_analytics.api:app --reload
+```
+
+2. Access the API documentation:
+- Swagger UI: http://localhost:8000/docs
+- ReDoc: http://localhost:8000/redoc
+
+### API Endpoints
+
+1. **Basic Endpoints**
+   - `GET /`: Welcome message
+   - `GET /health`: Health check
+
+2. **Data Loading**
+   - `POST /load-data`: Load transaction data
+     - Parameters:
+       - `source`: Data source ('db' or 'csv')
+       - `file`: CSV file (required if source is 'csv')
+     - Example:
+       ```bash
+       # Load from database
+       curl -X POST "http://localhost:8000/load-data?source=db"
+       
+       # Load from CSV
+       curl -X POST "http://localhost:8000/load-data?source=csv" -F "file=@path/to/your/file.csv"
+       ```
+
+3. **Analysis Endpoints**
+   - `POST /analyze/keyword`: Run keyword analysis
+   - `POST /analyze/consistent-amount`: Run consistent amount analysis
+   - `POST /analyze/transaction-type`: Run transaction type analysis
+
+4. **Report Generation**
+   - `POST /generate/reports`: Generate all reports
+   - `GET /download/{report_type}`: Download specific reports
+     - Available types:
+       - `high_earners`: High earner details
+       - `likely_earners`: Likely salary earners
+       - `final_table`: Final analysis table
+       - `consistent_plot`: Consistent earners plot
+       - `inconsistent_plot`: Inconsistent earners plot
+       - `hypothesis_plot`: Hypothesis overlap plot
+
+5. **Model Training**
+   - `POST /train/models`: Train prediction models
+
+6. **Pipeline**
+   - `POST /run/pipeline`: Run complete pipeline
+   - `POST /run/streaming-pipeline`: Run pipeline in batches
+     - Parameters:
+       - `source`: Data source ('db' or 'csv')
+       - `file`: CSV file (required if source is 'csv')
+       - `batch_size`: Number of rows to process in each batch (default: 10000)
+     - Example:
+       ```bash
+       # Run streaming pipeline from database
+       curl -X POST "http://localhost:8000/run/streaming-pipeline?source=db&batch_size=5000"
+       
+       # Run streaming pipeline from CSV
+       curl -X POST "http://localhost:8000/run/streaming-pipeline?source=csv&batch_size=5000" -F "file=@path/to/your/file.csv"
+       ```
+     - Response:
+       ```json
+       [
+         {
+           "batch_number": 1,
+           "total_batches": 10,
+           "processed_rows": 5000,
+           "results_path": "/app/output/csv/batch_results_20240315_123456/batch_1_results.csv",
+           "message": "Successfully processed batch 1 of 10"
+         },
+         // ... more batch responses ...
+       ]
+       ```
+
+### Workflow
+
+1. Start the API server
+2. Load data using the `/load-data` endpoint
+3. Run any of the analysis endpoints
+4. Generate and download reports as needed
+
+For large datasets, use the streaming pipeline endpoint:
+1. Start the API server
+2. Run the streaming pipeline with appropriate batch size
+3. Monitor batch processing progress
+4. Access results in the batch results directory
+
+Note: All analysis endpoints require data to be loaded first. If you try to run any analysis without loading data, you'll receive a 400 error with a message to load data first.
+
+## Docker Deployment
+
+1. Build the Docker image:
+```bash
+docker-compose build
+```
+
+2. Run the container with environment variables:
+```bash
+docker run -v $(pwd)/output:/app/output \
+           -e DB_USER=your_username \
+           -e DB_PASSWORD=your_password \
+           -e DB_NAME=your_database \
+           -e DB_PORT=your_port \
+           -e DB_HOST=your_host \
+           salary-analytics
+```
+
+The API will be available at http://localhost:8000
+
+## Output Structure
+
+```
+output/
+├── csv/
+│   ├── high_earner_details.csv
+│   ├── likely_salary_earner.csv
+│   └── final_table.csv
+├── plots/
+│   ├── consistent_earners_predictions.png
+│   ├── inconsistent_earners_predictions.png
+│   └── hypothesis_overlap.png
+└── models/
+    ├── consistent_model.joblib
+    ├── inconsistent_model.joblib
+    ├── consistent_scaler.joblib
+    └── inconsistent_scaler.joblib
+```

docker-compose.yml

@@ -0,0 +1,20 @@
+services:
+  digifi-analytics:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./output:/app/output
+    environment:
+      - DB_USER=salaryloan
+      - DB_PASSWORD=salaryloan
+      - DB_NAME=salaryloan
+      - DB_PORT=10532
+      - DB_HOST=dev-data.simbrellang.net
+    restart: unless-stopped
+    networks:
+      - salary_network
+
+networks:
+  salary_network:
+    driver: bridge

final_table.csv

@@ -0,0 +1,384 @@
+accountid,num_months,least_inflow_6m,avg_monthly_salary,estimated_next_amount,estimated_next_date,45daysalary,2monthssalary
+0171115023,14,4251.1458,4402.972435714286,4402.972435714286,2025-04-12,True,False
+0194052022,14,26589.45,27382.33928571429,27382.33928571429,2025-04-14,True,False
+0532452020,14,80000.0,83714.28571428571,83714.28571428571,2025-04-15,True,False
+0623122023,14,4100.3778,4297.567114285715,4297.567114285715,2025-04-16,True,False
+0676572026,14,42323.61,44590.94625,44590.94625,2025-04-15,True,False
+0761118025,14,10209.51,10727.278007142857,10727.278007142857,2025-04-09,True,False
+0820472015,14,202325.60880000002,210543.31560000003,210543.31560000003,2025-04-18,True,True
+0835181025,14,42886.261600000005,43769.9071,43769.9071,2025-04-10,True,False
+0835200018,14,40857.6708,42116.59062857144,42116.59062857144,2025-04-15,True,False
+0846827013,14,379.44,388.2085714285715,388.2085714285715,2025-04-11,True,False
+0852762012,14,20648.37,21385.811785714286,21385.811785714286,2025-04-12,True,False
+0853195013,14,9297.402399999999,9476.1986,9476.1986,2025-04-16,True,False
+0853687015,14,15977.2405,16463.111321428576,16463.111321428576,2025-04-17,True,False
+0856304027,14,77548.12,80927.00237142856,80927.00237142856,2025-04-16,True,False
+0875566019,14,5871.2424,6048.037514285715,6048.037514285715,2025-04-15,True,False
+0889240019,14,1150220.1988,1189265.8632571427,1189265.8632571427,2025-04-22,True,False
+0890507013,14,22330.07,23542.273800000003,23542.273800000003,2025-04-14,True,False
+0897678011,14,55141.2663,56632.604292857155,56632.604292857155,2025-04-19,True,False
+0903403013,14,67017.0855,69244.66896428574,69244.66896428574,2025-04-13,True,False
+0903407011,14,46434.82,49585.75421428571,49585.75421428571,2025-04-13,True,False
+0908634012,14,39312.76,41474.961800000005,41474.961800000005,2025-04-14,True,False
+0909149016,14,40457.1054,41572.96615714285,41572.96615714285,2025-04-17,True,False
+0910196014,14,40457.1054,41830.47248571429,41830.47248571429,2025-04-15,True,False
+0915538013,14,31310.0,32129.285714285714,32129.285714285714,2025-04-15,True,False
+0926111012,14,12120.0,12642.857142857143,12642.857142857143,2025-04-20,True,True
+0943467011,14,9421.1076,9678.406757142857,9678.406757142857,2025-04-09,True,False
+0950731013,14,24131.3997,23712.56365714286,23712.56365714286,2025-04-12,True,False
+0954497016,14,48274.38220000001,48046.672849999995,48046.672849999995,2025-04-12,True,False
+0961463013,14,45890.81,47628.104950000015,47628.104950000015,2025-04-20,True,False
+0966021014,14,41000.0,42786.42857142857,42786.42857142857,2025-04-17,True,False
+0977809014,14,12341.82,12800.230457142856,12800.230457142856,2025-04-11,True,False
+0983934012,14,22941.6304,23445.84205714286,23445.84205714286,2025-04-17,True,False
+0984548012,14,12621.74,13225.780414285715,13225.780414285715,2025-04-12,True,False
+1007718012,14,22418.0,23378.771428571432,23378.771428571432,2025-04-19,True,False
+1012475016,14,15642.5776,15900.422285714287,15900.422285714287,2025-04-14,True,False
+1024520010,14,20607.4643,21583.91416428571,21583.91416428571,2025-04-14,True,False
+1026391014,14,31812.27,32903.86750000001,32903.86750000001,2025-04-13,True,False
+1026407016,14,31305.5257,32545.348500000004,32545.348500000004,2025-04-10,True,False
+1048739025,14,22949.7957,23907.389721428575,23907.389721428575,2025-04-15,True,False
+1049872017,14,21443.806800000002,22044.473657142862,22044.473657142862,2025-04-20,True,False
+1050211014,14,48780.1,51706.90600000001,51706.90600000001,2025-04-17,True,False
+1050617018,14,30535.01,32149.003385714284,32149.003385714284,2025-04-18,True,False
+1050806027,14,12381.43,12965.125985714287,12965.125985714287,2025-04-13,True,False
+1055487016,14,50709.042400000006,52150.83902857143,52150.83902857143,2025-04-12,True,False
+1056604012,14,123586.78,128353.69865714286,128353.69865714286,2025-04-15,True,False
+1058148017,14,87361.58,91667.25787142858,91667.25787142858,2025-04-18,True,False
+1059568010,14,110597.92899999999,113413.71785714284,113413.71785714284,2025-04-20,True,False
+1062072018,14,15979.257000000001,16120.57015714286,16120.57015714286,2025-04-12,True,False
+1062596015,14,70018.654,72494.56114285714,72494.56114285714,2025-04-16,True,False
+1068775012,14,4652.6256,4840.178400000002,4840.178400000002,2025-04-14,True,False
+1071766010,14,11815.6264,12049.5992,12049.5992,2025-04-13,True,False
+1073657011,14,51075.2152,52736.78514285714,52736.78514285714,2025-04-11,True,False
+1074328022,14,23416.5174,24121.636621428577,24121.636621428577,2025-04-11,True,False
+1077617011,14,4960.04,5200.956228571428,5200.956228571428,2025-04-15,True,False
+1078726011,14,8161.355500000001,8478.805678571429,8478.805678571429,2025-04-16,True,False
+1078919011,14,19392.19,20334.09637142857,20334.09637142857,2025-04-12,True,False
+1084193010,14,7566.707200000001,7691.433142857144,7691.433142857144,2025-04-15,True,False
+1084222017,14,7658.759300000001,7945.827364285715,7945.827364285715,2025-04-11,True,False
+1090993013,14,7739.27,8098.593250000001,8098.593250000001,2025-04-14,True,False
+1092014011,14,14662.29,15343.039178571431,15343.039178571431,2025-04-18,True,False
+1092396010,14,93320.8589,96620.74782857145,96620.74782857145,2025-04-19,True,False
+1092753013,14,12456.44,12865.72302857143,12865.72302857143,2025-04-16,True,False
+1099421018,14,19668.6188,20364.114514285717,20364.114514285717,2025-04-18,True,False
+1099865016,14,23140.25,24743.538750000003,24743.538750000003,2025-04-20,True,False
+1100425013,14,23596.19,24961.398135714287,24961.398135714287,2025-04-16,True,False
+1101981019,14,15941.880400000002,16719.80808571429,16719.80808571429,2025-04-20,True,False
+1102352014,14,30702.0,31820.0,31820.0,2025-04-14,True,False
+1108793013,14,8138.6995,8364.460928571429,8364.460928571429,2025-04-15,True,False
+1110716019,14,7937.8936,8079.6417,8079.6417,2025-04-16,True,False
+1111025015,14,12812.961000000001,13356.651000000002,13356.651000000002,2025-04-17,True,False
+1112139010,14,6240.65,6543.767285714285,6543.767285714285,2025-04-16,True,False
+1112274016,14,9728.9159,10100.458657142857,10100.458657142857,2025-04-15,True,False
+1113984011,14,7638.5289999999995,7957.251214285714,7957.251214285714,2025-04-19,True,False
+1114046017,14,9856.06,10250.302399999999,10250.302399999999,2025-04-15,True,False
+1131035016,14,3831.085,3921.4157142857134,3921.4157142857134,2025-04-13,True,False
+1136038012,14,49953.71,52986.61382142857,52986.61382142857,2025-04-12,True,False
+1136840017,14,19030.68,19914.247285714286,19914.247285714286,2025-04-13,True,False
+1137979028,14,16475.6928,16810.283199999998,16810.283199999998,2025-04-10,True,False
+1144064018,14,34347.7368,34890.31780714286,34890.31780714286,2025-04-18,True,False
+1145715018,14,52231.6955,53709.25407142857,53709.25407142857,2025-04-20,True,False
+1148441013,14,71671.28,73616.64331428571,73616.64331428571,2025-04-17,True,False
+1149064013,14,13041.33,13730.657442857144,13730.657442857144,2025-04-12,True,False
+1153011012,14,66438.26,68953.42270000001,68953.42270000001,2025-04-15,True,False
+1155907018,14,78055.5169,80373.99760000002,80373.99760000002,2025-04-16,True,False
+1166621015,14,10458.3783,11131.442250000002,11131.442250000002,2025-04-15,True,False
+1166928013,14,117526.77,122899.42234285714,122899.42234285714,2025-04-13,True,False
+1175464018,14,21420.0,22185.0,22185.0,2025-04-16,True,False
+1175961012,14,21840.0,21945.0,21945.0,2025-04-13,True,False
+1181003016,14,53535.21,56785.56203571429,56785.56203571429,2025-04-19,True,False
+1181900012,14,348.45,362.4964285714286,362.4964285714286,2025-04-10,True,False
+1185534017,14,3883.652,4015.487428571429,4015.487428571429,2025-04-14,True,False
+1185747011,14,40857.6708,41973.53155714286,41973.53155714286,2025-04-18,True,False
+1190093026,14,28436.137000000002,29343.25371428571,29343.25371428571,2025-04-13,True,False
+1201866010,14,28981.34,30699.519442857145,30699.519442857145,2025-04-15,True,False
+1202335014,14,3358.0581,3472.051585714287,3472.051585714287,2025-04-15,True,False
+1214025019,14,11110.0,11526.42857142857,11526.42857142857,2025-04-17,True,False
+1214711015,14,40056.54,42259.64970000001,42259.64970000001,2025-04-18,True,False
+1216795013,14,3822.8008,3885.814,3885.814,2025-04-12,True,False
+1218765016,14,40056.54,41744.637042857146,41744.637042857146,2025-04-17,True,False
+1226527019,14,8968.5602,9198.682757142857,9198.682757142857,2025-04-19,True,False
+1226530017,14,11896.3153,12359.043264285716,12359.043264285716,2025-04-14,True,False
+1228194026,14,21420.0,22440.0,22440.0,2025-04-18,True,False
+1243004012,14,12207.18,12922.17197142857,12922.17197142857,2025-04-15,True,False
+1243592016,14,21000.0,22110.0,22110.0,2025-04-17,True,False
+1246671019,14,4519.9362,4659.205942857144,4659.205942857144,2025-04-16,True,False
+1250257010,14,8017.65,8372.717357142858,8372.717357142858,2025-04-14,True,False
+1254162017,14,17290.380800000003,17103.961600000002,17103.961600000002,2025-04-14,True,False
+1256499010,14,10000.0,10528.57142857143,10528.57142857143,2025-04-17,True,False
+1259062019,14,27510.5322,28281.13534285715,28281.13534285715,2025-04-08,True,False
+1261523014,14,54300.4875,54891.51321428572,54891.51321428572,2025-04-14,True,False
+1263325016,14,28431.6616,29557.667999999998,29557.667999999998,2025-04-13,True,False
+1270270019,14,40408.5546,42134.690335714295,42134.690335714295,2025-04-13,True,False
+1271085012,14,37000.0,38876.42857142857,38876.42857142857,2025-04-15,True,False
+1275044015,14,131415.45309999998,137549.41342857142,137549.41342857142,2025-04-16,True,False
+1279191010,14,37072.41,38604.11321428571,38604.11321428571,2025-04-18,True,False
+1285665028,14,27586.08,28551.592800000002,28551.592800000002,2025-04-20,True,False
+1291708012,14,113682.43,118473.33240714285,118473.33240714285,2025-04-20,True,False
+1291809012,14,30300.0,31864.285714285714,31864.285714285714,2025-04-14,True,False
+1291885010,14,14137.3122,14751.117071428578,14751.117071428578,2025-04-16,True,False
+1298114029,14,27598.57,29037.638292857147,29037.638292857147,2025-04-12,True,False
+1298415018,14,132727.26,139648.03855714286,139648.03855714286,2025-04-16,True,False
+1298420010,14,179163.11,189528.97564999998,189528.97564999998,2025-04-20,True,False
+1311857010,14,21944.381999999998,22743.477142857144,22743.477142857144,2025-04-15,True,False
+1313574018,28,10000.0,10489.285714285714,10489.285714285714,2025-03-28,True,True
+1317200010,14,8335.11,8728.0509,8728.0509,2025-04-17,True,False
+1331055016,14,102000.0,106000.0,106000.0,2025-04-14,True,False
+1333165010,14,21000.0,22125.0,22125.0,2025-04-17,True,False
+1342374016,14,45215.080200000004,46449.9458357143,46449.9458357143,2025-04-13,True,False
+1342804012,14,21290.5374,22103.613085714285,22103.613085714285,2025-04-17,True,False
+1343673022,14,21000.0,21795.0,21795.0,2025-04-19,True,False
+1345196019,14,34137.85,36088.58428571429,36088.58428571429,2025-04-19,True,False
+1350668019,14,5473.5637,5663.241649999999,5663.241649999999,2025-04-13,True,False
+1355277012,14,92034.91619999999,94226.22372857144,94226.22372857144,2025-04-17,True,False
+1356399027,14,40457.1054,42116.59062857144,42116.59062857144,2025-04-15,True,False
+1364716018,14,40457.1054,42145.202442857146,42145.202442857146,2025-04-15,True,False
+1365385015,14,8548.4438,8744.073928571426,8744.073928571426,2025-04-11,True,False
+1365921013,14,113682.43,118067.32372857141,118067.32372857141,2025-04-17,True,False
+1368292019,14,26473.8675,27353.83339285714,27353.83339285714,2025-04-20,True,False
+1375320015,14,64000.0,66925.71428571429,66925.71428571429,2025-04-18,True,False
+1377321016,14,67056.28,71079.65680000001,71079.65680000001,2025-04-17,True,False
+1383019015,14,40056.54,41744.63704285714,41744.63704285714,2025-04-15,True,False
+1384084016,14,11276.86,11913.197100000003,11913.197100000003,2025-04-16,True,False
+1395426014,14,52623.78,55217.380585714294,55217.380585714294,2025-04-19,True,False
+1396795012,14,37770.83,39929.16314285715,39929.16314285715,2025-04-16,True,False
+1399402010,14,13987.86,14597.331042857142,14597.331042857142,2025-04-14,True,False
+1408181013,14,9958.7111,10275.643207142859,10275.643207142859,2025-04-20,True,False
+1408676023,14,46018.778999999995,47146.69025,47146.69025,2025-04-16,True,False
+1410787014,14,25255.1775,25444.16182142857,25444.16182142857,2025-04-18,True,False
+1414502015,14,8270.08,8451.840000000002,8451.840000000002,2025-04-17,True,False
+1414536014,14,2224.02,2285.361428571429,2285.361428571429,2025-04-19,True,False
+1415176017,14,8031.52,8400.72,8400.72,2025-04-19,True,True
+1415211013,14,3929.3545,4084.9725,4084.9725,2025-04-20,True,False
+1417746018,14,4268.94,4485.436242857143,4485.436242857143,2025-04-09,True,False
+1428498012,14,14898.37,15558.154957142859,15558.154957142859,2025-04-14,True,False
+1436713013,14,33026.67,34583.641585714286,34583.641585714286,2025-04-13,True,False
+1443284010,14,22275.0,23516.035714285714,23516.035714285714,2025-04-14,True,False
+1443552014,14,22720.5,23372.839285714286,23372.839285714286,2025-04-16,True,False
+1444444015,14,23273.136,23859.853714285717,23859.853714285717,2025-04-19,True,False
+1452711015,14,8568.35,8923.3245,8923.3245,2025-04-17,True,False
+1457606015,14,6201.086899999999,6385.277599999999,6385.277599999999,2025-04-15,True,False
+1462318019,14,40857.6708,41801.86067142857,41801.86067142857,2025-04-18,True,False
+1465238011,14,15977.089,16519.451285714287,16519.451285714287,2025-04-12,True,False
+1465352016,14,23419.3326,24354.137892857147,24354.137892857147,2025-04-17,True,False
+1469561010,14,30370.065000000002,31317.813214285718,31317.813214285718,2025-04-16,True,False
+1472225019,14,6153.1155,6199.159221428571,6199.159221428571,2025-04-18,True,False
+1479990013,14,120868.53820000001,125484.45125714291,125484.45125714291,2025-04-20,True,False
+1486608013,14,14280.0,14710.0,14710.0,2025-04-15,True,False
+1501871011,14,10605.3,11090.113714285713,11090.113714285713,2025-04-18,True,False
+1511221015,14,134753.29,139180.89810000002,139180.89810000002,2025-04-13,True,False
+1511356014,14,4070.02,4314.221199999999,4314.221199999999,2025-04-18,True,False
+1512095013,14,3997.5432,4120.7168,4120.7168,2025-04-13,True,False
+1512253017,14,25946.4234,26691.383735714284,26691.383735714284,2025-04-10,True,False
+1512290014,14,76173.8251,77600.10337857142,77600.10337857142,2025-04-19,True,False
+1516029010,14,140222.5136,142630.18038571425,142630.18038571425,2025-04-15,True,False
+1522821017,14,45567.614499999996,47050.012142857144,47050.012142857144,2025-04-17,True,False
+1523440019,14,42496.2752,43818.64868571429,43818.64868571429,2025-04-17,True,False
+1530552019,14,9070.9918,9359.672585714286,9359.672585714286,2025-04-12,True,False
+1533377013,14,21840.0,22215.0,22215.0,2025-04-13,True,False
+1538119014,14,40056.54,41916.30792857143,41916.30792857143,2025-04-18,True,False
+1538899011,14,5923.854,6164.458714285714,6164.458714285714,2025-04-12,True,False
+1543039017,14,43414.55,45864.37103571429,45864.37103571429,2025-04-09,True,False
+1543041018,14,34915.95,36686.687464285715,36686.687464285715,2025-04-15,True,False
+1543044019,14,26606.08,27860.366628571435,27860.366628571435,2025-04-11,True,False
+1543345015,14,17942.2662,18437.137757142857,18437.137757142857,2025-04-15,True,False
+1543391012,14,13000.0,13631.42857142857,13631.42857142857,2025-04-22,True,False
+1547865012,14,57734.6278,57734.6278,57734.6278,2025-04-12,True,False
+1566054028,14,69690.3333,72499.63245,72499.63245,2025-04-15,True,False
+1567984016,14,67767.02519999999,70092.36429999999,70092.36429999999,2025-04-18,True,False
+1587630018,14,21000.0,22020.0,22020.0,2025-04-14,True,False
+1593487015,14,1380.0,1429.2857142857144,1429.2857142857144,2025-04-13,True,False
+1593684010,14,34093.7346,35096.491500000004,35096.491500000004,2025-04-19,True,False
+1602958019,14,6180.0,6342.857142857143,6342.857142857143,2025-04-18,True,False
+1609825013,14,33091.55,34864.31160714286,34864.31160714286,2025-04-12,True,False
+1613386012,14,11391.21,11903.81445,11903.81445,2025-04-11,True,False
+1616744013,14,6035.1439,6218.673735714286,6218.673735714286,2025-04-11,True,False
+1617111010,14,61800.0,63685.71428571428,63685.71428571428,2025-04-16,True,False
+1618434011,14,30000.0,31435.714285714286,31435.714285714286,2025-04-13,True,False
+1620472016,14,40056.54,42002.14337142858,42002.14337142858,2025-04-17,True,False
+1631034014,14,7227.3349,7427.815757142858,7427.815757142858,2025-04-15,True,False
+1635580014,14,50968.0946,53347.086285714286,53347.086285714286,2025-04-15,True,False
+1642340016,14,143009.0007,149886.37838571428,149886.37838571428,2025-04-13,True,False
+1642419011,14,66300.0,68758.92857142857,68758.92857142857,2025-04-18,True,False
+1643680010,14,318280.88,332830.8630857143,332830.8630857143,2025-04-19,True,False
+1648142016,14,25500.0,26321.428571428572,26321.428571428572,2025-04-14,True,False
+1650352010,14,44328.51,46861.56771428571,46861.56771428571,2025-04-15,True,False
+1664348010,14,46692.25,49093.565714285716,49093.565714285716,2025-04-17,True,False
+1668954015,14,30900.0,31757.14285714286,31757.14285714286,2025-04-15,True,False
+1671967011,14,42059.367000000006,42059.36700000001,42059.36700000001,2025-04-18,True,False
+1675953014,14,10775.38,11275.6655,11275.6655,2025-04-11,True,False
+1676417016,14,4306.02,4502.86662857143,4502.86662857143,2025-04-09,True,False
+1676704017,14,26252.26,27321.102014285716,27321.102014285716,2025-04-18,True,False
+1681835010,14,21000.0,21945.0,21945.0,2025-04-14,True,False
+1683968019,14,4534.4758,4739.713742857143,4739.713742857143,2025-04-15,True,False
+1684383017,14,22105.1832,23136.966857142856,23136.966857142856,2025-04-19,True,False
+1687874013,14,21210.0,22155.0,22155.0,2025-04-18,True,False
+1703943017,14,20200.0,21128.571428571428,21128.571428571428,2025-04-17,True,False
+1708145012,14,26019.916,26341.590785714292,26341.590785714292,2025-04-12,True,False
+1714166012,14,24532.1692,25144.622799999994,25144.622799999994,2025-04-19,True,False
+1732006017,14,28209.48,28693.84607142858,28693.84607142858,2025-04-15,True,False
+1732103019,14,14140.0,14600.0,14600.0,2025-04-14,True,False
+1734119012,14,7137.7458,7267.704757142857,7267.704757142857,2025-04-13,True,False
+1736676012,14,15000.0,15385.714285714286,15385.714285714286,2025-04-15,True,False
+1740753015,14,40056.54,41944.91974285715,41944.91974285715,2025-04-20,True,False
+1755401019,14,45450.0,46896.42857142857,46896.42857142857,2025-04-18,True,False
+1757447013,14,5287.37,5593.282121428572,5593.282121428572,2025-04-14,True,False
+1757605017,14,196316.2048,205895.99131428575,205895.99131428575,2025-04-11,True,False
+1759380013,14,3678.2628,3819.9325857142862,3819.9325857142862,2025-04-14,True,False
+1766882018,14,36501.42,38404.708328571425,38404.708328571425,2025-04-13,True,False
+1774623010,14,83294.7648,84610.54748571428,84610.54748571428,2025-04-15,True,False
+1776036014,14,363.19,380.5712357142857,380.5712357142857,2025-04-16,True,False
+1777475014,14,110687.73,116696.49248571429,116696.49248571429,2025-04-20,True,False
+1779276019,14,86907.5003,90656.69232142858,90656.69232142858,2025-04-17,True,False
+1785264017,14,110026.6326,114384.12299999998,114384.12299999998,2025-04-16,True,False
+1793393017,14,79427.03,83114.71353571431,83114.71353571431,2025-04-14,True,False
+1793495014,14,11193.92,11641.676800000001,11641.676800000001,2025-04-13,True,False
+1795523018,14,50000.0,52571.42857142857,52571.42857142857,2025-04-17,True,False
+1799104017,14,47500.0,49875.0,49875.0,2025-04-19,True,False
+1801542017,14,12504.129,12924.435857142857,12924.435857142857,2025-04-17,True,False
+1804659017,14,24533.601,25341.079464285714,25341.079464285714,2025-04-15,True,False
+1808809016,14,5002.1765,5207.357714285715,5207.357714285715,2025-04-15,True,False
+1814233016,14,28714.5828,29790.871971428565,29790.871971428565,2025-04-17,True,False
+1815571016,14,21000.0,21855.0,21855.0,2025-04-17,True,False
+1819235015,14,96150.0,100270.71428571429,100270.71428571429,2025-04-15,True,False
+1824974019,28,64750.0,67825.625,67825.625,2025-03-22,True,True
+1829881017,14,7864.28,8240.641971428571,8240.641971428571,2025-04-20,True,False
+1832273014,14,8880.394699999999,9244.654171428572,9244.654171428572,2025-04-15,True,False
+1832570012,14,10200.0,10307.142857142857,10307.142857142857,2025-04-19,True,False
+1833032010,14,32320.0,33165.71428571428,33165.71428571428,2025-04-16,True,False
+1839846011,14,61027.52820000001,62822.455500000025,62822.455500000025,2025-04-15,True,False
+1845307014,14,10100.0,10535.714285714286,10535.714285714286,2025-04-13,True,False
+1858230011,14,7780.03,8060.6392857142855,8060.6392857142855,2025-04-11,True,False
+1860928012,14,28840.0,29300.0,29300.0,2025-04-16,True,False
+1863257016,14,12260.0163,12667.527449999998,12667.527449999998,2025-04-14,True,False
+1867743014,14,2651.0480000000002,2806.6611428571427,2806.6611428571427,2025-04-19,True,False
+1874878019,14,21210.0,21870.0,21870.0,2025-04-14,True,False
+1890774014,14,6463.8182,6824.950899999999,6824.950899999999,2025-04-13,True,False
+1893248017,14,6961.23,7294.374578571428,7294.374578571428,2025-04-19,True,False
+1893502016,14,59180.3592,60962.40082857142,60962.40082857142,2025-04-15,True,False
+1904132012,14,27975.285,28641.363214285717,28641.363214285717,2025-04-10,True,False
+1905813013,14,26619.484800000002,27320.969142857135,27320.969142857135,2025-04-15,True,False
+1907631011,14,52231.8975,55002.33053571428,55002.33053571428,2025-04-17,True,False
+1908562013,14,24625.43,25891.880685714284,25891.880685714284,2025-04-11,True,False
+1908717019,14,30989.6604,31749.210900000005,31749.210900000005,2025-04-20,True,True
+1909045012,14,55935.69,58572.658242857135,58572.658242857135,2025-04-17,True,False
+1910312013,14,68931.05,72525.31189285716,72525.31189285716,2025-04-14,True,False
+1918274012,14,15952.640000000001,16395.15428571429,16395.15428571429,2025-04-20,True,False
+1924705014,14,101000.0,105214.28571428571,105214.28571428571,2025-04-19,True,False
+1931759013,14,68431.4078,70044.90839999999,70044.90839999999,2025-04-18,True,False
+1938793018,14,11589.75,12160.959107142859,12160.959107142859,2025-04-18,True,True
+1939714016,14,46692.25,49460.43339285714,49460.43339285714,2025-04-18,True,False
+1940684016,14,13218.0882,13838.264607142857,13838.264607142857,2025-04-10,True,False
+1944111019,14,23813.17,24816.72502142857,24816.72502142857,2025-04-11,True,False
+1944186013,14,20400.0,20885.714285714286,20885.714285714286,2025-04-17,True,False
+1944801013,14,8190.0,8505.9,8505.9,2025-04-13,True,False
+1946636017,14,42860.497800000005,42660.21510000001,42660.21510000001,2025-04-19,True,False
+1949225012,14,40056.54,41916.307928571434,41916.307928571434,2025-04-18,True,False
+1949359014,14,40857.6708,42402.70877142857,42402.70877142857,2025-04-15,True,False
+1951369012,14,3829.94,4013.2299857142857,4013.2299857142857,2025-04-15,True,False
+1954035015,14,51714.55,55075.99575000001,55075.99575000001,2025-04-18,True,False
+1955200012,14,18854.9931,19721.736064285717,19721.736064285717,2025-04-16,True,True
+1955976014,14,40857.6708,41716.02522857143,41716.02522857143,2025-04-15,True,False
+1957290019,14,21630.0,22200.0,22200.0,2025-04-15,True,False
+1957372012,14,28104.5,29429.426428571427,29429.426428571427,2025-04-17,True,False
+1957430013,14,28104.5,29128.30678571429,29128.30678571429,2025-04-16,True,False
+1957861013,14,56466.27,58644.2547,58644.2547,2025-04-21,True,False
+1959768013,14,5816.45,6115.581714285714,6115.581714285714,2025-04-11,True,False
+1972698016,14,34643.0,36480.5,36480.5,2025-04-18,True,False
+1972978011,14,5000.0,5282.142857142857,5282.142857142857,2025-04-12,True,False
+1974761013,14,45000.0,47282.142857142855,47282.142857142855,2025-04-19,True,False
+1985487014,14,57206.4,60078.857142857145,60078.857142857145,2025-04-15,True,False
+1987937018,14,41258.2362,42602.99147142858,42602.99147142858,2025-04-18,True,False
+1988801019,14,28804.31,29977.056907142858,29977.056907142858,2025-04-17,True,False
+1991880016,14,17159.613,17808.50592857143,17808.50592857143,2025-04-15,True,False
+1995835014,14,3518.4,3706.8857142857155,3706.8857142857155,2025-04-14,True,False
+1995942013,14,4740.48,4923.327085714285,4923.327085714285,2025-04-16,True,False
+1998110015,14,12953.25,13319.67857142857,13319.67857142857,2025-04-14,True,False
+1999144011,14,63053.17,66385.9804142857,66385.9804142857,2025-04-13,True,False
+2002086018,14,5707.4051,5845.934350000001,5845.934350000001,2025-04-12,True,False
+2002869019,14,4590.6,4833.246,4833.246,2025-04-15,True,False
+2002918010,14,4473.414,4633.178785714285,4633.178785714285,2025-04-14,True,False
+2002982017,14,3295.43,3476.6786499999994,3476.6786499999994,2025-04-15,True,False
+2006029015,14,3762.5126000000005,3900.8794000000007,3900.8794000000007,2025-04-15,True,False
+2010494016,14,40857.6708,41944.91974285715,41944.91974285715,2025-04-16,True,False
+2015047015,14,41258.2362,42087.97881428572,42087.97881428572,2025-04-17,True,False
+2015759017,14,47355.7,49926.438,49926.438,2025-04-20,True,False
+2020313017,14,5000.0,5253.571428571428,5253.571428571428,2025-04-11,True,False
+2034595018,14,153940.62,161417.73582857143,161417.73582857143,2025-04-19,True,False
+2039362015,14,100000.0,105071.42857142857,105071.42857142857,2025-04-18,True,False
+2043399010,14,41658.8016,42145.202442857146,42145.202442857146,2025-04-15,True,False
+2047786012,14,28150.93,29009.90428571428,29009.90428571428,2025-04-20,True,False
+2048850019,14,21000.0,21960.0,21960.0,2025-04-15,True,False
+2051114016,14,30357.600000000002,30816.299999999996,30816.299999999996,2025-04-16,True,False
+2055259010,14,22207.58,23127.608314285717,23127.608314285717,2025-04-15,True,False
+2059595017,14,13750.0,14683.035714285714,14683.035714285714,2025-04-12,True,False
+2059835014,14,66500.0,69920.0,69920.0,2025-04-20,True,False
+2062857010,14,20200.0,21057.14285714286,21057.14285714286,2025-04-20,True,True
+2066217012,14,57600.0,59739.42857142857,59739.42857142857,2025-04-16,True,False
+2076926027,14,30917.33,32220.274621428576,32220.274621428576,2025-04-10,True,False
+2077079016,14,44055.84,45786.60514285713,45786.60514285713,2025-04-15,True,False
+2083803018,14,39063.08,40374.48340000001,40374.48340000001,2025-04-16,True,False
+2084273016,14,15300.0,15975.0,15975.0,2025-04-17,True,False
+2085388018,14,300000.0,318214.28571428574,318214.28571428574,2025-04-16,True,True
+2088066012,14,33323.9804,34620.17482857143,34620.17482857143,2025-04-13,True,False
+2089870018,14,4242.0,4386.0,4386.0,2025-04-17,True,False
+2090266019,14,45982.0386,47237.85057857142,47237.85057857142,2025-04-15,True,False
+2090875017,14,49180.0462,49279.46677857143,49279.46677857143,2025-04-11,True,False
+2094484013,14,46363.3834,47970.035299999996,47970.035299999996,2025-04-17,True,False
+2095040012,14,81600.0,84000.0,84000.0,2025-04-18,True,False
+2101243011,14,12000.0,12677.142857142857,12677.142857142857,2025-04-16,True,False
+2101560013,14,12254.0573,12982.0211,12982.0211,2025-04-18,True,True
+2105827011,14,101000.0,105357.14285714286,105357.14285714286,2025-04-18,True,False
+2105903012,14,136350.0,142328.57142857142,142328.57142857142,2025-04-18,True,False
+2109081015,14,47626.095,49427.081785714276,49427.081785714276,2025-04-17,True,False
+2109496019,14,40457.1054,42288.26151428572,42288.26151428572,2025-04-21,True,False
+2113221014,14,66277.0701,68529.20354999999,68529.20354999999,2025-04-17,True,False
+2117066019,14,41658.8016,42173.814257142854,42173.814257142854,2025-04-19,True,False
+2117394015,14,4492.21,4752.116435714285,4752.116435714285,2025-04-09,True,False
+2119186010,14,57485.33,60564.901249999995,60564.901249999995,2025-04-16,True,False
+2122558013,14,8062.78,8385.2912,8385.2912,2025-04-13,True,False
+2123350018,14,37744.25,40143.70589285715,40143.70589285715,2025-04-11,True,False
+2124152013,14,41258.2362,42545.767842857145,42545.767842857145,2025-04-15,True,False
+2130118023,14,38787.71,41253.50013571428,41253.50013571428,2025-04-12,True,False
+2140539016,14,10819.872,10922.9184,10922.9184,2025-04-15,True,False
+2141435012,14,26000.0,26178.571428571428,26178.571428571428,2025-04-18,True,False
+2141748019,14,113300.0,115735.71428571429,115735.71428571429,2025-04-12,True,False
+2142999014,14,22500.0,23528.571428571428,23528.571428571428,2025-04-10,True,False
+2146179014,14,74375.087,76005.65821428572,76005.65821428572,2025-04-20,True,False
+2146523019,14,24240.0,25388.571428571428,25388.571428571428,2025-04-14,True,False
+2165506013,14,50762.0034,52788.217821428574,52788.217821428574,2025-04-16,True,False
+2165587016,14,5893.25,6137.398928571428,6137.398928571428,2025-04-17,True,False
+2168451011,14,40056.54,42173.81425714287,42173.81425714287,2025-04-19,True,False
+2168911018,14,34483.3594,36141.6822,36141.6822,2025-04-19,True,False
+2178112018,14,6801.7578,7082.782807142859,7082.782807142859,2025-04-13,True,False
+2180261010,14,41266.58,42930.08428571429,42930.08428571429,2025-04-20,True,False
+2180749017,14,18532.8,18761.914285714283,18761.914285714283,2025-04-19,True,False
+2182865010,14,38250.0,39750.0,39750.0,2025-04-17,True,False
+2197472016,14,25862.5145,27069.675714285717,27069.675714285717,2025-04-21,True,False
+2198899018,14,30641.945600000003,32202.214400000004,32202.214400000004,2025-04-18,True,False
+2214244019,14,50968.0946,52409.907742857155,52409.907742857155,2025-04-15,True,False
+2217047015,14,16160.0,16902.85714285714,16902.85714285714,2025-04-10,True,False
+2219928019,14,5452.0,5709.022857142858,5709.022857142858,2025-04-21,True,False
+2225726018,14,46800.0,47635.71428571428,47635.71428571428,2025-04-21,True,False
+2225954011,14,32640.0,33348.57142857143,33348.57142857143,2025-04-17,True,False
+2226559011,14,25500.0,26500.0,26500.0,2025-04-20,True,False
+2226613014,14,24239.99,25209.589600000003,25209.589600000003,2025-04-14,True,False
+2226717015,14,25750.0,26375.0,26375.0,2025-04-19,True,False
+2226808018,14,65233.75,68122.67321428572,68122.67321428572,2025-04-14,True,False
+2227088017,14,35000.0,36775.0,36775.0,2025-04-16,True,False
+2227138015,14,46800.0,47153.57142857143,47153.57142857143,2025-04-17,True,False
+2227213019,14,65233.75,67936.29107142858,67936.29107142858,2025-04-15,True,False
+2227418010,14,45900.0,47282.142857142855,47282.142857142855,2025-04-18,True,False
+2227600013,14,26520.0,27058.571428571428,27058.571428571428,2025-04-17,True,False
+2234193018,14,45000.0,46928.57142857143,46928.57142857143,2025-04-20,True,False
+2234892012,14,65886.09760000001,68821.6168,68821.6168,2025-04-14,True,False
+2242042016,14,4800.0,4964.571428571428,4964.571428571428,2025-04-14,True,False
+2242872011,14,30300.0,31542.85714285714,31542.85714285714,2025-04-21,True,False
+2244212015,14,35000.0,36775.0,36775.0,2025-04-17,True,False
+2253348011,14,25721.5286,26722.012385714283,26722.012385714283,2025-04-13,True,False
+2253502017,14,26780.0,27114.285714285714,27114.285714285714,2025-04-14,True,False
+2253954018,14,25391.834300000002,26594.983450000003,26594.983450000003,2025-04-17,True,False
+2267310019,14,21000.0,22245.0,22245.0,2025-04-13,True,False
+2271021019,14,52800.0,55779.42857142857,55779.42857142857,2025-04-17,True,False
+2272954015,14,7977.42,8329.365,8329.365,2025-04-19,True,False

high_earner_details.csv

@@ -0,0 +1,308 @@
+accountid,least_inflow_6m
+0194052022,26589.45
+0532452020,80000.0
+0676572026,42323.61
+0761118025,10209.51
+0820472015,202325.60880000002
+0835181025,42886.261600000005
+0835200018,40857.6708
+0852762012,20648.37
+0853687015,15977.2405
+0856304027,77548.12
+0889240019,1150220.1988
+0890507013,22330.07
+0897678011,55141.2663
+0903403013,67017.0855
+0903407011,46434.82
+0908634012,39312.76
+0909149016,40457.1054
+0910196014,40457.1054
+0915538013,31310.0
+0926111012,12120.0
+0950731013,24131.3997
+0954497016,48274.38220000001
+0961463013,45890.81
+0966021014,41000.0
+0977809014,12341.82
+0983934012,22941.6304
+0984548012,12621.74
+1007718012,22418.0
+1012475016,15642.5776
+1024520010,20607.4643
+1026391014,31812.27
+1026407016,31305.5257
+1048739025,22949.7957
+1049872017,21443.806800000002
+1050211014,48780.1
+1050617018,30535.01
+1050806027,12381.43
+1055487016,50709.042400000006
+1056604012,123586.78
+1058148017,87361.58
+1059568010,110597.92899999999
+1062072018,15979.257000000001
+1062596015,70018.654
+1071766010,11815.6264
+1073657011,51075.2152
+1074328022,23416.5174
+1078919011,19392.19
+1092014011,14662.29
+1092396010,93320.8589
+1092753013,12456.44
+1099421018,19668.6188
+1099865016,23140.25
+1100425013,23596.19
+1101981019,15941.880400000002
+1102352014,30702.0
+1111025015,12812.961000000001
+1112274016,9728.9159
+1114046017,9856.06
+1136038012,49953.71
+1136840017,19030.68
+1137979028,16475.6928
+1144064018,34347.7368
+1145715018,52231.6955
+1148441013,71671.28
+1149064013,13041.33
+1153011012,66438.26
+1155907018,78055.5169
+1166621015,10458.3783
+1166928013,117526.77
+1175464018,21420.0
+1175961012,21840.0
+1181003016,53535.21
+1185747011,40857.6708
+1190093026,28436.137000000002
+1201866010,28981.34
+1214025019,11110.0
+1214711015,40056.54
+1218765016,40056.54
+1226530017,11896.3153
+1228194026,21420.0
+1243004012,12207.18
+1243592016,21000.0
+1254162017,17290.380800000003
+1256499010,10000.0
+1259062019,27510.5322
+1261523014,54300.4875
+1263325016,28431.6616
+1270270019,40408.5546
+1271085012,37000.0
+1275044015,131415.45309999998
+1279191010,37072.41
+1285665028,27586.08
+1291708012,113682.43
+1291809012,30300.0
+1291885010,14137.3122
+1298114029,27598.57
+1298415018,132727.26
+1298420010,179163.11
+1311857010,21944.381999999998
+1313574018,10000.0
+1331055016,102000.0
+1333165010,21000.0
+1342374016,45215.080200000004
+1342804012,21290.5374
+1343673022,21000.0
+1345196019,34137.85
+1355277012,92034.91619999999
+1356399027,40457.1054
+1364716018,40457.1054
+1365921013,113682.43
+1368292019,26473.8675
+1375320015,64000.0
+1377321016,67056.28
+1383019015,40056.54
+1384084016,11276.86
+1395426014,52623.78
+1396795012,37770.83
+1399402010,13987.86
+1408181013,9958.7111
+1408676023,46018.778999999995
+1410787014,25255.1775
+1428498012,14898.37
+1436713013,33026.67
+1443284010,22275.0
+1443552014,22720.5
+1444444015,23273.136
+1462318019,40857.6708
+1465238011,15977.089
+1465352016,23419.3326
+1469561010,30370.065000000002
+1479990013,120868.53820000001
+1486608013,14280.0
+1501871011,10605.3
+1511221015,134753.29
+1512253017,25946.4234
+1512290014,76173.8251
+1516029010,140222.5136
+1522821017,45567.614499999996
+1523440019,42496.2752
+1533377013,21840.0
+1538119014,40056.54
+1543039017,43414.55
+1543041018,34915.95
+1543044019,26606.08
+1543345015,17942.2662
+1543391012,13000.0
+1547865012,57734.6278
+1566054028,69690.3333
+1567984016,67767.02519999999
+1587630018,21000.0
+1593684010,34093.7346
+1609825013,33091.55
+1613386012,11391.21
+1617111010,61800.0
+1618434011,30000.0
+1620472016,40056.54
+1635580014,50968.0946
+1642340016,143009.0007
+1642419011,66300.0
+1643680010,318280.88
+1648142016,25500.0
+1650352010,44328.51
+1664348010,46692.25
+1668954015,30900.0
+1671967011,42059.367000000006
+1675953014,10775.38
+1676704017,26252.26
+1681835010,21000.0
+1684383017,22105.1832
+1687874013,21210.0
+1703943017,20200.0
+1708145012,26019.916
+1714166012,24532.1692
+1732006017,28209.48
+1732103019,14140.0
+1736676012,15000.0
+1740753015,40056.54
+1755401019,45450.0
+1757605017,196316.2048
+1766882018,36501.42
+1774623010,83294.7648
+1777475014,110687.73
+1779276019,86907.5003
+1785264017,110026.6326
+1793393017,79427.03
+1793495014,11193.92
+1795523018,50000.0
+1799104017,47500.0
+1801542017,12504.129
+1804659017,24533.601
+1814233016,28714.5828
+1815571016,21000.0
+1819235015,96150.0
+1824974019,64750.0
+1832570012,10200.0
+1833032010,32320.0
+1839846011,61027.52820000001
+1845307014,10100.0
+1860928012,28840.0
+1863257016,12260.0163
+1874878019,21210.0
+1893502016,59180.3592
+1904132012,27975.285
+1905813013,26619.484800000002
+1907631011,52231.8975
+1908562013,24625.43
+1908717019,30989.6604
+1909045012,55935.69
+1910312013,68931.05
+1918274012,15952.640000000001
+1924705014,101000.0
+1931759013,68431.4078
+1938793018,11589.75
+1939714016,46692.25
+1940684016,13218.0882
+1944111019,23813.17
+1944186013,20400.0
+1946636017,42860.497800000005
+1949225012,40056.54
+1949359014,40857.6708
+1954035015,51714.55
+1955200012,18854.9931
+1955976014,40857.6708
+1957290019,21630.0
+1957372012,28104.5
+1957430013,28104.5
+1957861013,56466.27
+1972698016,34643.0
+1974761013,45000.0
+1985487014,57206.4
+1987937018,41258.2362
+1988801019,28804.31
+1991880016,17159.613
+1998110015,12953.25
+1999144011,63053.17
+2010494016,40857.6708
+2015047015,41258.2362
+2015759017,47355.7
+2034595018,153940.62
+2039362015,100000.0
+2043399010,41658.8016
+2047786012,28150.93
+2048850019,21000.0
+2051114016,30357.600000000002
+2055259010,22207.58
+2059595017,13750.0
+2059835014,66500.0
+2062857010,20200.0
+2066217012,57600.0
+2076926027,30917.33
+2077079016,44055.84
+2083803018,39063.08
+2084273016,15300.0
+2085388018,300000.0
+2088066012,33323.9804
+2090266019,45982.0386
+2090875017,49180.0462
+2094484013,46363.3834
+2095040012,81600.0
+2101243011,12000.0
+2101560013,12254.0573
+2105827011,101000.0
+2105903012,136350.0
+2109081015,47626.095
+2109496019,40457.1054
+2113221014,66277.0701
+2117066019,41658.8016
+2119186010,57485.33
+2123350018,37744.25
+2124152013,41258.2362
+2130118023,38787.71
+2140539016,10819.872
+2141435012,26000.0
+2141748019,113300.0
+2142999014,22500.0
+2146179014,74375.087
+2146523019,24240.0
+2165506013,50762.0034
+2168451011,40056.54
+2168911018,34483.3594
+2180261010,41266.58
+2180749017,18532.8
+2182865010,38250.0
+2197472016,25862.5145
+2198899018,30641.945600000003
+2214244019,50968.0946
+2217047015,16160.0
+2225726018,46800.0
+2225954011,32640.0
+2226559011,25500.0
+2226613014,24239.99
+2226717015,25750.0
+2226808018,65233.75
+2227088017,35000.0
+2227138015,46800.0
+2227213019,65233.75
+2227418010,45900.0
+2227600013,26520.0
+2234193018,45000.0
+2234892012,65886.09760000001
+2242872011,30300.0
+2244212015,35000.0
+2253348011,25721.5286
+2253502017,26780.0
+2253954018,25391.834300000002
+2267310019,21000.0
+2271021019,52800.0

requirements.txt

@@ -0,0 +1,15 @@
+sqlalchemy>=2.0.0
+pandas>=1.5.0
+numpy>=1.21.0
+matplotlib>=3.5.0
+seaborn>=0.12.0
+matplotlib-venn>=0.11.7
+wordcloud>=1.8.0
+scikit-learn>=1.0.0
+psycopg2-binary>=2.9.0
+fastapi>=0.68.0
+uvicorn>=0.15.0
+pydantic>=1.8.0
+python-multipart>=0.0.5 
+python-dotenv>=0.19.0
+joblib>=1.1.0

salary_analytics/__init__.py

@@ -0,0 +1,6 @@
+"""
+Salary Analytics Package
+A package for analyzing and predicting salary patterns from transaction data.
+"""
+
+__version__ = "0.1.0" 

salary_analytics/api.py

@@ -0,0 +1,598 @@
+"""
+FastAPI application for salary analytics.
+"""
+
+from fastapi import FastAPI, HTTPException, BackgroundTasks, UploadFile, File, Depends
+from fastapi.responses import FileResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import Optional, Dict, List, Union
+import os
+import socket
+import logging
+import pandas as pd
+import tempfile
+from datetime import datetime
+from sqlalchemy import text, Table, Column, Integer, String, Float, DateTime, MetaData
+import numpy as np
+import warnings
+import time
+from .main import SalaryAnalyticsPipeline
+from .config import OUTPUT_PATHS, TABLE_NAME, BATCH_RESULTS_TABLE
+from .data_loader import DataLoader
+from .salary_predictor import SalaryPredictor
+from .salary_earner_analyzer import SalaryEarnerAnalyzer
+from .db_operations import DatabaseOperations
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Suppress warnings
+warnings.filterwarnings('ignore', category=RuntimeWarning, module='numpy')
+pd.options.mode.chained_assignment = None
+
+app = FastAPI(
+    title="Salary Analytics API",
+    description="API for analyzing and predicting salary patterns from transaction data",
+    version="1.0.0"
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allows all origins
+    allow_credentials=True,
+    allow_methods=["*"],  # Allows all methods
+    allow_headers=["*"],  # Allows all headers
+)
+
+# Global pipeline instance
+pipeline = SalaryAnalyticsPipeline()
+
+# Global variables to store loaded data and models
+data_loader = None
+df = None
+salary_predictor = None
+salary_earner_analyzer = None
+
+class AnalysisResponse(BaseModel):
+    """Response model for analysis endpoints."""
+    message: str
+    data: Optional[Dict] = None
+    file_path: Optional[str] = None
+
+class BatchResponse(BaseModel):
+    """Response model for batch processing."""
+    batch_number: int
+    total_batches: int
+    processed_rows: int
+    results_path: str
+    message: str
+
+def check_data_loaded():
+    """Check if data is loaded before running analytics."""
+    if pipeline.df is None:
+        raise HTTPException(
+            status_code=400,
+            detail="No data loaded. Please load data first using the /load-data endpoint."
+        )
+
+@app.on_event("startup")
+async def startup_event():
+    """Initialize the pipeline on startup."""
+    try:
+        logger.info("Initializing pipeline...")
+        
+        # Print network information
+        hostname = socket.gethostname()
+        ip_address = socket.gethostbyname(hostname)
+        logger.info(f"Server running on hostname: {hostname}")
+        logger.info(f"Server IP address: {ip_address}")
+        logger.info(f"Server is accessible at:")
+        logger.info(f"- http://localhost:8000")
+        logger.info(f"- http://127.0.0.1:8000")
+        logger.info(f"- http://{ip_address}:8000")
+        logger.info("Pipeline initialized successfully")
+    except Exception as e:
+        logger.error(f"Error during startup: {str(e)}")
+        raise
+
+@app.get("/")
+async def root():
+    """Root endpoint."""
+    start_time = time.time()
+    logger.info("Root endpoint accessed")
+    response = {"message": "Welcome to Salary Analytics API"}
+    logger.info(f"Root endpoint completed in {time.time() - start_time:.2f} seconds")
+    return response
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint."""
+    start_time = time.time()
+    logger.info("Health check endpoint accessed")
+    response = {"status": "healthy"}
+    logger.info(f"Health check completed in {time.time() - start_time:.2f} seconds")
+    return response
+
+@app.post("/analyze/keyword", response_model=AnalysisResponse)
+async def analyze_keyword():
+    """Run keyword-based salary transaction analysis."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info("Starting keyword analysis...")
+        data = pipeline.run_keyword_analysis()
+        logger.info(f"Keyword analysis completed. Found {len(data)} matches")
+        response = AnalysisResponse(
+            message="Keyword analysis completed successfully",
+            data={"count": len(data)}
+        )
+        logger.info(f"Keyword analysis endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error in keyword analysis: {str(e)}")
+        logger.info(f"Keyword analysis endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/analyze/consistent-amount", response_model=AnalysisResponse)
+async def analyze_consistent_amount():
+    """Run consistent amount transaction analysis."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info("Starting consistent amount analysis...")
+        data = pipeline.run_consistent_amount_analysis()
+        logger.info(f"Consistent amount analysis completed. Found {len(data)} matches")
+        response = AnalysisResponse(
+            message="Consistent amount analysis completed successfully",
+            data={"count": len(data)}
+        )
+        logger.info(f"Consistent amount analysis endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error in consistent amount analysis: {str(e)}")
+        logger.info(f"Consistent amount analysis endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/analyze/transaction-type", response_model=AnalysisResponse)
+async def analyze_transaction_type():
+    """Run transaction type analysis."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info("Starting transaction type analysis...")
+        data = pipeline.run_transaction_type_analysis()
+        logger.info(f"Transaction type analysis completed. Found {len(data)} matches")
+        response = AnalysisResponse(
+            message="Transaction type analysis completed successfully",
+            data={"count": len(data)}
+        )
+        logger.info(f"Transaction type analysis endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error in transaction type analysis: {str(e)}")
+        logger.info(f"Transaction type analysis endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/generate/reports", response_model=AnalysisResponse)
+async def generate_reports(background_tasks: BackgroundTasks):
+    """Generate salary earner reports."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info("Starting report generation...")
+        reports = pipeline.generate_salary_earner_reports()
+        logger.info("Reports generated successfully")
+        response = AnalysisResponse(
+            message="Reports generated successfully",
+            data={
+                "verified_salary_earners": len(reports['final_table']),
+                "likely_salary_earners": len(reports['likely_salary_earner']),
+                "high_earners": reports['total_high_earners']
+            }
+        )
+        logger.info(f"Report generation endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error in report generation: {str(e)}")
+        logger.info(f"Report generation endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/train/models", response_model=AnalysisResponse)
+async def train_models():
+    """Train salary prediction models."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info("Starting model training...")
+        pipeline.train_salary_prediction_models()
+        logger.info("Models trained successfully")
+        response = AnalysisResponse(
+            message="Models trained successfully"
+        )
+        logger.info(f"Model training endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error in model training: {str(e)}")
+        logger.info(f"Model training endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/download/{report_type}")
+async def download_report(report_type: str):
+    """Download generated reports."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info(f"Attempting to download report: {report_type}")
+        file_paths = {
+            "high_earners": OUTPUT_PATHS["high_earner_details"],
+            "likely_earners": OUTPUT_PATHS["likely_salary_earner"],
+            "final_table": OUTPUT_PATHS["final_table"],
+            "consistent_plot": OUTPUT_PATHS["consistent_earners_plot"],
+            "inconsistent_plot": OUTPUT_PATHS["inconsistent_earners_plot"],
+            "hypothesis_plot": OUTPUT_PATHS["hypothesis_overlap_plot"]
+        }
+
+        if report_type not in file_paths:
+            logger.error(f"Report type not found: {report_type}")
+            logger.info(f"Download endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=404, detail="Report type not found")
+
+        file_path = file_paths[report_type]
+        if not os.path.exists(file_path):
+            logger.error(f"Report file not found: {file_path}")
+            logger.info(f"Download endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=404, detail="Report file not found")
+
+        logger.info(f"Successfully found report file: {file_path}")
+        response = FileResponse(
+            path=file_path,
+            filename=os.path.basename(file_path),
+            media_type="application/octet-stream"
+        )
+        logger.info(f"Download endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error downloading report: {str(e)}")
+        logger.info(f"Download endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/run/pipeline", response_model=AnalysisResponse)
+async def run_full_pipeline():
+    """Run the complete salary analytics pipeline."""
+    start_time = time.time()
+    try:
+        check_data_loaded()
+        logger.info("Starting full pipeline...")
+        success = pipeline.run_full_pipeline()
+        if not success:
+            logger.error("Pipeline failed")
+            logger.info(f"Full pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=500, detail="Pipeline failed")
+        
+        logger.info("Pipeline completed successfully")
+        response = AnalysisResponse(
+            message="Pipeline completed successfully"
+        )
+        logger.info(f"Full pipeline endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error in pipeline: {str(e)}")
+        logger.info(f"Full pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/load-data")
+async def load_data(source: str = "db", file: Optional[UploadFile] = File(None)):
+    """
+    Load data from either database or CSV file.
+    
+    Args:
+        source (str): Source of data ('db' or 'csv')
+        file (UploadFile, optional): CSV file to load (required if source is 'csv')
+    
+    Returns:
+        dict: Status of data loading
+    """
+    start_time = time.time()
+    try:
+        if source not in ['db', 'csv']:
+            logger.error(f"Invalid source: {source}")
+            logger.info(f"Load data endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=400, detail="Source must be either 'db' or 'csv'")
+        
+        if source == 'csv' and not file:
+            logger.error("No file provided for CSV source")
+            logger.info(f"Load data endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=400, detail="File must be provided when loading from CSV")
+        
+        if source == 'csv':
+            # Save uploaded file temporarily
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as temp_file:
+                content = await file.read()
+                temp_file.write(content)
+                temp_file_path = temp_file.name
+            
+            try:
+                success = pipeline.load_data(source='csv', file_path=temp_file_path)
+            finally:
+                # Clean up temporary file
+                os.unlink(temp_file_path)
+        else:
+            success = pipeline.load_data(source='db')
+        
+        if not success:
+            logger.error("Failed to load data")
+            logger.info(f"Load data endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=500, detail="Failed to load data")
+        
+        response = {
+            "status": "success",
+            "message": f"Successfully loaded {len(pipeline.df)} rows of data",
+            "columns": pipeline.df.columns.tolist(),
+            "row_count": len(pipeline.df)
+        }
+        logger.info(f"Load data endpoint completed in {time.time() - start_time:.2f} seconds")
+        return response
+    except Exception as e:
+        logger.error(f"Error loading data: {str(e)}")
+        logger.info(f"Load data endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e))
+
+async def get_file_if_csv(source: str, file: Optional[UploadFile] = File(None)):
+    """Dependency to handle file upload only when source is csv."""
+    if source == 'csv' and not file:
+        raise HTTPException(status_code=400, detail="File must be provided when loading from CSV")
+    return file
+
+@app.post("/run/streaming-pipeline", response_model=List[BatchResponse])
+async def run_streaming_pipeline(
+    source: str = "db",
+    batch_size: int = 10000,
+    file: Optional[Union[UploadFile, str]] = File(None)
+):
+    """
+    Run the complete salary analytics pipeline in batches.
+    
+    Args:
+        source (str): Source of data ('db' or 'csv')
+        batch_size (int): Number of rows to process in each batch
+        file (UploadFile, optional): CSV file to load (required if source is 'csv')
+    
+    Returns:
+        List[BatchResponse]: List of responses for each batch processed
+    """
+    start_time = time.time()
+    try:
+        if source not in ['db', 'csv']:
+            logger.error(f"Invalid source: {source}")
+            logger.info(f"Streaming pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=400, detail="Source must be either 'db' or 'csv'")
+        
+        if source == 'csv' and not file:
+            logger.error("No file provided for CSV source")
+            logger.info(f"Streaming pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=400, detail="File must be provided when loading from CSV")
+        
+        # Initialize data loader
+        data_loader = DataLoader()
+        data_loader.chunk_size = batch_size
+        
+        # Create output directory for batch results
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        batch_output_dir = os.path.join(os.path.dirname(OUTPUT_PATHS['final_table']), f"batch_results_{timestamp}")
+        os.makedirs(batch_output_dir, exist_ok=True)
+        
+        # Initialize database operations
+        if not data_loader.connect():
+            logger.error("Failed to connect to database")
+            logger.info(f"Streaming pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=500, detail="Failed to connect to database")
+        
+        db_ops = DatabaseOperations(data_loader.engine)
+        if not db_ops.create_batch_results_table():
+            logger.error("Failed to create batch results table")
+            logger.info(f"Streaming pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+            raise HTTPException(status_code=500, detail="Failed to create batch results table")
+        
+        responses = []
+        batch_number = 0
+        batch_start_time = time.time()
+
+        def preprocess_chunk(chunk):
+            """Preprocess a chunk of data with the same logic as DataLoader."""
+            # Convert dates
+            chunk['trx_start_date'] = pd.to_datetime(chunk['trx_start_date'])
+            chunk['trx_end_date'] = pd.to_datetime(chunk['trx_end_date'])
+            
+            # Rename columns
+            chunk = chunk.rename(columns={
+                'd1': 'trx_type',
+                'd2': 'trx_subtype',
+                'd3': 'initiated_by',
+                'd4': 'customer_id'
+            })
+            
+            chunk = chunk.dropna()
+            
+            return chunk
+        
+        if source == 'csv':
+            # Save uploaded file temporarily
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as temp_file:
+                content = await file.read()
+                temp_file.write(content)
+                temp_file_path = temp_file.name
+            
+            try:
+                # Process CSV in chunks
+                for chunk in pd.read_csv(temp_file_path, chunksize=batch_size):
+                    batch_number += 1
+                    logger.info(f"Processing batch {batch_number}")
+                    
+                    # Preprocess chunk
+                    chunk = preprocess_chunk(chunk)
+                    
+                    # Run pipeline on chunk
+                    pipeline = SalaryAnalyticsPipeline()
+                    pipeline.df = chunk
+                    
+                    try:
+                        batch_start_time = time.time()
+                        # Run analyses
+                        pipeline.run_keyword_analysis()
+                        pipeline.run_consistent_amount_analysis()
+                        pipeline.run_transaction_type_analysis()
+                        
+                        # Generate reports
+                        reports = pipeline.generate_salary_earner_reports()
+                        
+                        # Add batch metadata to results
+                        results_df = reports['final_table'].copy()
+                        results_df['batch_number'] = batch_number
+                        results_df['total_batches'] = -1  # Unknown for CSV
+                        results_df['processed_at'] = datetime.now()
+                        
+                        # Save batch results to CSV
+                        batch_results_path = os.path.join(batch_output_dir, f"batch_{batch_number}_results.csv")
+                        results_df.to_csv(batch_results_path, index=False)
+                        
+                        # Save to database
+                        db_ops.save_batch_to_db(
+                            batch_number=batch_number,
+                            total_batches=-1,  # Unknown for CSV
+                            results_df=results_df,
+                            status="success"
+                        )
+                        
+                        logger.info(f"Batch {batch_number} processed in {time.time() - batch_start_time:.2f} seconds")
+                        
+                        responses.append(BatchResponse(
+                            batch_number=batch_number,
+                            total_batches=-1,  # Unknown for CSV
+                            processed_rows=len(chunk),
+                            results_path=batch_results_path,
+                            message=f"Successfully processed batch {batch_number}"
+                        ))
+                    except Exception as e:
+                        error_message = str(e)
+                        logger.error(f"Error processing batch {batch_number}: {error_message}")
+                        
+                        # Save error to database
+                        db_ops.save_batch_to_db(
+                            batch_number=batch_number,
+                            total_batches=-1,
+                            results_df=pd.DataFrame(),  # Empty DataFrame for error case
+                            status="error"
+                        )
+                        
+                        responses.append(BatchResponse(
+                            batch_number=batch_number,
+                            total_batches=-1,
+                            processed_rows=len(chunk),
+                            results_path="",
+                            message=f"Error processing batch {batch_number}: {error_message}"
+                        ))
+            finally:
+                # Clean up temporary file
+                os.unlink(temp_file_path)
+        else:
+            # Process database in chunks
+            if not data_loader.connect():
+                raise HTTPException(status_code=500, detail="Failed to connect to database")
+            
+            # Get total row count
+            with data_loader.engine.connect() as conn:
+                count_query = text(f"SELECT COUNT(*) FROM {TABLE_NAME}")
+                total_rows = conn.execute(count_query).scalar()
+            
+            total_batches = (total_rows + batch_size - 1) // batch_size
+            offset = 0
+            
+            while offset < total_rows:
+                batch_number += 1
+                logger.info(f"Processing batch {batch_number} of {total_batches}")
+                
+                # Load chunk from database
+                query = f"SELECT * FROM {TABLE_NAME} LIMIT {batch_size} OFFSET {offset}"
+                chunk = pd.read_sql(query, data_loader.engine)
+                
+                if chunk.empty:
+                    break
+                
+                # Preprocess chunk
+                chunk = preprocess_chunk(chunk)
+                
+                # Run pipeline on chunk
+                pipeline = SalaryAnalyticsPipeline()
+                pipeline.df = chunk
+                
+                try:
+                    batch_start_time = time.time()
+                    # Run analyses
+                    pipeline.run_keyword_analysis()
+                    pipeline.run_consistent_amount_analysis()
+                    pipeline.run_transaction_type_analysis()
+                    
+                    # Generate reports
+                    reports = pipeline.generate_salary_earner_reports()
+                    
+                    # Add batch metadata to results
+                    results_df = reports['final_table'].copy()
+                    results_df['batch_number'] = batch_number
+                    results_df['total_batches'] = total_batches
+                    results_df['processed_at'] = datetime.now()
+                    
+                    # Save batch results to CSV
+                    batch_results_path = os.path.join(batch_output_dir, f"batch_{batch_number}_results.csv")
+                    results_df.to_csv(batch_results_path, index=False)
+                    
+                    # Save to database
+                    db_ops.save_batch_to_db(
+                        batch_number=batch_number,
+                        total_batches=total_batches,
+                        results_df=results_df,
+                        status="success"
+                    )
+                    
+                    logger.info(f"Batch {batch_number} of {total_batches} processed in {time.time() - batch_start_time:.2f} seconds")
+                    
+                    responses.append(BatchResponse(
+                        batch_number=batch_number,
+                        total_batches=total_batches,
+                        processed_rows=len(chunk),
+                        results_path=batch_results_path,
+                        message=f"Successfully processed batch {batch_number} of {total_batches}"
+                    ))
+                except Exception as e:
+                    error_message = str(e)
+                    logger.error(f"Error processing batch {batch_number}: {error_message}")
+                    
+                    # Save error to database
+                    db_ops.save_batch_to_db(
+                        batch_number=batch_number,
+                        total_batches=total_batches,
+                        results_df=pd.DataFrame(),  # Empty DataFrame for error case
+                        status="error"
+                    )
+                    
+                    responses.append(BatchResponse(
+                        batch_number=batch_number,
+                        total_batches=total_batches,
+                        processed_rows=len(chunk),
+                        results_path="",
+                        message=f"Error processing batch {batch_number}: {error_message}"
+                    ))
+                
+                offset += batch_size
+        
+        logger.info(f"Streaming pipeline endpoint completed in {time.time() - start_time:.2f} seconds")
+        return responses
+    except Exception as e:
+        logger.error(f"Error in streaming pipeline: {str(e)}")
+        logger.info(f"Streaming pipeline endpoint failed after {time.time() - start_time:.2f} seconds")
+        raise HTTPException(status_code=500, detail=str(e)) 

salary_analytics/config.py

@@ -0,0 +1,72 @@
+"""
+Configuration settings for the salary analytics package.
+"""
+
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Base directories
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+OUTPUT_DIR = os.path.join(BASE_DIR, "output")
+PLOTS_DIR = os.path.join(OUTPUT_DIR, "plots")
+CSV_DIR = os.path.join(OUTPUT_DIR, "csv")
+MODEL_DIR = os.path.join(OUTPUT_DIR, "models")
+
+# Create directories if they don't exist
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+os.makedirs(PLOTS_DIR, exist_ok=True)
+os.makedirs(CSV_DIR, exist_ok=True)
+os.makedirs(MODEL_DIR, exist_ok=True)
+
+# Database Configuration
+DB_CONFIG = {
+    "user": os.getenv("DB_USER", "salaryloan"),  # Default value as fallback
+    "password": os.getenv("DB_PASSWORD", "salaryloan"),
+    "name": os.getenv("DB_NAME", "salaryloan"),
+    "port": os.getenv("DB_PORT", "10532"),
+    "host": os.getenv("DB_HOST", "dev-data.simbrellang.net")
+}
+
+# Table Configuration
+TABLE_NAME = "customer_account_transaction_hx"
+BATCH_RESULTS_TABLE = "salary_analytics_batch_results"
+
+# Salary Keywords
+SALARY_KEYWORDS = [
+    "salary", "payroll", "income", "wage", "wages",
+    "earnings", "earning", "monthly pay", "net pay", "gross pay", "compensation",
+    "monthlypay", "netpay", "grosspay",
+    "remuneration", "stipend", "allowance", "bonus", "commission",
+    "pension", "retirement", "dividend", "benefits", "reimbursement",
+    "overtime", "incentive", "paycheck", "paycheque", "salary advance",
+    "monthly income", "income tax refund", "employer deposit",
+    "payroll deposit", "salary credit", "income credit", "salary transfer",
+    "income transfer", "salary received", "income received", "hr deposit",
+    "company deposit", "employer payment", "employee payment",
+    "sal",
+]
+
+# Model Configuration
+MODEL_CONFIG = {
+    "cv_threshold": 0.10,
+    "min_transactions": 3,
+    "threshold": 0.7,
+    "high_earner_threshold": 10000
+}
+
+# File Paths
+OUTPUT_PATHS = {
+    "high_earner_details": os.path.join(CSV_DIR, "high_earner_details.csv"),
+    "likely_salary_earner": os.path.join(CSV_DIR, "likely_salary_earner.csv"),
+    "final_table": os.path.join(CSV_DIR, "final_table.csv"),
+    "consistent_earners_plot": os.path.join(PLOTS_DIR, "consistent_earners_predictions.png"),
+    "inconsistent_earners_plot": os.path.join(PLOTS_DIR, "inconsistent_earners_predictions.png"),
+    "hypothesis_overlap_plot": os.path.join(PLOTS_DIR, "hypothesis_overlap.png"),
+    "consistent_model": os.path.join(MODEL_DIR, "consistent_model.joblib"),
+    "inconsistent_model": os.path.join(MODEL_DIR, "inconsistent_model.joblib"),
+    "consistent_scaler": os.path.join(MODEL_DIR, "consistent_scaler.joblib"),
+    "inconsistent_scaler": os.path.join(MODEL_DIR, "inconsistent_scaler.joblib")
+} 

salary_analytics/consistent_amount_analyzer.py

@@ -0,0 +1,64 @@
+"""
+Consistent amount transaction analysis module.
+"""
+
+import pandas as pd
+from .config import MODEL_CONFIG
+
+class ConsistentAmountAnalyzer:
+    def __init__(self, df):
+        self.df = df
+        self.const_df = None
+
+    def calculate_coefficient_of_variation(self, group):
+        """Calculate coefficient of variation for a group of transactions."""
+        amounts = group[group['initiated_by'] == 'C']['amount']
+        mean = amounts.mean()
+        std = amounts.std(ddof=0)
+
+        if mean == 0:
+            return float('nan')
+        return std / mean
+
+    def flag_consistent_amounts(self, group, cv_threshold=None):
+        """Flag accounts with low variance in transaction amounts."""
+        if cv_threshold is None:
+            cv_threshold = MODEL_CONFIG['cv_threshold']
+
+        filtered_group = group[group['initiated_by'] == 'C']
+        cv = self.calculate_coefficient_of_variation(filtered_group)
+        is_consistent = cv <= cv_threshold if not pd.isna(cv) else False
+
+        return pd.Series(
+            [is_consistent] * len(group),
+            index=group.index,
+            name='is_consistent_amount'
+        )
+
+    def identify_consistent_amount_accounts(self, cv_threshold=None):
+        """Identify accounts with consistent transaction amounts."""
+        if cv_threshold is None:
+            cv_threshold = MODEL_CONFIG['cv_threshold']
+
+        # Create a copy of the original DataFrame
+        self.const_df = self.df.copy()
+        
+        # Calculate consistent amount flags
+        consistent_flags = self.const_df.groupby('accountid').apply(
+            lambda group: self.flag_consistent_amounts(group, cv_threshold)
+        ).reset_index(level=0, drop=True)
+        
+        # Add the flags to the original DataFrame
+        self.const_df['is_consistent_amount'] = consistent_flags
+        
+        return self.const_df
+
+    def get_consistent_amount_data(self):
+        """Get transactions identified as having consistent amounts."""
+        if self.const_df is None:
+            self.identify_consistent_amount_accounts()
+        
+        return self.const_df[
+            (self.const_df['is_consistent_amount']) &
+            (self.const_df['initiated_by'] == 'C')
+        ] 

salary_analytics/data_loader.py

@@ -0,0 +1,170 @@
+"""
+Data loading and preprocessing module.
+"""
+
+from sqlalchemy import create_engine, text
+import pandas as pd
+from datetime import datetime
+import logging
+import os
+from .config import DB_CONFIG, TABLE_NAME
+
+logger = logging.getLogger(__name__)
+
+class DataLoader:
+    def __init__(self):
+        self.engine = None
+        self.df = None
+        self.chunk_size = 10000  # Load 10,000 rows at a time
+
+    def connect(self):
+        """Establish database connection."""
+        try:
+            logger.info("Attempting to connect to database...")
+            DATABASE_URL = f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}"
+            self.engine = create_engine(DATABASE_URL)
+            with self.engine.connect() as conn:
+                # First check if table exists
+                check_table = text(f"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = '{TABLE_NAME}')")
+                table_exists = conn.execute(check_table).scalar()
+                
+                if not table_exists:
+                    logger.error(f"Table {TABLE_NAME} does not exist in the database")
+                    return False
+                
+                # Get row count
+                count_query = text(f"SELECT COUNT(*) FROM {TABLE_NAME}")
+                row_count = conn.execute(count_query).scalar()
+                logger.info(f"Table {TABLE_NAME} exists with {row_count} rows")
+                
+                # Get version
+                result = conn.execute(text("SELECT version();"))
+                logger.info("Connected successfully to database!")
+                return True
+        except Exception as e:
+            logger.error(f"Error connecting to database: {str(e)}")
+            return False
+
+    def load_from_csv(self, file_path):
+        """Load data from a CSV file."""
+        try:
+            logger.info(f"Loading data from CSV file: {file_path}")
+            if not os.path.exists(file_path):
+                logger.error(f"CSV file not found: {file_path}")
+                return None
+
+            # Load data in chunks
+            chunks = []
+            for chunk in pd.read_csv(file_path, chunksize=self.chunk_size):
+                # Preprocess chunk
+                chunk['trx_start_date'] = pd.to_datetime(chunk['trx_start_date'])
+                chunk['trx_end_date'] = pd.to_datetime(chunk['trx_end_date'])
+                
+                # Rename columns if needed
+                if 'd1' in chunk.columns:
+                    chunk = chunk.rename(columns={
+                        'd1': 'trx_type',
+                        'd2': 'trx_subtype',
+                        'd3': 'initiated_by',
+                        'd4': 'customer_id'
+                    })
+                
+                chunk = chunk.dropna()
+                chunks.append(chunk)
+            
+            # Combine all chunks
+            self.df = pd.concat(chunks, ignore_index=True)
+            logger.info(f"Successfully loaded {len(self.df)} rows from CSV")
+            
+            # Basic data validation
+            logger.info("Performing data validation...")
+            logger.info(f"Columns in dataset: {self.df.columns.tolist()}")
+            logger.info(f"Data types:\n{self.df.dtypes}")
+            logger.info(f"Missing values:\n{self.df.isnull().sum()}")
+            
+            return self.df
+        except Exception as e:
+            logger.error(f"Error loading data from CSV: {str(e)}")
+            return None
+
+    def load_from_db(self):
+        """Load and preprocess transaction data from database in chunks."""
+        if not self.engine:
+            logger.info("No database connection. Attempting to connect...")
+            if not self.connect():
+                logger.error("Failed to establish database connection")
+                return None
+
+        try:
+            logger.info(f"Loading data from table: {TABLE_NAME}")
+            
+            # First get total count
+            with self.engine.connect() as conn:
+                count_query = text(f"SELECT COUNT(*) FROM {TABLE_NAME}")
+                total_rows = conn.execute(count_query).scalar()
+                logger.info(f"Total rows to process: {total_rows}")
+            
+            # Load data in chunks
+            chunks = []
+            offset = 0
+            
+            while True:
+                logger.info(f"Loading chunk starting at offset {offset}")
+                query = f"SELECT * FROM {TABLE_NAME} LIMIT {self.chunk_size} OFFSET {offset}"
+                chunk = pd.read_sql(query, self.engine)
+                
+                if chunk.empty:
+                    break
+                    
+                # Preprocess chunk
+                chunk['trx_start_date'] = pd.to_datetime(chunk['trx_start_date'])
+                chunk['trx_end_date'] = pd.to_datetime(chunk['trx_end_date'])
+                
+                # Rename columns
+                chunk = chunk.rename(columns={
+                    'd1': 'trx_type',
+                    'd2': 'trx_subtype',
+                    'd3': 'initiated_by',
+                    'd4': 'customer_id'
+                })
+                
+                chunk = chunk.dropna()
+                chunks.append(chunk)
+                offset += self.chunk_size
+                
+                if offset >= total_rows:
+                    break
+            
+            # Combine all chunks
+            self.df = pd.concat(chunks, ignore_index=True)
+            logger.info(f"Successfully loaded {len(self.df)} rows of data")
+            
+            # Basic data validation
+            logger.info("Performing data validation...")
+            logger.info(f"Columns in dataset: {self.df.columns.tolist()}")
+            logger.info(f"Data types:\n{self.df.dtypes}")
+            logger.info(f"Missing values:\n{self.df.isnull().sum()}")
+            
+            return self.df
+        except Exception as e:
+            logger.error(f"Error loading data: {str(e)}")
+            return None
+
+    def load_data(self, source='db', file_path=None):
+        """Load data from either database or CSV file."""
+        if source == 'db':
+            return self.load_from_db()
+        elif source == 'csv':
+            if not file_path:
+                logger.error("File path must be provided when loading from CSV")
+                return None
+            return self.load_from_csv(file_path)
+        else:
+            logger.error(f"Invalid source: {source}. Must be 'db' or 'csv'")
+            return None
+
+    def get_data(self):
+        """Get the loaded DataFrame."""
+        if self.df is None:
+            logger.warning("No data loaded. Call load_data() first.")
+        return self.df 

salary_analytics/db_operations.py

@@ -0,0 +1,137 @@
+"""
+Database operations module for salary analytics.
+"""
+
+import logging
+from sqlalchemy import text
+from .config import BATCH_RESULTS_TABLE
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+class DatabaseOperations:
+    def __init__(self, engine):
+        """Initialize with SQLAlchemy engine."""
+        self.engine = engine
+
+    def create_batch_results_table(self):
+        """Create the batch results table if it doesn't exist."""
+        try:
+            with self.engine.connect() as conn:
+                # Check if table exists
+                check_table = text(f"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = '{BATCH_RESULTS_TABLE}')")
+                table_exists = conn.execute(check_table).scalar()
+                
+                if not table_exists:
+                    # Create table
+                    create_table = text(f"""
+                        CREATE TABLE {BATCH_RESULTS_TABLE} (
+                            id SERIAL PRIMARY KEY,
+                            batch_number INTEGER,
+                            total_batches INTEGER,
+                            processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                            accountid TEXT,
+                            num_months INTEGER,
+                            least_inflow_6m DECIMAL,
+                            avg_monthly_salary DECIMAL,
+                            estimated_next_amount DECIMAL,
+                            estimated_next_date DATE,
+                            is_45day_salary BOOLEAN,
+                            is_2months_salary BOOLEAN,
+                            status TEXT
+                        )
+                    """)
+                    conn.execute(create_table)
+                    conn.commit()
+                    logger.info(f"Created table {BATCH_RESULTS_TABLE}")
+                return True
+        except Exception as e:
+            logger.error(f"Error creating batch results table: {str(e)}")
+            return False
+
+    def save_batch_to_db(self, batch_number, total_batches, results_df, status="success"):
+        """Save batch processing results to database."""
+        try:
+            with self.engine.connect() as conn:
+                # Add batch metadata to the DataFrame
+                results_df['batch_number'] = batch_number
+                results_df['total_batches'] = total_batches
+                results_df['processed_at'] = datetime.now()
+                
+                # Convert DataFrame to list of dictionaries
+                records = results_df.to_dict('records')
+                
+                # Insert each record
+                for record in records:
+                    insert_query = text(f"""
+                        INSERT INTO {BATCH_RESULTS_TABLE} 
+                        (batch_number, total_batches, processed_at, accountid, num_months, 
+                         least_inflow_6m, avg_monthly_salary, estimated_next_amount, 
+                         estimated_next_date, is_45day_salary, is_2months_salary, status)
+                        VALUES 
+                        (:batch_number, :total_batches, :processed_at, :accountid, :num_months,
+                         :least_inflow_6m, :avg_monthly_salary, :estimated_next_amount,
+                         :estimated_next_date, :is_45day_salary, :is_2months_salary, :status)
+                    """)
+                    
+                    # Convert boolean columns to proper format
+                    record['is_45day_salary'] = record.get('45daysalary', False)
+                    record['is_2months_salary'] = record.get('2monthssalary', False)
+                    
+                    # Add status
+                    record['status'] = status
+                    
+                    conn.execute(insert_query, record)
+                
+                conn.commit()
+                logger.info(f"Successfully saved batch {batch_number} results to database")
+                return True
+        except Exception as e:
+            logger.error(f"Error saving batch {batch_number} to database: {str(e)}")
+            return False
+
+    def get_batch_status(self, batch_number):
+        """Get the status of a specific batch."""
+        try:
+            with self.engine.connect() as conn:
+                query = text(f"""
+                    SELECT 
+                        batch_number,
+                        total_batches,
+                        processed_at,
+                        COUNT(*) as total_records,
+                        SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as successful_records,
+                        SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as failed_records
+                    FROM {BATCH_RESULTS_TABLE}
+                    WHERE batch_number = :batch_number
+                    GROUP BY batch_number, total_batches, processed_at
+                    ORDER BY processed_at DESC
+                    LIMIT 1
+                """)
+                result = conn.execute(query, {"batch_number": batch_number}).fetchone()
+                return dict(result) if result else None
+        except Exception as e:
+            logger.error(f"Error getting batch {batch_number} status: {str(e)}")
+            return None
+
+    def get_all_batches(self):
+        """Get all batch processing results."""
+        try:
+            with self.engine.connect() as conn:
+                query = text(f"""
+                    SELECT 
+                        batch_number,
+                        total_batches,
+                        processed_at,
+                        COUNT(*) as total_records,
+                        SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) as successful_records,
+                        SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as failed_records
+                    FROM {BATCH_RESULTS_TABLE}
+                    GROUP BY batch_number, total_batches, processed_at
+                    ORDER BY batch_number
+                """)
+                results = conn.execute(query).fetchall()
+                return [dict(row) for row in results]
+        except Exception as e:
+            logger.error(f"Error getting all batches: {str(e)}")
+            return [] 

salary_analytics/keyword_analyzer.py

@@ -0,0 +1,47 @@
+"""
+Keyword-based salary transaction analysis module.
+"""
+
+import re
+import pandas as pd
+from .config import SALARY_KEYWORDS
+
+class KeywordAnalyzer:
+    def __init__(self, df):
+        self.df = df
+        self.desc_df = None
+
+    def identify_salary_transactions(self):
+        """
+        Identifies potential salary-related transactions based on keywords
+        and month-year patterns in the 'description' column.
+        """
+        month_year_patterns = [
+            r"\b(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)\s?\d{2,4}\b",
+            r"\b(?:JANUARY|FEBRUARY|MARCH|APRIL|MAY|JUNE|JULY|AUGUST|SEPTEMBER|OCTOBER|NOVEMBER|DECEMBER)\s?\d{2,4}\b"
+        ]
+
+        escaped_keywords = [re.escape(keyword.lower()) for keyword in SALARY_KEYWORDS]
+        combined_pattern = (
+            r'\b(?:' + '|'.join(escaped_keywords) + r')\b|' +
+            '|'.join(month_year_patterns)
+        )
+
+        self.df['is_salary_related'] = self.df['description'].str.lower().str.contains(
+            combined_pattern,
+            na=False,
+            regex=True
+        )
+
+        self.desc_df = self.df.copy()
+        return self.df
+
+    def get_salary_related_data(self):
+        """Get transactions identified as salary-related."""
+        if self.desc_df is None:
+            self.identify_salary_transactions()
+        
+        return self.desc_df[
+            (self.desc_df['is_salary_related'] == True) & 
+            (self.desc_df['initiated_by'] == 'C')
+        ] 

salary_analytics/main.py

@@ -0,0 +1,154 @@
+"""
+Main module for running the salary analytics pipeline.
+"""
+
+import logging
+from .data_loader import DataLoader
+from .keyword_analyzer import KeywordAnalyzer
+from .consistent_amount_analyzer import ConsistentAmountAnalyzer
+from .transaction_type_analyzer import TransactionTypeAnalyzer
+from .salary_earner_analyzer import SalaryEarnerAnalyzer
+from .salary_predictor import SalaryPredictor
+
+logger = logging.getLogger(__name__)
+
+class SalaryAnalyticsPipeline:
+    def __init__(self):
+        logger.info("Initializing SalaryAnalyticsPipeline")
+        self.data_loader = None
+        self.df = None
+        self.keyword_analyzer = None
+        self.consistent_amount_analyzer = None
+        self.transaction_type_analyzer = None
+        self.salary_earner_analyzer = None
+        self.salary_predictor = None
+
+    def load_data(self, source='db', file_path=None):
+        """Load and preprocess the transaction data."""
+        logger.info("Starting data loading process")
+        self.data_loader = DataLoader()
+        self.df = self.data_loader.load_data(source=source, file_path=file_path)
+        if self.df is not None:
+            logger.info(f"Successfully loaded data with {len(self.df)} rows")
+        else:
+            logger.error("Failed to load data")
+        return self.df is not None
+
+    def run_keyword_analysis(self):
+        """Run keyword-based salary transaction analysis."""
+        if self.df is None:
+            logger.error("Data not loaded. Call load_data() first.")
+            raise ValueError("Data not loaded. Call load_data() first.")
+        
+        logger.info("Starting keyword analysis")
+        self.keyword_analyzer = KeywordAnalyzer(self.df)
+        self.keyword_analyzer.identify_salary_transactions()
+        keyword_data = self.keyword_analyzer.get_salary_related_data()
+        
+        # Update main DataFrame with keyword analysis results
+        self.df['is_salary_related'] = self.df.index.isin(keyword_data.index)
+        return keyword_data
+
+    def run_consistent_amount_analysis(self):
+        """Run consistent amount transaction analysis."""
+        if self.df is None:
+            logger.error("Data not loaded. Call load_data() first.")
+            raise ValueError("Data not loaded. Call load_data() first.")
+        
+        logger.info("Starting consistent amount analysis")
+        self.consistent_amount_analyzer = ConsistentAmountAnalyzer(self.df)
+        self.consistent_amount_analyzer.identify_consistent_amount_accounts()
+        consistent_data = self.consistent_amount_analyzer.get_consistent_amount_data()
+        
+        # Update main DataFrame with consistent amount analysis results
+        self.df['is_consistent_amount'] = self.df.index.isin(consistent_data.index)
+        return consistent_data
+
+    def run_transaction_type_analysis(self):
+        """Run transaction type analysis."""
+        if self.df is None:
+            logger.error("Data not loaded. Call load_data() first.")
+            raise ValueError("Data not loaded. Call load_data() first.")
+        
+        logger.info("Starting transaction type analysis")
+        self.transaction_type_analyzer = TransactionTypeAnalyzer(self.df)
+        self.transaction_type_analyzer.flag_salary_type_transactions()
+        type_data = self.transaction_type_analyzer.get_salary_type_data()
+        
+        # Update main DataFrame with transaction type analysis results
+        self.df['is_salary_type'] = self.df.index.isin(type_data.index)
+        return type_data
+
+    def generate_salary_earner_reports(self):
+        """Generate salary earner reports."""
+        if self.df is None:
+            logger.error("Data not loaded. Call load_data() first.")
+            raise ValueError("Data not loaded. Call load_data() first.")
+        
+        # Ensure all analysis flags are present
+        required_columns = ['is_salary_related', 'is_consistent_amount', 'is_salary_type']
+        missing_columns = [col for col in required_columns if col not in self.df.columns]
+        
+        if missing_columns:
+            logger.error(f"Missing required columns: {missing_columns}")
+            raise ValueError(f"Missing required columns: {missing_columns}. Run all analyses first.")
+        
+        logger.info("Starting salary earner report generation")
+        self.salary_earner_analyzer = SalaryEarnerAnalyzer(self.df)
+        return self.salary_earner_analyzer.generate_reports()
+
+    def train_salary_prediction_models(self):
+        """Train salary prediction models."""
+        if self.df is None:
+            logger.error("Data not loaded. Call load_data() first.")
+            raise ValueError("Data not loaded. Call load_data() first.")
+        
+        logger.info("Starting model training")
+        self.salary_predictor = SalaryPredictor(self.df)
+        
+        # Get accounts from the salary earner analyzer
+        if self.salary_earner_analyzer is None:
+            logger.info("Salary earner analyzer not initialized. Generating reports first.")
+            self.generate_salary_earner_reports()
+        
+        consistent_accounts = self.salary_earner_analyzer.final_table['accountid'].unique()
+        inconsistent_accounts = self.salary_earner_analyzer.likely_salary_earner['accountid'].unique()
+        
+        self.salary_predictor.train_and_evaluate(consistent_accounts, inconsistent_accounts)
+
+    def run_full_pipeline(self, source='db', file_path=None):
+        """Run the complete salary analytics pipeline."""
+        logger.info("Starting full pipeline execution")
+        if not self.load_data(source=source, file_path=file_path):
+            logger.error("Failed to load data. Exiting pipeline.")
+            return False
+
+        try:
+            logger.info("Running keyword analysis...")
+            self.run_keyword_analysis()
+
+            logger.info("Running consistent amount analysis...")
+            self.run_consistent_amount_analysis()
+
+            logger.info("Running transaction type analysis...")
+            self.run_transaction_type_analysis()
+
+            logger.info("Generating salary earner reports...")
+            self.generate_salary_earner_reports()
+
+            logger.info("Training salary prediction models...")
+            self.train_salary_prediction_models()
+
+            logger.info("Pipeline completed successfully!")
+            return True
+        except Exception as e:
+            logger.error(f"Pipeline failed: {str(e)}")
+            return False
+
+def main():
+    """Main function to run the salary analytics pipeline."""
+    pipeline = SalaryAnalyticsPipeline()
+    pipeline.run_full_pipeline()
+
+if __name__ == "__main__":
+    main() 

salary_analytics/salary_earner_analyzer.py

@@ -0,0 +1,176 @@
+"""
+Salary earner analysis and report generation module.
+"""
+
+import pandas as pd
+import matplotlib.pyplot as plt
+from matplotlib_venn import venn3
+from datetime import datetime, timedelta
+import logging
+from .config import MODEL_CONFIG, OUTPUT_PATHS
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class SalaryEarnerAnalyzer:
+    def __init__(self, df):
+        self.df = df
+        self.final_table = None
+        self.likely_salary_earner = None
+        self.high_earner_details = None
+
+    def filter_venn_section(self, **kwargs):
+        """Filter accounts based on specified combinations of hypothesis flags."""
+        valid_columns = {'is_salary_related', 'is_consistent_amount', 'is_salary_type'}
+        df1 = self.df[self.df['initiated_by'] == 'C'].copy()
+        
+        invalid_keys = set(kwargs.keys()) - valid_columns
+        if invalid_keys:
+            raise ValueError(f"Invalid keys: {invalid_keys}. Valid keys are {valid_columns}.")
+
+        condition = pd.Series([True] * len(df1), index=df1.index)
+        for key, value in kwargs.items():
+            condition &= (df1[key] == value)
+
+        filtered_df = df1[condition]
+        
+        # Drop any rows with NaN values in critical columns
+        critical_cols = ['accountid', 'trx_start_date', 'amount']
+        filtered_df = filtered_df.dropna(subset=critical_cols)
+        
+        return filtered_df
+
+    def plot_hypothesis_overlap(self, hypothesis1_df, hypothesis3_df, hypothesis4_df, account_col='accountid'):
+        """Plot and save Venn diagram showing overlap between hypotheses."""
+        set2 = set(hypothesis3_df[account_col][hypothesis3_df['is_consistent_amount']])
+        set3 = set(hypothesis1_df[account_col][hypothesis1_df['is_salary_related']])
+        set4 = set(hypothesis4_df[account_col][hypothesis4_df['is_salary_type']])
+
+        plt.figure(figsize=(10, 10))
+        venn3([set2, set3, set4], set_labels=('Consistent Amount',
+                                            'Salary Description', 'Transaction Type'))
+        plt.title('Overlap Between Hypotheses')
+        plt.savefig(OUTPUT_PATHS['hypothesis_overlap_plot'])
+        plt.close()
+
+    def generate_salary_earners_table(self, all_three_hypotheses):
+        """Generate a table of salary earners with their metrics."""
+        results = []
+        for accountid, group in all_three_hypotheses.groupby('accountid'):
+            # Skip if group is empty
+            if group.empty:
+                continue
+                
+            # Calculate required metrics
+            num_months = len(group)
+            
+            # Handle last 6 months calculation
+            last_6_months = group[group['trx_start_date'] >= (datetime.now() - timedelta(days=180))]
+            if last_6_months.empty:
+                least_inflow = 0
+            else:
+                least_inflow = last_6_months['amount'].min()
+            
+            # Handle average salary calculation
+            if group['amount'].notna().any():
+                avg_salary = group['amount'].mean()
+            else:
+                avg_salary = 0
+            
+            # Calculate days_since_last_trx with NaN handling
+            group['days_since_last_trx'] = group['trx_start_date'].diff().dt.days
+            median_interval = group['days_since_last_trx'].median()
+            if pd.isna(median_interval):
+                median_interval = 30  # Default to 30 days if no interval data
+
+            last_date = group['trx_start_date'].max()
+            next_date = last_date + timedelta(days=median_interval)
+            next_amount = avg_salary
+
+            # Boolean flags with NaN handling
+            days_since_last = (datetime.now() - last_date).days
+            has_45d = days_since_last <= 45
+            has_2m = len(group[group['trx_start_date'] >= (datetime.now() - timedelta(days=60))]) >= 2
+
+            results.append({
+                'accountid': accountid,
+                'num_months': num_months,
+                'least_inflow_6m': least_inflow,
+                'avg_monthly_salary': avg_salary,
+                'estimated_next_amount': next_amount,
+                'estimated_next_date': next_date,
+                '45daysalary': has_45d,
+                '2monthssalary': has_2m
+            })
+
+        final_df = pd.DataFrame(results)
+        # Drop rows where all numeric columns are NaN
+        numeric_cols = ['num_months', 'least_inflow_6m', 'avg_monthly_salary', 'estimated_next_amount']
+        final_df = final_df.dropna(subset=numeric_cols, how='all')
+        return final_df
+
+    def analyze_salary_earners(self, final_df):
+        """Analyze salary earners and identify high earners."""
+        high_earners = final_df[final_df['estimated_next_amount'] >= MODEL_CONFIG['high_earner_threshold']].copy()
+        high_earner_details = high_earners[['accountid', 'least_inflow_6m']].reset_index(drop=True)
+        count_high = len(high_earners)
+        
+        return high_earner_details, count_high
+
+    def generate_reports(self):
+        """Generate all salary earner reports."""
+        # Get accounts flagged by all three hypotheses
+        all_three_hypotheses = self.filter_venn_section(
+            is_salary_related=True,
+            is_consistent_amount=True,
+            is_salary_type=True
+        )
+
+        # Generate final table
+        self.final_table = self.generate_salary_earners_table(all_three_hypotheses)
+        logger.info(f"Found {self.final_table['accountid'].nunique()} verified salary earners")
+
+        # Generate likely salary earner table
+        green_section = self.filter_venn_section(
+            is_salary_related=True,
+            is_consistent_amount=False,
+            is_salary_type=True
+        )
+
+        yellow_section = self.filter_venn_section(
+            is_salary_related=False,
+            is_consistent_amount=True,
+            is_salary_type=True
+        )
+
+        self.likely_salary_earner = pd.concat([yellow_section, green_section])
+        self.likely_salary_earner = self.likely_salary_earner.drop_duplicates(subset=['id'])
+        self.likely_salary_earner = self.generate_salary_earners_table(self.likely_salary_earner)
+        logger.info(f"Found {self.likely_salary_earner['accountid'].nunique()} likely salary earners")
+
+        # Analyze high earners
+        self.high_earner_details, total_high_earners = self.analyze_salary_earners(self.final_table)
+        logger.info(f"\nTotal High Earners: {total_high_earners}")
+
+        # Plot hypothesis overlap
+        self.plot_hypothesis_overlap(
+            self.df[self.df['is_salary_related']],
+            self.df[self.df['is_consistent_amount']],
+            self.df[self.df['is_salary_type']]
+        )
+
+        # Save reports
+        self.high_earner_details.to_csv(OUTPUT_PATHS['high_earner_details'], index=False)
+        self.likely_salary_earner.to_csv(OUTPUT_PATHS['likely_salary_earner'], index=False)
+        self.final_table.to_csv(OUTPUT_PATHS['final_table'], index=False)
+
+        return {
+            'final_table': self.final_table,
+            'likely_salary_earner': self.likely_salary_earner,
+            'high_earner_details': self.high_earner_details,
+            'total_high_earners': total_high_earners
+        } 

salary_analytics/salary_predictor.py

@@ -0,0 +1,171 @@
+"""
+Salary prediction module using machine learning.
+"""
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
+from joblib import dump
+from .config import OUTPUT_PATHS
+
+class SalaryPredictor:
+    def __init__(self, df):
+        self.df = df
+        self.model_cons = None
+        self.model_incons = None
+        self.scaler_cons = None
+        self.scaler_incons = None
+
+    def add_feature_engineering(self, df):
+        """Engineer features for salary prediction."""
+        df['month'] = df['trx_start_date'].dt.month
+        df['month_seq'] = df.groupby(['accountid', 'month']).ngroup() + 1
+
+        # Categorical encoding
+        encoder = OneHotEncoder(sparse_output=False)
+        encoded_trx_type = encoder.fit_transform(df[['trx_type']])
+        encoded_df = pd.DataFrame(encoded_trx_type, columns=encoder.get_feature_names_out(['trx_type']))
+        df = pd.concat([df, encoded_df], axis=1)
+
+        # Rolling statistics
+        df = df.sort_values(['accountid', 'trx_start_date'])
+        df['rolling_sum_3m'] = df.groupby('accountid')['amount'].rolling(window=3,
+                                                                     min_periods=1).sum().reset_index(0, drop=True)
+        df['rolling_avg_3m'] = df.groupby('accountid')['amount'].rolling(window=3,
+                                                                     min_periods=1).mean().reset_index(0, drop=True)
+
+        return df
+
+    def prepare_data(self, df_transactions, accounts):
+        """Prepare data for training and testing."""
+        df_filtered = df_transactions[df_transactions['accountid'].isin(accounts)].copy()
+        print(f"Filtered data for {len(accounts)} accounts.")
+        print(f"Total transactions: {len(df_filtered)}")
+
+        # Drop unnecessary columns
+        df_filtered = df_filtered.drop(['description', 'id', 'customer_id',
+                                    'trx_end_date', 'is_salary_related',
+                                    'is_consistent_amount', 'is_salary_type'], axis=1)
+
+        # Add feature engineering
+        df_filtered = self.add_feature_engineering(df_filtered)
+
+        # Aggregate monthly data
+        agg_funcs = {
+            'amount': 'mean',
+            'rolling_sum_3m': 'last',
+            'rolling_avg_3m': 'last',
+            'month': 'first'
+        }
+        encoded_cols = [col for col in df_filtered.columns if col.startswith('trx_type_')]
+        for col in encoded_cols:
+            agg_funcs[col] = 'sum'
+
+        monthly_data = df_filtered.groupby(['accountid', 'month_seq']).agg(agg_funcs).reset_index()
+
+        # Filter accounts with at least 12 months
+        account_month_counts = monthly_data.groupby('accountid')['month_seq'].max()
+        valid_accounts = account_month_counts[account_month_counts >= 12].index
+        monthly_data = monthly_data[monthly_data['accountid'].isin(valid_accounts)]
+
+        # Create sequences
+        X_train, y_train, X_test, y_test = [], [], [], []
+        feature_cols = ['accountid', 'amount', 'rolling_sum_3m', 'rolling_avg_3m',
+                    'month'] + encoded_cols
+
+        for account in valid_accounts:
+            account_data = monthly_data[monthly_data['accountid'] == account].sort_values('month_seq')
+
+            if len(account_data) >= 12:
+                for t in range(5, 8):
+                    X_train.append(account_data.iloc[t-5:t][feature_cols].values.flatten())
+                    y_train.append(account_data['amount'].iloc[t])
+                for t in range(8, 12):
+                    X_test.append(account_data.iloc[t-5:t][feature_cols].values.flatten())
+                    y_test.append(account_data['amount'].iloc[t])
+            else:
+                print(f"Skipping account {account} due to insufficient data (less than 12 months).")
+
+        return np.array(X_train), np.array(y_train), np.array(X_test), np.array(y_test)
+
+    def train_model(self, X_train, y_train, X_test, y_test):
+        """Train and evaluate a Random Forest model."""
+        # Scale features
+        scaler = StandardScaler()
+        X_train_scaled = scaler.fit_transform(X_train)
+        X_test_scaled = scaler.transform(X_test)
+
+        # Train model
+        model = RandomForestRegressor(n_estimators=100, random_state=42)
+        model.fit(X_train_scaled, y_train)
+
+        # Evaluate
+        y_pred = model.predict(X_test_scaled)
+        mae = mean_absolute_error(y_test, y_pred)
+        rmse = np.sqrt(mean_squared_error(y_test, y_pred))
+        r2 = r2_score(y_test, y_pred)
+        print(f"MAE: {mae:.2f}, RMSE: {rmse:.2f}, R-squared: {r2:.2f}")
+        
+        return model, scaler
+
+    def plot_predictions(self, y_test, y_pred, title, output_path):
+        """Plot actual vs predicted values and save to file."""
+        plt.figure(figsize=(10, 5))
+        plt.scatter(y_test, y_pred, alpha=0.5)
+        plt.xlabel("Actual Salary")
+        plt.ylabel("Predicted Salary")
+        plt.title(title)
+        plt.plot([min(y_test), max(y_test)], [min(y_test), max(y_test)], 'r--')
+        plt.savefig(output_path)
+        plt.close()
+
+    def train_and_evaluate(self, consistent_accounts, inconsistent_accounts):
+        """Train and evaluate models for both consistent and inconsistent salary earners."""
+        # Train model for consistent salary earners
+        X_train_cons, y_train_cons, X_test_cons, y_test_cons = self.prepare_data(self.df, consistent_accounts)
+        if len(X_train_cons) > 0:
+            self.model_cons, self.scaler_cons = self.train_model(X_train_cons, y_train_cons, X_test_cons, y_test_cons)
+            print("Model trained for consistent salary earners.")
+            
+            # Save model and scaler
+            dump(self.model_cons, OUTPUT_PATHS['consistent_model'])
+            dump(self.scaler_cons, OUTPUT_PATHS['consistent_scaler'])
+            print("Saved consistent salary earner model and scaler.")
+            
+            # Plot predictions
+            X_test_cons_scaled = self.scaler_cons.transform(X_test_cons)
+            y_pred = self.model_cons.predict(X_test_cons_scaled)
+            self.plot_predictions(
+                y_test_cons, 
+                y_pred, 
+                "Actual vs. Predicted Salary (Consistent Earners)",
+                OUTPUT_PATHS['consistent_earners_plot']
+            )
+        else:
+            print("No accounts with sufficient data for consistent salary earners.")
+
+        # Train model for inconsistent salary earners
+        X_train_incons, y_train_incons, X_test_incons, y_test_incons = self.prepare_data(self.df, inconsistent_accounts)
+        if len(X_train_incons) > 0:
+            print("\nTraining model for inconsistent salary earners...")
+            self.model_incons, self.scaler_incons = self.train_model(X_train_incons, y_train_incons, X_test_incons, y_test_incons)
+            
+            # Save model and scaler
+            dump(self.model_incons, OUTPUT_PATHS['inconsistent_model'])
+            dump(self.scaler_incons, OUTPUT_PATHS['inconsistent_scaler'])
+            print("Saved inconsistent salary earner model and scaler.")
+            
+            # Plot predictions
+            X_test_incons_scaled = self.scaler_incons.transform(X_test_incons)
+            y_pred = self.model_incons.predict(X_test_incons_scaled)
+            self.plot_predictions(
+                y_test_incons, 
+                y_pred, 
+                "Actual vs. Predicted Salary (Inconsistent Earners)",
+                OUTPUT_PATHS['inconsistent_earners_plot']
+            )
+        else:
+            print("No accounts with sufficient data for inconsistent salary earners.") 

salary_analytics/transaction_type_analyzer.py

@@ -0,0 +1,43 @@
+"""
+Transaction type analysis module.
+"""
+
+import pandas as pd
+from .config import MODEL_CONFIG
+
+class TransactionTypeAnalyzer:
+    def __init__(self, df):
+        self.df = df
+        self.trx_df = None
+
+    def flag_salary_type_transactions(self):
+        """Flag transactions that match salary criteria based on type and subtype."""
+        self.df['is_salary_type'] = (
+            ((self.df['trx_type'] == 'T') | (self.df['trx_type'] == 'C')) &
+            ((self.df['trx_subtype'] == 'BI') | (self.df['trx_subtype'] == 'I') |
+             (self.df['trx_subtype'] == 'BS') | (self.df['trx_subtype'] == 'CI')) &
+            (self.df['initiated_by'] == 'C') &
+            (self.df['amount'] > 0)
+        )
+        
+        self.trx_df = self.df.copy()
+        return self.df
+
+    def is_salary_earner_by_type(self, group, min_transactions=None, threshold=None):
+        """Determine if an account likely belongs to a salary earner."""
+        if min_transactions is None:
+            min_transactions = MODEL_CONFIG['min_transactions']
+        if threshold is None:
+            threshold = MODEL_CONFIG['threshold']
+
+        if len(group) < min_transactions:
+            return False
+        valid_ratio = group['is_salary_type'].mean()
+        return valid_ratio >= threshold
+
+    def get_salary_type_data(self):
+        """Get transactions identified as salary type."""
+        if self.trx_df is None:
+            self.flag_salary_type_transactions()
+        
+        return self.trx_df[self.trx_df['is_salary_type']]