Enhance API with data loading functionality and update README.

- Added `/load-data` endpoint to load transaction data from either a database or a CSV file.
- Updated `SalaryAnalyticsPipeline` and `DataLoader` to support loading from CSV.
- Implemented data validation and error handling for loading processes.
- Revised README to include new data loading instructions and workflow steps.
- Added checks to ensure data is loaded before running analysis endpoints.

salary_analytics/main.py

@@ -23,11 +23,11 @@ class SalaryAnalyticsPipeline:
         self.salary_earner_analyzer = None
         self.salary_predictor = None
 
-    def load_data(self):
+    def load_data(self, source='db', file_path=None):
         """Load and preprocess the transaction data."""
         logger.info("Starting data loading process")
         self.data_loader = DataLoader()
-        self.df = self.data_loader.load_data()
+        self.df = self.data_loader.load_data(source=source, file_path=file_path)
         if self.df is not None:
             logger.info(f"Successfully loaded data with {len(self.df)} rows")
         else:
@@ -43,7 +43,11 @@ class SalaryAnalyticsPipeline:
         logger.info("Starting keyword analysis")
         self.keyword_analyzer = KeywordAnalyzer(self.df)
         self.keyword_analyzer.identify_salary_transactions()
-        return self.keyword_analyzer.get_salary_related_data()
+        keyword_data = self.keyword_analyzer.get_salary_related_data()
+        
+        # Update main DataFrame with keyword analysis results
+        self.df['is_salary_related'] = self.df.index.isin(keyword_data.index)
+        return keyword_data
 
     def run_consistent_amount_analysis(self):
         """Run consistent amount transaction analysis."""
@@ -54,7 +58,11 @@ class SalaryAnalyticsPipeline:
         logger.info("Starting consistent amount analysis")
         self.consistent_amount_analyzer = ConsistentAmountAnalyzer(self.df)
         self.consistent_amount_analyzer.identify_consistent_amount_accounts()
-        return self.consistent_amount_analyzer.get_consistent_amount_data()
+        consistent_data = self.consistent_amount_analyzer.get_consistent_amount_data()
+        
+        # Update main DataFrame with consistent amount analysis results
+        self.df['is_consistent_amount'] = self.df.index.isin(consistent_data.index)
+        return consistent_data
 
     def run_transaction_type_analysis(self):
         """Run transaction type analysis."""
@@ -65,7 +73,11 @@ class SalaryAnalyticsPipeline:
         logger.info("Starting transaction type analysis")
         self.transaction_type_analyzer = TransactionTypeAnalyzer(self.df)
         self.transaction_type_analyzer.flag_salary_type_transactions()
-        return self.transaction_type_analyzer.get_salary_type_data()
+        type_data = self.transaction_type_analyzer.get_salary_type_data()
+        
+        # Update main DataFrame with transaction type analysis results
+        self.df['is_salary_type'] = self.df.index.isin(type_data.index)
+        return type_data
 
     def generate_salary_earner_reports(self):
         """Generate salary earner reports."""
@@ -73,6 +85,14 @@ class SalaryAnalyticsPipeline:
             logger.error("Data not loaded. Call load_data() first.")
             raise ValueError("Data not loaded. Call load_data() first.")
         
+        # Ensure all analysis flags are present
+        required_columns = ['is_salary_related', 'is_consistent_amount', 'is_salary_type']
+        missing_columns = [col for col in required_columns if col not in self.df.columns]
+        
+        if missing_columns:
+            logger.error(f"Missing required columns: {missing_columns}")
+            raise ValueError(f"Missing required columns: {missing_columns}. Run all analyses first.")
+        
         logger.info("Starting salary earner report generation")
         self.salary_earner_analyzer = SalaryEarnerAnalyzer(self.df)
         return self.salary_earner_analyzer.generate_reports()
@@ -96,10 +116,10 @@ class SalaryAnalyticsPipeline:
         
         self.salary_predictor.train_and_evaluate(consistent_accounts, inconsistent_accounts)
 
-    def run_full_pipeline(self):
+    def run_full_pipeline(self, source='db', file_path=None):
         """Run the complete salary analytics pipeline."""
         logger.info("Starting full pipeline execution")
-        if not self.load_data():
+        if not self.load_data(source=source, file_path=file_path):
             logger.error("Failed to load data. Exiting pipeline.")
             return False