Merge branch 'penal_check' of DigiFi/digifi-EventManager into master

pulled changes from main

app/config.py

@@ -55,6 +55,28 @@ class Config:
     MAIL_DEFAULT_SENDER = ('FirstAdvance', 'firstadvance@dynamikservices.tech')
     MAIL_RECEIVER= os.getenv('MAIL_RECEIVER', 'chinenyeumeaku@gmail.com,umeakuchinenye@gmail.com')
 
+
+    # Processing Overdue LOANS sections
+    OVERDUE_LOAN_BATCH_SIZE = int(os.getenv("OVERDUE_LOAN_BATCH_SIZE", 10))
+    OVERDUE_LOAN_DELAY_SECONDS = int(os.getenv("OVERDUE_LOAN_DELAY_SECONDS", 5))
+    OVERDUE_LOAN_BATCH_DELAY_SECONDS = int(
+        os.getenv("OVERDUE_LOAN_BATCH_DELAY_SECONDS", 5)
+    )
+    OVERDUE_GRACE_PERIOD_DAYS = int(os.getenv("OVERDUE_GRACE_PERIOD_DAYS", 30))
+    OVERDUE_PROCESSING_LIST_LIMIT = int(os.getenv("OVERDUE_PROCESSING_LIST_LIMIT", 100))
+    PENAL_CHARGE_PERCENTAGE = os.getenv("PENAL_CHARGE_PERCENTAGE", 1)
+    PENAL_CHARGE_INTERVAL_DAYS = os.getenv("PENAL_CHARGE_INTERVAL_DAYS", 30)
+    PENAL_CHARGE_MAXIMUM_COUNT = os.getenv("PENAL_CHARGE_MAXIMUM_COUNT", 6)
+    
+    #processing failed disbursement sections
+    FAILED_DISBURSEMENT_BATCH_SIZE = int(os.getenv("FAILED_DISBURSEMENT_BATCH_SIZE", 10))
+    FAILED_DISBURSEMENT_DELAY_SECONDS = int(os.getenv("FAILED_DISBURSEMENT_DELAY_SECONDS", 5))
+    FAILED_DISBURSEMENT_BATCH_DELAY_SECONDS = int(
+        os.getenv("FAILED_DISBURSEMENT_BATCH_DELAY_SECONDS", 5)
+    )
+    FAILED_RETRY_TIME_INTERVAL_SECONDS = int(os.getenv("FAILED_RETRY_TIME_INTERVAL_SECONDS", 86400)) #24 hours = 86400 seconds
+
+
     BANK_CALL_API_TIME_OUT = os.getenv("BANK_CALL_API_TIME_OUT", 100)
     BANK_CALL_BASE_URL = os.getenv("BANK_CALL_BASE_URL", "https://bank-emulator.dev.simbrellang.net/api")
     BANK_CALL_SMS_BASE_URL= os.getenv("BANK_CALL_SMS_BASE_URL","https://first-advance-middleware-develop.fbn-devops-dev-asenv.appserviceenvironment.net/SMS")

app/enums/__init__.py

@@ -1,3 +1,3 @@
 from .transaction_type import TransactionType
 from .loan_status import LoanStatus
-from .repayment_schedule_status import RepaymentScheduleStatus
+from .repayment_schedule_status import RepaymentScheduleStatus

app/enums/loan_status.py

@@ -5,4 +5,5 @@ class LoanStatus(str, Enum):
     ACTIVE = "active"
     ACTIVE_PARTIAL = "active_partial"
     START_REPAY = "start_repay"
-    REPAID = "repaid"
+    REPAID = "repaid"
+    FAILED = "failed"

app/enums/repayment_schedule_status.py

@@ -2,4 +2,6 @@ from enum import Enum
 
 class RepaymentScheduleStatus(str, Enum):
     PARTIALLY_PAID = "partially_paid"
-    REPAID = "repaid"
+    REPAID = "repaid"
+    ACTIVE = "active"
+    OVERDUE = "overdue"

app/helpers/collect_loan_helper.py

@@ -53,5 +53,11 @@ class CollectLoanHelper:
             "countryId": "NG",
             "comment": "COLLECT LOAN"
         }
-        
+     
+    @staticmethod   
+    def chunk_list(data, chunk_size):
+        """Yield successive chunk_size chunks from data."""
+        for i in range(0, len(data), chunk_size):
+            yield data[i:i + chunk_size]
+
     

app/integrations/simbrella.py

@@ -2,6 +2,7 @@ import requests
 from app.config import settings
 from app.helpers.response_helper import ResponseHelper
 # from app.routes.autocall import verify_transaction
+from app.models.customer import Customer
 from app.services.loan import LoanService
 from app.services.loan_repayment_schedule import LoanRepaymentScheduleService
 from app.utils.auth import get_headers
@@ -91,9 +92,8 @@ class SimbrellaClient:
         vat_fee = loan_charges.get("VAT")['amount']
         interest_fee = loan_charges.get("INTEREST")['amount']
         insurance_fee = loan_charges.get("INSURANCE")['amount']
-
+        product_id = str(loan_data.get('productId', ""))
         debtId = str(loan_data.get('debtId', "")).strip().zfill(6)
-
         disbursement_data = {
             "transactionId": loan_data.get('transactionId'),
             "fbnTransactionId": loan_data.get('transactionId'),
@@ -102,7 +102,7 @@ class SimbrellaClient:
             "accountId": loan_data.get('accountId'),
             "productId": str(loan_data.get('productId', "")),
             "provideAmount": loan_data.get('currentLoanAmount'),
-            "collectAmountInterest": interest_fee,
+            "collectAmountInterest": interest_fee if product_id != '3MPC' else 0,
             "collectAmountMgtFee": mgt_fee,
             "collectAmountInsurance": insurance_fee,
             "collectAmountVAT": vat_fee,
@@ -137,6 +137,8 @@ class SimbrellaClient:
                 logger.error("")
                 LoanService.set_disbursement_loan_description(loan_data['debtId'],
                                                               "Disbursement Service url not found (404)")
+                logger.info(f"Loan status updated to {LoanStatus.FAILED} for loan ID {loan_data['debtId']} due to disbursement failure")
+                LoanService.update_status(loan_data['debtId'], LoanStatus.FAILED)
                 return ResponseHelper.error("Disbursement Service url not found (404)", status_code=404)
 
             logger.info(f"Disbursement response: {response.json()}")
@@ -147,7 +149,17 @@ class SimbrellaClient:
                                                     result.get('responseMessage', ''))
                 reload_loan = LoanService.get_loan_by_transaction_id(transaction_id=data['transactionId'])
                 reload_loan_data = reload_loan.to_dict()
+                #mark repayment schedule as active
+                repayment_schedule = LoanRepaymentScheduleService.get_repayment_schedule_by_loan_id(
+                    reload_loan_data['debtId'], include_paid=False)
+                logger.info(f'Loan repayment schedule: {repayment_schedule}')
+                if repayment_schedule:
+                    for schedule in repayment_schedule:
+                        logger.info(f"Updating repayment schedule ID {schedule.id} status to ACTIVE")
+                        LoanRepaymentScheduleService.update_repayment_schedule_status_to_active(schedule.id)
+                        
                 SimbrellaClient.verify_disbursement_transaction(reload_loan_data)
+                
                 return ResponseHelper.success(response.json(), "Successful")
 
             else:
@@ -155,6 +167,8 @@ class SimbrellaClient:
                 errorMessage = "Unable to complete Disbursement Service with HTTP status code: " + str(
                     response.status_code)
                 LoanService.set_disbursement_loan_description(loan_data['debtId'], errorMessage)
+                updatedLoan = LoanService.update_status(loan_data['debtId'], LoanStatus.FAILED)
+                logger.info(f"Loan status updated to {updatedLoan.get('status')} for loan ID {loan_data['debtId']} due to disbursement failure")
                 return ResponseHelper.error(errorMessage, status_code=response.status_code)
 
         except requests.exceptions.HTTPError as errh:
@@ -278,11 +292,20 @@ class SimbrellaClient:
                 logger.error("Received 404 from external service")
                 return ResponseHelper.error("Verify Service url not found (404)", status_code=404)
             result = response.json()
+            #check for res 00 and status 200
             logger.info(f"this is verify result, {result}")
             LoanService.set_disburse_verify_result(loan_data['debtId'], result.get('responseCode', ''),
                                                    result.get('responseMessage', ''))
+            
+            customer = Customer.get_customer(loan_data.get('customerId'))
+            if customer:
+                misisdn = customer.msisdn
+            else:
+                logger.info(f"Customer does not exist for customer id: {loan_data.get('customerId')}")
+                misisdn = settings.TEST_NO
+
             sms_data = {
-                "dest": transaction_data.get('phone_number') or settings.TEST_NO,
+                "dest": misisdn,
                 "text": f"Transaction {loan_data.get('transactionId')} verified successfully",
                 "unicode": True
             }

app/models/loan.py

@@ -9,6 +9,7 @@ from app.utils.logger import logger
 from app.extensions import db 
 from decimal import Decimal, ROUND_HALF_UP
 from datetime import datetime, timezone
+from app.config import settings
 
 class Loan(db.Model):
     __tablename__ = "loans"
@@ -46,6 +47,8 @@ class Loan(db.Model):
     verify_result = db.Column(db.String(10), nullable=True)
     verify_description = db.Column(db.String(100), nullable=True)
     reference = db.Column(db.String(50), nullable=True)
+    total_penal_charge = db.Column(db.Float, default=0.0)
+    last_penal_date = db.Column(db.DateTime, nullable=True)
 
     customer = relationship(
         "Customer",
@@ -91,12 +94,27 @@ class Loan(db.Model):
             'disburseVerify': self.disburse_verify.isoformat() if self.disburse_verify else None,
             'reference': self.reference,
             'balance': self.balance,
-            'tenor': self.tenor,   
+            'tenor': self.tenor,
+            'totalPenalCharge': self.total_penal_charge,
+            'lastPenalDate': self.last_penal_date 
         }
 
     @classmethod
     def get_loan_by_transaction_id(cls, transaction_id):
         return cls.query.filter_by(transaction_id=transaction_id).first()
+    
+    #update loan status
+    @classmethod
+    def update_status(cls, loan_id, status):
+        loan = cls.query.get(loan_id)
+        if loan:
+            loan.status = status
+            db.session.commit()
+            return loan.to_dict()
+        else:
+            raise ValueError(f"Loan with ID {loan_id} not found.")
+        
+        
 
     @classmethod
     def get_loan_by_loan_id(cls, loan_id):
@@ -236,9 +254,38 @@ class Loan(db.Model):
         """
         Get the latest loan without a disbursement date.
         """
-        return cls.query.filter(
-            cls.disburse_date.is_(None)
-        ).order_by(cls.created_at.desc()).first()
+        logger.info("Fetching latest loan without disburse date")
+        
+        try:
+            return cls.query.filter(
+                cls.disburse_date.is_(None)
+            ).order_by(cls.created_at.desc()).first()
+        except Exception as e:
+            logger.error(f"Error fetching latest loan without disburse date: {e}")
+            raise 
+        
+    @classmethod
+    def get_failed_disbursements(cls):
+        """
+        Get all loans with failed disbursement.
+        """
+        try:
+            last_time_interval = settings.FAILED_RETRY_TIME_INTERVAL_SECONDS or 0
+            failed_loans = cls.query.filter(
+            cls.disburse_date.is_(None),
+            cls.created_at >= datetime.utcnow() - timedelta(seconds=last_time_interval)
+        ).all()
+
+            if not failed_loans:
+                logger.info("No failed disbursements found.")
+                return []
+
+            logger.info(f"Found {len(failed_loans)} failed disbursements.")
+            return failed_loans
+
+        except Exception as e:
+            logger.error(f"Error fetching failed disbursements: {e}")
+            return []
 
     @classmethod
     def get_latest_loan_with_disburse_date(cls):
@@ -388,3 +435,17 @@ class Loan(db.Model):
         except Exception as e:
             logger.error(f"Error fetching overdue loans: {e}")
             return []
+        
+    @classmethod
+    def apply_penal_to_loan(cls, loan_id, penal_amount):
+
+        loan = cls.query.get(loan_id)
+
+        if not loan:
+            raise ValueError("Loan not found")
+        penal_amount = Decimal(str(penal_amount))
+
+        loan.total_penal_charge = Decimal(str(loan.total_penal_charge or 0)) + penal_amount
+        loan.last_penal_date = datetime.now(timezone.utc)
+
+        db.session.commit()

app/models/loan_charge.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timezone, timedelta
 from os.path import devnull
-
+from sqlalchemy.exc import IntegrityError
 from app.extensions import db
 from sqlalchemy.orm import relationship
 
@@ -43,7 +43,85 @@ class LoanCharge(db.Model):
             'description': self.description,
             'due': self.due
         }
+    #get last penal
+    @classmethod
+    def get_last_penal_no(cls, loan_id):
+        """
+        Returns the last penal number created for a loan.
+        Example:
+            PENAL1 -> returns 1
+            PENAL3 -> returns 3
+        If none exists, returns 0.
+        """
+        last_penal = (
+            cls.query
+            .filter(cls.loan_id == loan_id)
+            .filter(cls.code.like("PENAL%"))
+            .order_by(cls.id.desc())
+            .first()
+        )
 
+        if not last_penal:
+            return 0
+
+        try:
+            return int(last_penal.code.replace("PENAL", ""))
+        except ValueError:
+            return 0
+    @classmethod
+    def get_penal_charges_by_loan_id(cls, loan_id):
+        """
+        Returns all penal charges for a specific loan.
+        """
+        return cls.query.filter(
+            cls.loan_id == loan_id,
+            cls.code.like("PENAL%")
+        ).all()
+    
+    
     @classmethod
     def get_loan_charge_by_debt_id(cls, debt_id):
-        return cls.query.filter_by(loan_id=debt_id)
+        return cls.query.filter_by(loan_id=debt_id)
+    
+    #create penal charge
+    @classmethod
+    def create_penal_charges_for_loan(cls, loan_id, transaction_id, percent, penal_no, schedule_number, penal_amount=0.0):
+        """
+        Create a penal charge for a given loan and schedule.
+        """
+
+        if loan_id is None:
+            raise ValueError("loan_id cannot be None")
+
+        code = f"PENAL{penal_no:02d}-SCHEDULE{schedule_number:02d}"
+
+        # Check if this penal charge already exists
+        existing = cls.query.filter_by(
+            loan_id=loan_id,
+            code=code
+        ).first()
+
+        if existing:
+            return existing
+
+        now = datetime.now(timezone.utc)
+
+        penal_charge = cls(
+            loan_id=loan_id,
+            transaction_id=transaction_id,
+            code=code,
+            amount=penal_amount,
+            percent=percent,
+            description=f"Penal Charge {penal_no} for loan {loan_id} schedule {schedule_number}",
+            due=True,
+            due_date=now
+        )
+
+        try:
+            db.session.add(penal_charge)
+            db.session.commit()
+        except IntegrityError as err:
+            db.session.rollback()
+            raise ValueError(f"Database integrity error: {err}")
+
+        return penal_charge

app/models/loan_repayment_schedule.py

@@ -1,8 +1,10 @@
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from app.extensions import db
 from app.utils.logger import logger
 from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import or_
 from app.enums.repayment_schedule_status import RepaymentScheduleStatus
+from app.config import settings
 
 from decimal import Decimal, ROUND_HALF_UP
 # from dateutil.relativedelta import relativedelta
@@ -27,6 +29,9 @@ class LoanRepaymentSchedule(db.Model):
     partial_balance = db.Column(db.Float, default=0.0)
     created_at = db.Column(db.DateTime, default=datetime.now(timezone.utc))
     updated_at = db.Column(db.DateTime, default=datetime.now(timezone.utc), onupdate=datetime.now(timezone.utc))
+    penal_charge = db.Column(db.Float, default=0.0)
+    penal_count = db.Column(db.Integer, default=0)
+    last_penal_date = db.Column(db.DateTime, nullable=True)
 
 
 
@@ -48,7 +53,11 @@ class LoanRepaymentSchedule(db.Model):
             'partial_balance': self.partial_balance,
             'paid_at': self.paid_at.isoformat() if self.paid_at else None,
             'created_at': self.created_at.isoformat() if self.created_at else None,
-            'updated_at': self.updated_at.isoformat() if self.updated_at else None
+            'updated_at': self.updated_at.isoformat() if self.updated_at else None,
+            'penal_charge': self.penal_charge,
+            'penal_count': self.penal_count,
+            'last_penal_date': self.last_penal_date.isoformat() if self.last_penal_date else None
+            
         }
 
     def __repr__(self):
@@ -89,13 +98,72 @@ class LoanRepaymentSchedule(db.Model):
     @classmethod
     def get_overdue_repayment_schedule(cls):
         """
-        Get all overdue repayment schedules.
+        Get all overdue repayment schedules that are not  repaid.
         """
         try:
-            return cls.query.filter(cls.due_date < datetime.now(timezone.utc), cls.paid == False).all()
+            return cls.query.filter(cls.due_date < datetime.now(timezone.utc), cls.paid == False).order_by(cls.due_date.asc()).all()
         except Exception as e:
             logger.error(f"Error fetching overdue repayment schedules: {e}")
             return []
+    @classmethod
+    def get_active_overdue_repayment_schedule(cls):
+        """
+        Get all overdue repayment schedules that are active.
+        """
+        try:
+            return (
+                cls.query
+                .filter(
+                    cls.due_date < datetime.now(timezone.utc),
+                    cls.paid_status == RepaymentScheduleStatus.ACTIVE
+                )
+                .order_by(cls.due_date.asc())
+                .all()
+            )
+        except Exception as e:
+            logger.error(f"Error fetching active overdue repayment schedules: {e}")
+            return []
+    
+
+    @classmethod
+    def get_overdue_repayment_schedule_with_grace_period(cls, grace_period_days, limit=None):
+        try:
+            now = datetime.now(timezone.utc)
+            grace_period_date = now - timedelta(days=grace_period_days)
+            penal_interval = timedelta(days=settings.PENAL_CHARGE_INTERVAL_DAYS)
+
+            return cls.query.filter(
+                cls.due_date < grace_period_date,
+                cls.paid == False,
+                or_(
+                    cls.last_penal_date == None,  # never penalized before
+                    cls.last_penal_date < now - penal_interval
+                )
+            ).order_by(cls.due_date.asc()).limit(limit).all()
+
+        except Exception as e:
+            logger.error(f"Error fetching overdue repayment schedules with grace period: {e}")
+            return []
+            
+    @classmethod
+    def get_partially_paid_overdue_repayment_schedule(cls):
+        """
+        Get all overdue repayment schedules that are partially paid.
+        """
+        try:
+            return (
+                cls.query
+                .filter(
+                    cls.due_date < datetime.now(timezone.utc),
+                    cls.paid_status == RepaymentScheduleStatus.PARTIALLY_PAID
+                )
+                .order_by(cls.due_date.asc())
+                .all()
+            )
+        except Exception as e:
+            logger.error(f"Error fetching partially paid overdue repayment schedules: {e}")
+            return []
+    
 
     @classmethod
     def get_repayment_schedule_by_transaction_id(cls, transaction_id):
@@ -163,7 +231,25 @@ class LoanRepaymentSchedule(db.Model):
             logger.error(f"Error updating repayment schedule {schedule_id} after loan repayment: {e}")
             raise
 
-        
+    @classmethod
+    def update_repayment_schedule_status_to_active(cls, schedule_id):
+        """
+        Update repayment schedule status to ACTIVE.
+        """
+        try:
+            schedule = cls.query.get(schedule_id)
+            if not schedule:
+                raise ValueError(f"Schedule with ID {schedule_id} does not exist.")
+            schedule.paid_status = RepaymentScheduleStatus.ACTIVE
+            schedule.updated_at = datetime.now(timezone.utc)
+            db.session.commit()
+            logger.info(f"Updated repayment schedule ID {schedule_id} status to ACTIVE")
+            return schedule.to_dict()
+        except Exception as e:
+            db.session.rollback()
+            logger.error(f"Error updating repayment schedule status for schedule {schedule_id}: {e}")
+            raise
+         
     @classmethod
     def update_repayment_schedule_balance(cls, schedule_id, amount_collected):
         """
@@ -222,5 +308,41 @@ class LoanRepaymentSchedule(db.Model):
             db.session.rollback()
             logger.error(f"Error applying repayment for schedule {schedule_id}: {e}")
             raise
-        
+    from decimal import Decimal
+
+    @classmethod
+    def apply_penal_to_schedule(cls, schedule_id, penal_amount):
+
+        schedule = cls.query.get(schedule_id)
+
+        now = datetime.now(timezone.utc)
+        penal_amount = Decimal(str(penal_amount))
+
+        current_penal = Decimal(str(schedule.penal_charge)) if schedule.penal_charge else Decimal("0")
+
+        schedule.penal_count = (schedule.penal_count or 0) + 1
+        schedule.penal_charge = current_penal + penal_amount
+        schedule.last_penal_date = now
+        schedule.due_process_date = now
+        schedule.updated_at = now
+
+        db.session.commit()
+    # Calculate penal charge
+    @classmethod
+    def calculate_penal_charge(cls, schedule):
+
+        if schedule.paid_status == RepaymentScheduleStatus.PARTIALLY_PAID:
+            outstanding = Decimal(str(schedule.partial_balance))
+        else:
+            outstanding = Decimal(str(schedule.installment_amount))
+
+        rate = Decimal(str(settings.PENAL_CHARGE_PERCENTAGE)) / 100
+
+        penal_charge = (outstanding * rate).quantize(
+            Decimal("0.01"),
+            rounding=ROUND_HALF_UP
+        )
+
+        return penal_charge
+    
     

app/routes/autocall.py

@@ -1,9 +1,10 @@
-from threading import Thread
+import time as time_module
 from flask import Blueprint, request, jsonify, current_app
 import requests
 from app.extensions import db
 from app.config import settings
 from app.helpers.response_helper import ResponseHelper
+from app.helpers.collect_loan_helper import CollectLoanHelper
 from app.utils.auth import get_headers
 from app.utils.logger import logger
 from app.integrations.simbrella import SimbrellaClient
@@ -11,16 +12,26 @@ from app.services.loan import LoanService
 from app.services.repayment import RepaymentService
 from app.services.salary import SalaryService
 from app.services.loan_repayment_schedule import LoanRepaymentScheduleService
+from app.services.loan_charge import LoanChargesService
 from app.enums.loan_status import LoanStatus
 from app.enums.repayment_schedule_status import RepaymentScheduleStatus
 from app.utils.mail import send_report_email, get_report_data
+from datetime import datetime, timezone, timedelta
 from app.config import settings
 
 autocall_bp = Blueprint("autocall", __name__)
 
-
+#refresh-verify-disbursement
+#
+#
 @autocall_bp.route("/refresh-verify-disbursement", methods=["GET"])
 def verify_transaction():
+    """
+    Get the latest loan with disbursement date and no verification date.
+    Then call the verify transaction endpoint with the loan data. 
+    This is called after disbursement to ensure the loan was actually disbursed
+    Then a verification date is set
+    """
     logger.info(f"Calling VerifyTransaction Components")
 
     loan = LoanService.get_latest_loan_with_disburse_date()
@@ -44,6 +55,11 @@ def verify_transaction():
 
 @autocall_bp.route("/refresh-disbursement", methods=["GET"])
 def disbursement():
+    """
+    Get the latest loan without disbursement date. 
+    Then call the disburse loan endpoint with the loan data.
+    """
+    
     # data = request.json()
     logger.info(f"Calling Disbursement Components")
     loan = LoanService.get_latest_loan_without_disburse_date()
@@ -67,6 +83,14 @@ def disbursement():
 
 @autocall_bp.route("/retry-disbursement", methods=["POST"])
 def retry_disbursement():
+    """
+    This takes in a transaction id as input and 
+    retries the disbursement for the loan with that transaction id.
+     This is to be used in cases where the disbursement failed due to network issues
+     or other transient issues. It will call the disbursement endpoint
+     with the loan data for the loan with the given transaction id.
+    """
+    
     try:
         data = request.get_json()
         logger.info(f"Retry Transaction ID Data Received for :::: {data}")
@@ -98,51 +122,64 @@ def retry_disbursement():
     except Exception as e:
         logger.error(f"Failed to call retry disbursement  {data}: {e}")
 
-
-@autocall_bp.route("/verify-disbursement", methods=["POST"])
-def retry_verify_disbursement():
+@autocall_bp.route("/retry-failed-disbursements", methods=["GET"])
+def retry_failed_disbursements():
+    """
+    This endpoint is for retrying failed disbursements.
+    It will get the list of failed disbursements and call the disbursement endpoint for each of them.
+    This is to be used in cases where the disbursement failed due to network issues or other transient issues. It will call the disbursement endpoint
+    with the loan data for the loan with the given transaction id.
+    """
+    
     try:
-        data = request.get_json()
-        logger.info(f"Verify Disbursement Transaction ID Data Received for :::: {data}")
+        logger.info(f"Calling Retry Failed Disbursements Components")
+        failed_disbursements = LoanService.get_failed_disbursements()
+        if not failed_disbursements:
+            logger.info(f"No failed disbursements found")
+            return ResponseHelper.success(message="No failed disbursements found", status_code=200)
 
-        transactionId = data["transactionId"]
-        logger.info(f"Starting Transaction ID Data Received for :::: {transactionId}")
+        logger.info(f"Found {len(failed_disbursements)} failed disbursements to retry")
+        #get batch size from settings
+         # Safe config values
+        batch_size = max(1, settings.FAILED_DISBURSEMENT_BATCH_SIZE or 1)
+        delay_seconds = max(0, settings.FAILED_DISBURSEMENT_DELAY_SECONDS or 0)
+        batch_delay = max(0, settings.FAILED_DISBURSEMENT_BATCH_DELAY_SECONDS or 0)
+        for i in range(0, len(failed_disbursements), batch_size):
+            batch = failed_disbursements[i:i + batch_size]
+            for loan in batch:
+                logger.info(f"Retrying disbursement for loan ID {loan.id}")
+                loan_data = loan.to_dict()
 
-        logger.info(f"Calling Disbursement Components for Retry Transaction ID Data Received for :::: {transactionId}")
-        loan = LoanService.get_loan_by_transaction_id(transactionId)
-        if not loan:
-            logger.info(f"No loan found without disbursement date")
-            return 0
-        logger.info(f"Calling DisburseLoan endpoint with data: {loan}")
-        loan_data = loan.to_dict()
-
-        data = {
-            "transactionId": loan_data.get('transactionId'),
-            "fbnTransactionId": loan_data.get('transactionId'),
-            "debtId": str(loan_data.get('debtId')),
-            "customerId": loan_data.get('customerId'),
-            "accountId": loan_data.get('accountId'),
-            "productId": str(loan_data.get('productId', "")),
-            "provideAmount": loan_data.get('currentLoanAmount'),
-        }
-        response = SimbrellaClient.verify_transaction(data)
-        return ResponseHelper.success(message="Retry Verify Disbursement Request Sent Successfully", status_code=200)
+                data = {
+                    "transactionId": loan_data.get('transactionId'),
+                    "FbnTransactionId": loan_data.get('transactionId'),
+                    "debtId": str(loan_data.get('debtId')),
+                    "customerId": loan_data.get('customerId'),
+                    "accountId": loan_data.get('accountId'),
+                    "productId": str(loan_data.get('productId', "")),
+                    "provideAmount": loan_data.get('currentLoanAmount'),
+                }
+                response = SimbrellaClient.disburse_loan(data)
+                logger.info(f"Retry Disbursement Result Received for loan ID {loan.id}: {response}")
+                time_module.sleep(delay_seconds)
+            time_module.sleep(batch_delay)
+        return ResponseHelper.success(message="Retry Failed Disbursements Request Sent Successfully", status_code=200)
     except Exception as e:
-        logger.error(f"Failed to call retry disbursement  {data}: {e}")
+        logger.error(f"Failed to retry disbursements: {e}")
+        return ResponseHelper.error("Failed to retry disbursements", status_code=500, error=str(e))
+
 
-@autocall_bp.route("/start-repayment", methods=["POST"])
-def start_repayment_office():
-    try:
-        data = request.get_json()
-        logger.info(f"Start Repay Transaction ID Data Received for :::: {data}")
-        loan_repayment_function(data, 'OFFICE')
-        return ResponseHelper.success(message="Retry Disbursement Request Sent Successfully", status_code=200)
-    except Exception as e:
-        logger.error(f"Failed to call retry disbursement  {data}: {e}")
 
 
 @autocall_bp.route("/direct/loan", methods=["POST"])
 def direct_loan():
+    """
+    This endpoint is for directly calling the disbursement endpoint with a transaction id.
+    This is to be used in cases where the disbursement failed due to network issues
+    or other transient issues. It will call the disbursement endpoint
+    with the loan data for the loan with the given transaction id.
+    """
+    
     data = request.get_json()
     logger.info(f"Data received: {data}")
 
@@ -165,18 +202,12 @@ def direct_loan():
 
     loan = LoanService.get_loan_by_transaction_id(transaction_id=transaction_id)
     if not loan:
-        first_error = f"Loan with transaction id {transaction_id} does not exist"
-        logger.warning(first_error)
-        loan = LoanService.get_latest_loan_without_disburse_date()
-        if not loan:
-            logger.info(f"No loan found without disbursement date")
-            return jsonify({
-                "status": "error",
-                "first_error": first_error,
-                "message": f"No loan found without disbursement date"
-            }), 400
+        logger.warning(f"Loan with transaction id {transaction_id} does not exist")
+        return jsonify({
+            "status": "error",
+            "message": f"Loan with transaction id {transaction_id} does not exist"
+        }), 400
 
-    # Tried 2 method to get the loan record at this point
     loan_data = loan.to_dict()
 
     # Prevent double disbursement
@@ -195,19 +226,15 @@ def direct_loan():
         "productId": str(loan_data.get('productId', "")),
         "provideAmount": loan_data.get('currentLoanAmount'),
     }
-
-    
-    logger.info(f"Starting  disbursement thread for {data_to_process['transactionId']}")
-    thread = Thread(target=SimbrellaClient.disburse_loan, args=(data_to_process,))
-    thread.start()
-
-    return jsonify({
-        "status": "success",
-        "message": f"Loan disbursement has been queued for processing."
-    }), 200
+    response = SimbrellaClient.disburse_loan(data_to_process)
+    return response
     
 @autocall_bp.route("/direct/repayment", methods=["POST"])
 def direct_repayment():
+    """
+    This endpoint is for directly calling the collect loan endpoint with a transaction id.
+    """
+    
     data = request.get_json()
     logger.info(f"Data received: {data}")
 
@@ -289,90 +316,11 @@ def direct_repayment():
     response = SimbrellaClient.collect_loan_user_initiated(data_to_process)
     return response
 
-
-def loan_repayment_function(data, initiatedBy):
-    logger.info(f"Data Received Loan Repayment: {data} By {initiatedBy}")
-
-    REQUIRED_KEYS = ["transactionId"]
-
-    # Check for missing keys
-    missing_keys = [key for key in REQUIRED_KEYS if key not in data or data[key] is None]
-    if missing_keys:
-        logger.warning(f"Missing required keys: {missing_keys}")
-        return jsonify({
-            "status": "error",
-            "message": f"Missing required fields: {', '.join(missing_keys)}"
-        }), 400
-
-    # Check if the loan exists
-    logger.info(f"Checking if loan with transaction id {data['transactionId']} exists")
-    loan = LoanService.get_loan_by_transaction_id(transaction_id=data['transactionId'])
-    if not loan:
-        logger.info(f"Loan with transaction id {data['transactionId']} does not exist")
-        return jsonify({
-            "status": "error",
-            "message": f"Loan with transaction id {data['transactionId']} does not exist"
-        }), 400
-
-    loan_data = loan.to_dict()
-
-    # check if loan has been repaid
-    if loan_data.get("status") == LoanStatus.REPAID and loan_data.get("balance") <= 0:
-        logger.info(f"Loan with Id {loan_data.get('debtId')} has been repaid")
-        return jsonify({
-            "status": "error",
-            "message": f"loan with Id {loan_data.get('debtId')} has been repaid"
-        }), 400
-
-    repayment_data = {
-        "customerId": loan_data.get("customerId"),
-        "loanId": loan_data.get("debtId"),
-        "productId": loan_data.get("productId"),
-        "transactionId": loan_data.get("transactionId"),
-        "initiatedBy": initiatedBy,
-        "salaryAmount": 0,
-        "LoanStatus": loan_data.get("status"),
-    }
-
-    logger.info(f"Creating repayment with data: {repayment_data}")
-
-    try:
-        repayment = RepaymentService.create_repayment(repayment_data)
-        logger.info(f"Repayment created: {repayment}")
-    except Exception as e:
-        db.session.rollback()
-        logger.error(f"Repayment creation raised exception: {e}")
-        return jsonify({
-            "status": "error",
-            "message": "Failed to create repayment"
-        }), 500
-
-    if not repayment or (isinstance(repayment, dict) and "error" in repayment):
-        db.session.rollback()
-        logger.error(f"Repayment creation failed for loan ID {loan_data.get('debtId')}: {repayment}")
-
-    try:
-        if loan_data.get('status') == LoanStatus.ACTIVE:
-            LoanService.update_status(loan_id=loan_data.get('debtId'), status=LoanStatus.START_REPAY)
-    except Exception as e:
-        db.session.rollback()
-        logger.error(f"Failed to update loan status for loan ID {loan_data.get('debtId')}: {e}")
-    repayment_data_dict = repayment.to_dict()
-
-    data_to_process = {
-        "transactionId": repayment_data_dict['transactionId'],
-        "debtId": repayment_data_dict['loanId'],
-        "customerId": repayment_data_dict['customerId'],
-        "productId": repayment_data_dict['productId'],
-        "Id": repayment_data_dict['Id']
-    }
-
-    response = SimbrellaClient.collect_loan_user_initiated(data_to_process)
-    return response
-
-
 @autocall_bp.route("/refresh-verify-collection", methods=["GET"])
 def refresh_verify_collection():
+    """
+    This endpoint is for directly calling the verify collection endpoint.
+    """
     data = request.get_json()
     logger.info(f"Calling Verify Collection")
 
@@ -382,6 +330,11 @@ def refresh_verify_collection():
 
 @autocall_bp.route("/refresh-collection", methods=["GET"])
 def refresh_collection():
+    """
+    This endpoint is for directly calling the collect loan endpoint.
+    It will get the latest repayment with no repay date and call the collect loan endpoint with that repayment data.
+    This is to be used in cases where the collection failed due to network issues or other transient issues. It will call the collection endpoint
+    """
     #data = request.get_json()
     logger.info(f"Calling Collection ")
     #grab the last repayments with repay date is none
@@ -417,7 +370,11 @@ def payment_callback():
     return response
 
 @autocall_bp.route("/penal-charge", methods=["POST"])
+    
 def penal_charge():
+    """
+    This endpoint is for directly calling the penal charge endpoint.
+    """
     data = request.get_json()
     logger.info(f"Calling Penal Charge Endpoint")
 
@@ -428,9 +385,15 @@ def penal_charge():
         logger.error(f"Error in Penal Charge: {e}")
         return ResponseHelper.error("Penal charge failed")
 
-
 @autocall_bp.route("/analytic-salary-detect", methods=["POST"])
 def salary_detect():
+    """
+    Creates new salary data
+    Gets the salary list not yet processed
+    then, gets the loan list for each salary and 
+    creates repayments for each loan with the salary data, 
+    then calls the collect loan endpoint for each repayment created.
+    """
     payload = request.get_json()
     logger.info("Calling Salary Detect endpoint")
 
@@ -456,13 +419,11 @@ def salary_detect():
     logger.info("Finished processing List")
     return ResponseHelper.success([], "AutoCall Add Salary Successful")
 
-
 @autocall_bp.route("/analytic-salary-process", methods=["POST"])
 def salary_process():
     response = process_salary_list()
     return ResponseHelper.success([], "AutoCall Successful")
 
-
 def process_salary_list():
     # Step 1: Get all pending salaries
     pending_salaries = SalaryService.get_pending_salaries()
@@ -556,10 +517,9 @@ def process_salary_list():
 
     return ResponseHelper.success([], "Processed all pending salaries")
 
-
-
 @autocall_bp.route("/report", methods=["GET"])
 def report():
+    """This endpoint is for generating the report and sending the email."""
     try:
         report_data = get_report_data()
         logger.info(f"Generated report data: {report_data}")
@@ -571,29 +531,172 @@ def report():
     except Exception as e:
         logger.error(f"Error generating or sending report: {e}")
         return ResponseHelper.error("Failed to send report", status_code=500, error=str(e))
+
+@autocall_bp.route("/process-penal-charges", methods=["GET"])
+def process_penal_charges():
+    """
+    This endpoint is for processing penal charges for overdue loans schedule with grace period.
+    It will check for overdue loans, calculate the penal charge, 
+    create a new penal charge record, update the loan and repayment schedule with the new penal charge, and call the Simbrella endpoint to update the penal charge on their system.
+    """
     
+    try:
+        OVERDUE_GRACE_PERIOD_DAYS = settings.OVERDUE_GRACE_PERIOD_DAYS
+        OVERDUE_PROCESSING_LIST_LIMIT = settings.OVERDUE_PROCESSING_LIST_LIMIT
+        PENAL_CHARGE_MAXIMUM_COUNT = settings.PENAL_CHARGE_MAXIMUM_COUNT
+        PENAL_CHARGE_INTERVAL_DAYS = settings.PENAL_CHARGE_INTERVAL_DAYS
+
+        now = datetime.now(timezone.utc)
+
+        overdue_schedules = (
+            LoanRepaymentScheduleService
+            .get_overdue_repayment_schedule_with_grace_period(
+                OVERDUE_GRACE_PERIOD_DAYS,
+                OVERDUE_PROCESSING_LIST_LIMIT
+            )
+        )
+
+        logger.info(f"Found {len(overdue_schedules)} overdue loan schedule.")
+
+        if not overdue_schedules:
+            return ResponseHelper.success(
+                message="No overdue loan schedule found",
+                status_code=200
+            )
+
+        processed_loans = []
+
+        for schedule in overdue_schedules:
+
+            loan = LoanService.get_loan_by_loan_id(schedule.loan_id)
+
+            if not loan:
+                logger.info(f"Loan with id {schedule.loan_id} not found")
+                continue
+
+            penal_count = schedule.penal_count or 0
+
+            # MAX PENAL CHECK
+            if penal_count >= PENAL_CHARGE_MAXIMUM_COUNT:
+                logger.info(
+                    f"Penal count for schedule {schedule.id} has reached the maximum limit."
+                )
+                continue
+
+            # INTERVAL CHECK (PER SCHEDULE)
+            if schedule.last_penal_date:
+                 # ensure last_penal_date is timezone-aware
+                last_penal = schedule.last_penal_date
+                if last_penal.tzinfo is None:
+                    last_penal = last_penal.replace(tzinfo=timezone.utc)
+
+                next_allowed_date = last_penal + timedelta(days=PENAL_CHARGE_INTERVAL_DAYS)
+
+                if now < next_allowed_date:
+                    logger.info(
+                        f"Penal interval for schedule {schedule.id} has not passed yet."
+                    )
+                    continue
+
+            # NEXT PENAL NUMBER
+            next_penal_no = penal_count + 1
+
+            # CALCULATE PENAL
+            penal_amount = LoanRepaymentScheduleService.calculate_penal_charge(schedule)
+
+            # CREATE PENAL CHARGE
+            new_penal_charge = LoanChargesService.create_penal_charges_for_loan(
+                loan_id=schedule.loan_id,
+                transaction_id=schedule.transaction_id,
+                percent=settings.PENAL_CHARGE_PERCENTAGE,
+                penal_no=next_penal_no,
+                schedule_number=schedule.installment_number,
+                penal_amount=penal_amount
+            )
+
+            if not new_penal_charge:
+                logger.error(f"Failed to create penal charge for loan ID: {loan.id}")
+                continue
+
+            logger.info(f"Penal charge created: {new_penal_charge.to_dict()}")
+
+            # UPDATE SCHEDULE
+            LoanRepaymentScheduleService.apply_penal_to_schedule(
+                schedule.id,
+                penal_amount
+            )
+
+            logger.info(f"Penal charge applied to schedule {schedule.id}")
+
+            # UPDATE LOAN TOTAL
+            LoanService.apply_penal_to_loan(
+                loan.id,
+                penal_amount
+            )
+
+            logger.info(f"Penal charge applied to loan {loan.id}")
+
+            processed_loans.append(loan.to_dict())
+
+        return ResponseHelper.success(
+            message="Penal Charges Processed Successfully",
+            status_code=200,
+            data=processed_loans
+        )
+
+    except Exception as e:
+        logger.exception(f"Error processing penal charges: {e}")
+        return ResponseHelper.error(
+            "Failed to process penal charges",
+            status_code=500,
+            error=str(e)
+        )
 
 @autocall_bp.route("/overdue-loans", methods=["GET"])
 def overdue_loans():
+    """
+    This endpoint is for processing overdue loans.
+    It will get all active overdue loan schedules, 
+    and then for each loan schedule, it will create a repayment, update the loan status, and call the Simbrella endpoint to collect the loan.
+    """
+    
     try:
-        # Step 1: Get all overdue loans
-        overdue_loans = LoanRepaymentScheduleService.get_overdue_repayment_schedule()
+        # Step 1: Get all active overdue loans
+        overdue_loans = LoanRepaymentScheduleService.get_active_overdue_repayment_schedule()
         logger.info(f"Found {len(overdue_loans)} overdue loans.")
 
         if not overdue_loans:
+            logger.info("No overdue loans found.")
             return ResponseHelper.success(message="No overdue loans found", status_code=200)
+        #get batch size from settings
+        loan_delay_seconds = max(0, settings.OVERDUE_LOAN_DELAY_SECONDS)
+        batch_delay_seconds = max(0, settings.OVERDUE_LOAN_BATCH_DELAY_SECONDS)
+        batch_size = max(1, settings.OVERDUE_LOAN_BATCH_SIZE)
+
+        loan_chunks = list(CollectLoanHelper.chunk_list(overdue_loans, batch_size))
+        logger.info(f"Found {len(loan_chunks)} loan chunks to process.")
+
 
         # Step 2: Process each loan
-        for loan in overdue_loans:
-            process_overdue_loan(loan)
+        for chunk_index, loan_chunk in enumerate(loan_chunks):
+            logger.info(f"Processing chunk {chunk_index + 1} of {len(loan_chunks)} with {len(loan_chunk)} loans.")
+            for loan in loan_chunk:
+                try:
+                    process_overdue_loan(loan)
+                except Exception:
+                    logger.exception(f"Failed processing loan {loan.id}")
+                finally:
+                    time_module.sleep(loan_delay_seconds)
+            if chunk_index < len(loan_chunks) - 1:
+                logger.info(f"Waiting {batch_delay_seconds} seconds before processing next chunk...")
+                time_module.sleep(batch_delay_seconds)  # Delay between chunks
 
         return ResponseHelper.success(message="Processed overdue loans successfully", status_code=200)
 
     except Exception as e:
-        logger.error(f"Error fetching overdue loans: {e}")
+        logger.exception(f"Error fetching overdue loans: {e}")
         return ResponseHelper.error("Failed to fetch overdue loans", status_code=500, error=str(e))
 
-
 def process_overdue_loan(loan):
     """
     Handles repayment creation, loan status update, and Simbrella call
@@ -639,6 +742,7 @@ def process_overdue_loan(loan):
 
             # Update loan status
             try:
+                logger.info(f"Updating loan status for loan ID {loan.loan_id}")
                 LoanService.update_status(loan_id=loan.loan_id, status=LoanStatus.START_REPAY)
             except Exception as e:
                 db.session.rollback()
@@ -656,6 +760,7 @@ def process_overdue_loan(loan):
                 repayment_data["overdueLoanScheduleAmount"] = amount
                 repayment_data["overdueLoanScheduleId"] = loan.id
                 repayment_data["Id"] = repayment.id
+                logger.info(f"Calling Simbrella for with repayment data: {repayment_data}")
                 simbrella_response = SimbrellaClient.collect_loan_user_due_payment(repayment_data)
 
                 if isinstance(simbrella_response, tuple):

app/services/loan.py

@@ -58,7 +58,22 @@ class LoanService:
     @classmethod
     def set_disbursement_loan_description(cls,loan_id,description):
 
-        return Loan.set_disbursement_message(loan_id, result, description)
+        return Loan.set_disbursement_message(loan_id,  description)
+    
+    @classmethod
+    def get_failed_disbursements(cls):
+        """
+        Get all loans with failed disbursement.
+        """
+        return Loan.get_failed_disbursements()
+    
+    @classmethod
+    def update_status(cls, loan_id, status):
+        """
+        Update the status of the loan with the given loan_id.
+        """
+        # Retrieve loan
+        return Loan.update_status(loan_id, status)
 
 
     @classmethod
@@ -117,6 +132,9 @@ class LoanService:
         Get all overdue loans.
         """
         return Loan.get_overdue_loans()
+    @classmethod
+    def apply_penal_to_loan(cls,loan_id,penal_charge):
+        return Loan.apply_penal_to_loan(loan_id,penal_charge)
     @staticmethod
     def _update_loan_after_collection(loan, loan_data, updated_loan, amount_collected, data, response_message):
         if loan.balance is None or loan.balance <= 0:

app/services/loan_charge.py

@@ -0,0 +1,13 @@
+from app.models.loan_charge import LoanCharge
+
+class LoanChargesService:
+    @classmethod
+    def create_penal_charges_for_loan(cls, loan_id, transaction_id, percent, penal_no, schedule_number, penal_amount=0.0,):
+        return LoanCharge.create_penal_charges_for_loan(loan_id, transaction_id, percent, penal_no,schedule_number, penal_amount)
+    @classmethod
+    def get_last_penal_no(cls,loan_id):
+        return LoanCharge.get_last_penal_no(loan_id)
+
+    @classmethod
+    def get_penal_charges_by_loan_id(cls,loan_id):
+        return LoanCharge.get_penal_charges_by_loan_id(loan_id)

app/services/loan_repayment_schedule.py

@@ -13,6 +13,12 @@ class LoanRepaymentScheduleService:
     def get_overdue_repayment_schedule(cls):
         return LoanRepaymentSchedule.get_overdue_repayment_schedule()
     @classmethod
+    def get_active_overdue_repayment_schedule(cls):
+        return LoanRepaymentSchedule.get_active_overdue_repayment_schedule()
+    @classmethod
+    def get_partially_paid_overdue_repayment_schedule(cls):
+        return LoanRepaymentSchedule.get_partially_paid_overdue_repayment_schedule()
+    @classmethod
     def get_repayment_schedule_by_id_and_transaction_id(cls, id, transaction_id):
         return LoanRepaymentSchedule.get_repayment_schedule_by_id_and_transaction_id(id, transaction_id)
 
@@ -26,13 +32,24 @@ class LoanRepaymentScheduleService:
         Update repayment schedule status.
         """
         return LoanRepaymentSchedule.update_repayment_schedule_status(schedule_id)
-
+    @classmethod
+    def update_repayment_schedule_status_to_active(cls, schedule_id):
+        """
+        Update repayment schedule status.
+        """
+        return LoanRepaymentSchedule.update_repayment_schedule_status_to_active(schedule_id)
     @classmethod
     def update_repayment_schedule_balance(cls, schedule_id, amount_collected):
         """
         Update repayment schedule balance.
         """
         return LoanRepaymentSchedule.update_repayment_schedule_balance(schedule_id, amount_collected)
+    @classmethod
+    def calculate_penal_charge(cls, schedule):
+        """
+        Calculate penal charge for a repayment schedule.
+        """
+        return LoanRepaymentSchedule.calculate_penal_charge(schedule)
     
     @classmethod
     def update_repayment_schedule_description(cls, schedule_id, description):
@@ -41,7 +58,16 @@ class LoanRepaymentScheduleService:
         """
         return LoanRepaymentSchedule.update_repayment_schedule_description(schedule_id, description)
     
+    @classmethod
+    def get_overdue_repayment_schedule_with_grace_period(cls, grace_period_days, limit=None):
+        return LoanRepaymentSchedule.get_overdue_repayment_schedule_with_grace_period(grace_period_days, limit=limit)
     
+    @classmethod
+    def apply_penal_to_schedule(cls, schedule_id, penal_amount):
+        """
+        Apply penal charge to a repayment schedule.
+        """
+        return LoanRepaymentSchedule.apply_penal_to_schedule(schedule_id, penal_amount)
     @staticmethod
     def handle_schedule_updates(updated_loan, data, amount_collected, message, loan_data):
         """

openapi.yml

@@ -110,6 +110,12 @@ paths:
       responses:
         200:
           description: A successful response
+  /autocall/retry-failed-disbursements:
+    get:
+      summary: Retry failed disbursements
+      responses:
+        200:
+          description: A successful response
   /autocall/refresh-disbursement:
     get:
       summary: Refresh the disbursement
@@ -216,6 +222,12 @@ paths:
       responses:
         200:
           description: A successful response
+  /autocall/process-penal-charges:
+    get:
+      summary: Get all overdue loans with grace period
+      responses:
+        200:
+          description: A successful response
   /autocall/direct/loan:
     post:
       summary: Direct call for loan disbursement