From 8236462b83c00dba8d67ee3d9a9cfeaa1f2418e9 Mon Sep 17 00:00:00 2001 From: VivianDee <115420678+VivianDee@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:02:36 +0100 Subject: [PATCH] [add]: database connection --- app.log | 219 ++++++++++++++++++ app/config.py | 16 +- app/models/batch_results.py | 16 +- app/models/customer_account_transaction_hx.py | 100 ++++++++ app/models/db_operations.py | 2 +- 5 files changed, 336 insertions(+), 17 deletions(-) create mode 100644 app/models/customer_account_transaction_hx.py diff --git a/app.log b/app.log index d1fa785..a55a9c9 100644 --- a/app.log +++ b/app.log @@ -188,3 +188,222 @@ 2025-09-10 10:35:40,300 - INFO - Loading chunk starting at offset 900000 2025-09-10 10:36:05,763 - INFO - Loading chunk starting at offset 910000 2025-09-10 10:36:15,199 - INFO - Loading chunk starting at offset 920000 +2025-09-10 10:36:46,036 - INFO - Loading chunk starting at offset 930000 +2025-09-10 10:36:53,221 - INFO - Loading chunk starting at offset 940000 +2025-09-10 10:37:06,754 - INFO - Loading chunk starting at offset 950000 +2025-09-10 10:37:15,558 - INFO - Loading chunk starting at offset 960000 +2025-09-10 10:37:21,798 - INFO - [2025-09-10 10:37:21] Detecting salary... +2025-09-10 10:37:22,954 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:37:22,955 - INFO - [2025-09-10 10:37:22] Salary detection complete +2025-09-10 10:37:29,088 - INFO - Loading chunk starting at offset 970000 +2025-09-10 10:37:54,730 - INFO - Loading chunk starting at offset 980000 +2025-09-10 10:38:13,667 - INFO - Loading chunk starting at offset 990000 +2025-09-10 10:38:27,880 - INFO - Loading chunk starting at offset 1000000 +2025-09-10 10:39:07,546 - INFO - Loading chunk starting at offset 1010000 +2025-09-10 10:39:17,369 - INFO - Loading chunk starting at offset 1020000 +2025-09-10 10:39:23,176 - INFO - [2025-09-10 10:39:23] Detecting salary... +2025-09-10 10:39:24,444 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:39:24,445 - INFO - [2025-09-10 10:39:24] Salary detection complete +2025-09-10 10:39:26,048 - INFO - Loading chunk starting at offset 1030000 +2025-09-10 10:39:45,494 - INFO - Loading chunk starting at offset 1040000 +2025-09-10 10:39:50,480 - INFO - Loading chunk starting at offset 1050000 +2025-09-10 10:39:54,793 - INFO - Loading chunk starting at offset 1060000 +2025-09-10 10:39:59,415 - INFO - Loading chunk starting at offset 1070000 +2025-09-10 10:40:04,306 - INFO - Loading chunk starting at offset 1080000 +2025-09-10 10:40:10,517 - INFO - Loading chunk starting at offset 1090000 +2025-09-10 10:40:15,671 - INFO - Loading chunk starting at offset 1100000 +2025-09-10 10:40:21,662 - INFO - Loading chunk starting at offset 1110000 +2025-09-10 10:40:30,551 - INFO - Loading chunk starting at offset 1120000 +2025-09-10 10:40:47,391 - INFO - Loading chunk starting at offset 1130000 +2025-09-10 10:40:55,609 - INFO - Loading chunk starting at offset 1140000 +2025-09-10 10:41:04,496 - INFO - Loading chunk starting at offset 1150000 +2025-09-10 10:41:14,608 - INFO - Loading chunk starting at offset 1160000 +2025-09-10 10:41:24,447 - INFO - [2025-09-10 10:41:24] Detecting salary... +2025-09-10 10:41:28,537 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:41:28,548 - INFO - [2025-09-10 10:41:28] Salary detection complete +2025-09-10 10:42:00,867 - INFO - Loading chunk starting at offset 1170000 +2025-09-10 10:42:13,069 - INFO - Loading chunk starting at offset 1180000 +2025-09-10 10:42:21,593 - INFO - Loading chunk starting at offset 1190000 +2025-09-10 10:42:32,011 - INFO - Loading chunk starting at offset 1200000 +2025-09-10 10:42:37,982 - INFO - Loading chunk starting at offset 1210000 +2025-09-10 10:42:45,458 - INFO - Loading chunk starting at offset 1220000 +2025-09-10 10:42:54,545 - INFO - Loading chunk starting at offset 1230000 +2025-09-10 10:43:15,705 - INFO - Loading chunk starting at offset 1240000 +2025-09-10 10:43:28,549 - INFO - [2025-09-10 10:43:28] Detecting salary... +2025-09-10 10:43:31,640 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:43:31,641 - INFO - [2025-09-10 10:43:31] Salary detection complete +2025-09-10 10:43:43,344 - INFO - Loading chunk starting at offset 1250000 +2025-09-10 10:43:50,004 - INFO - Loading chunk starting at offset 1260000 +2025-09-10 10:43:59,430 - INFO - Loading chunk starting at offset 1270000 +2025-09-10 10:44:07,478 - INFO - Loading chunk starting at offset 1280000 +2025-09-10 10:44:18,927 - INFO - Loading chunk starting at offset 1290000 +2025-09-10 10:44:28,523 - INFO - Loading chunk starting at offset 1300000 +2025-09-10 10:44:38,629 - INFO - Loading chunk starting at offset 1310000 +2025-09-10 10:44:56,977 - INFO - Loading chunk starting at offset 1320000 +2025-09-10 10:45:09,203 - INFO - Loading chunk starting at offset 1330000 +2025-09-10 10:45:18,657 - INFO - Loading chunk starting at offset 1340000 +2025-09-10 10:45:27,201 - INFO - Loading chunk starting at offset 1350000 +2025-09-10 10:45:31,641 - INFO - [2025-09-10 10:45:31] Detecting salary... +2025-09-10 10:45:32,261 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:45:32,262 - INFO - [2025-09-10 10:45:32] Salary detection complete +2025-09-10 10:45:35,308 - INFO - Loading chunk starting at offset 1360000 +2025-09-10 10:45:44,783 - INFO - Loading chunk starting at offset 1370000 +2025-09-10 10:45:52,048 - INFO - Loading chunk starting at offset 1380000 +2025-09-10 10:45:58,491 - INFO - Loading chunk starting at offset 1390000 +2025-09-10 10:46:13,270 - INFO - Loading chunk starting at offset 1400000 +2025-09-10 10:46:25,221 - INFO - Loading chunk starting at offset 1410000 +2025-09-10 10:46:34,358 - INFO - Loading chunk starting at offset 1420000 +2025-09-10 10:46:38,777 - INFO - Loading chunk starting at offset 1430000 +2025-09-10 10:46:42,534 - INFO - Loading chunk starting at offset 1440000 +2025-09-10 10:46:46,147 - INFO - Loading chunk starting at offset 1450000 +2025-09-10 10:46:50,727 - INFO - Loading chunk starting at offset 1460000 +2025-09-10 10:46:55,939 - INFO - Loading chunk starting at offset 1470000 +2025-09-10 10:47:01,696 - INFO - Loading chunk starting at offset 1480000 +2025-09-10 10:47:07,411 - INFO - Loading chunk starting at offset 1490000 +2025-09-10 10:47:13,449 - INFO - Loading chunk starting at offset 1500000 +2025-09-10 10:47:23,058 - INFO - Loading chunk starting at offset 1510000 +2025-09-10 10:47:30,097 - INFO - Loading chunk starting at offset 1520000 +2025-09-10 10:47:32,259 - INFO - [2025-09-10 10:47:32] Detecting salary... +2025-09-10 10:47:32,923 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:47:32,924 - INFO - [2025-09-10 10:47:32] Salary detection complete +2025-09-10 10:47:36,423 - INFO - Loading chunk starting at offset 1530000 +2025-09-10 10:47:45,086 - INFO - Loading chunk starting at offset 1540000 +2025-09-10 10:47:57,079 - INFO - Loading chunk starting at offset 1550000 +2025-09-10 10:48:19,741 - INFO - Loading chunk starting at offset 1560000 +2025-09-10 10:48:41,300 - INFO - Loading chunk starting at offset 1570000 +2025-09-10 10:48:51,349 - INFO - Loading chunk starting at offset 1580000 +2025-09-10 10:48:58,892 - INFO - Loading chunk starting at offset 1590000 +2025-09-10 10:49:04,857 - INFO - Loading chunk starting at offset 1600000 +2025-09-10 10:49:10,299 - INFO - Loading chunk starting at offset 1610000 +2025-09-10 10:49:16,650 - INFO - Loading chunk starting at offset 1620000 +2025-09-10 10:49:23,107 - INFO - Loading chunk starting at offset 1630000 +2025-09-10 10:49:32,320 - INFO - Loading chunk starting at offset 1640000 +2025-09-10 10:49:32,927 - INFO - [2025-09-10 10:49:32] Detecting salary... +2025-09-10 10:49:33,484 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:49:33,485 - INFO - [2025-09-10 10:49:33] Salary detection complete +2025-09-10 10:49:40,707 - INFO - Loading chunk starting at offset 1650000 +2025-09-10 10:49:50,677 - INFO - Loading chunk starting at offset 1660000 +2025-09-10 10:49:59,664 - INFO - Loading chunk starting at offset 1670000 +2025-09-10 10:50:07,452 - INFO - Loading chunk starting at offset 1680000 +2025-09-10 10:50:13,583 - INFO - Loading chunk starting at offset 1690000 +2025-09-10 10:50:20,858 - INFO - Loading chunk starting at offset 1700000 +2025-09-10 10:50:27,519 - INFO - Loading chunk starting at offset 1710000 +2025-09-10 10:50:34,012 - INFO - Loading chunk starting at offset 1720000 +2025-09-10 10:50:39,628 - INFO - Loading chunk starting at offset 1730000 +2025-09-10 10:50:50,012 - INFO - Loading chunk starting at offset 1740000 +2025-09-10 10:51:06,647 - INFO - Loading chunk starting at offset 1750000 +2025-09-10 10:51:17,401 - INFO - Loading chunk starting at offset 1760000 +2025-09-10 10:51:26,360 - INFO - Loading chunk starting at offset 1770000 +2025-09-10 10:51:33,485 - INFO - [2025-09-10 10:51:33] Detecting salary... +2025-09-10 10:51:55,392 - INFO - Initializing pipeline... +2025-09-10 10:51:55,395 - INFO - [2025-09-10 10:51:55] Detecting salary... +2025-09-10 10:51:55,395 - INFO - Started autonomous salary detection loop. +2025-09-10 10:51:55,409 - INFO - Server running on hostname: 22bad35c69c3 +2025-09-10 10:51:55,416 - INFO - Server IP address: 172.25.0.2 +2025-09-10 10:51:55,424 - INFO - Server is accessible at: +2025-09-10 10:51:55,426 - INFO - - http://localhost:8000 +2025-09-10 10:51:55,444 - INFO - - http://127.0.0.1:8000 +2025-09-10 10:51:55,455 - INFO - - http://172.25.0.2:8000 +2025-09-10 10:51:55,456 - INFO - Pipeline initialized successfully +2025-09-10 10:51:56,024 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:51:56,024 - INFO - [2025-09-10 10:51:56] Salary detection complete +2025-09-10 10:52:36,450 - INFO - Initializing SalaryAnalyticsPipeline +2025-09-10 10:52:36,451 - INFO - Starting data loading process +2025-09-10 10:52:36,452 - INFO - No database connection. Attempting to connect... +2025-09-10 10:52:36,452 - INFO - Attempting to connect to database... +2025-09-10 10:52:36,538 - ERROR - Error connecting to database: No module named 'oracledb' +2025-09-10 10:52:36,539 - ERROR - Failed to establish database connection +2025-09-10 10:52:36,539 - ERROR - Failed to load data +2025-09-10 10:52:36,540 - ERROR - Failed to load data +2025-09-10 10:52:36,541 - INFO - Load data endpoint failed after 0.09 seconds +2025-09-10 10:52:36,541 - ERROR - Error loading data: 500: Failed to load data +2025-09-10 10:52:36,542 - INFO - Load data endpoint failed after 0.09 seconds +2025-09-10 10:52:58,149 - INFO - Shutting down Salary Analytics API... +2025-09-10 10:53:10,697 - INFO - generated new fontManager +2025-09-10 10:53:16,039 - INFO - Initializing pipeline... +2025-09-10 10:53:16,041 - INFO - [2025-09-10 10:53:16] Detecting salary... +2025-09-10 10:53:16,042 - INFO - Started autonomous salary detection loop. +2025-09-10 10:53:16,055 - INFO - Server running on hostname: 1c5d2376fb2a +2025-09-10 10:53:16,056 - INFO - Server IP address: 172.25.0.2 +2025-09-10 10:53:16,057 - INFO - Server is accessible at: +2025-09-10 10:53:16,058 - INFO - - http://localhost:8000 +2025-09-10 10:53:16,059 - INFO - - http://127.0.0.1:8000 +2025-09-10 10:53:16,060 - INFO - - http://172.25.0.2:8000 +2025-09-10 10:53:16,062 - INFO - Pipeline initialized successfully +2025-09-10 10:53:16,812 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: { + "data": [], + "error": {}, + "message": "AutoCall Add Salary Successful", + "status": true, + "statusCode": 200 +} + +2025-09-10 10:53:16,813 - INFO - [2025-09-10 10:53:16] Salary detection complete +2025-09-10 10:53:25,477 - INFO - Initializing SalaryAnalyticsPipeline +2025-09-10 10:53:25,482 - INFO - Starting data loading process +2025-09-10 10:53:25,483 - INFO - No database connection. Attempting to connect... +2025-09-10 10:53:25,484 - INFO - Attempting to connect to database... +2025-09-10 10:53:25,679 - ERROR - Error connecting to database: No module named 'oracledb' +2025-09-10 10:53:25,679 - ERROR - Failed to establish database connection +2025-09-10 10:53:25,680 - ERROR - Failed to load data +2025-09-10 10:53:25,680 - ERROR - Failed to load data +2025-09-10 10:53:25,681 - INFO - Load data endpoint failed after 0.20 seconds +2025-09-10 10:53:25,682 - ERROR - Error loading data: 500: Failed to load data +2025-09-10 10:53:25,682 - INFO - Load data endpoint failed after 0.20 seconds +2025-09-10 10:53:46,324 - INFO - Shutting down Salary Analytics API... diff --git a/app/config.py b/app/config.py index 2d28031..1fbc04f 100644 --- a/app/config.py +++ b/app/config.py @@ -34,19 +34,19 @@ DB_CONFIG = { } -# DNS = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={DB_CONFIG['host']})(PORT={DB_CONFIG['port']}))(CONNECT_DATA=(SID={DB_CONFIG['sid']})))" +DNS = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={DB_CONFIG['host']})(PORT={DB_CONFIG['port']}))(CONNECT_DATA=(SID={DB_CONFIG['sid']})))" # Database Connection -# SQLALCHEMY_DATABASE_URI_INTERNAL = (f"oracle+oracledb://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DNS}") -# SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI_FULL", SQLALCHEMY_DATABASE_URI_INTERNAL) +SQLALCHEMY_DATABASE_URI_INTERNAL = (f"oracle+oracledb://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DNS}") +SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI_FULL", SQLALCHEMY_DATABASE_URI_INTERNAL) -#SQLALCHEMY_DATABASE_URI_FULL = 'oracle+oracledb://FIRSTADVSTG:Pchanged_56789@10.2.110.30:1521/?service_name=firstadv' +# SQLALCHEMY_DATABASE_URI_FULL = 'oracle+oracledb://FIRSTADVSTG:Pchanged_56789@10.2.110.30:1521/?service_name=firstadv' # SQLAlchemy Configuration -SQLALCHEMY_DATABASE_URI = ( - f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@" - f"{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}" -) +# SQLALCHEMY_DATABASE_URI = ( +# f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@" +# f"{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}" +# ) SQLALCHEMY_TRACK_MODIFICATIONS = False diff --git a/app/models/batch_results.py b/app/models/batch_results.py index e65cce0..fcfdd91 100644 --- a/app/models/batch_results.py +++ b/app/models/batch_results.py @@ -24,7 +24,7 @@ class BatchResult(db.Model): @classmethod - def save_batch(cls, session: Session, batch_number, total_batches, results_df, status="success"): + def save_batch(cls, batch_number, total_batches, results_df, status="success"): """Save batch results into DB using ORM bulk insert.""" try: results_df["batch_number"] = batch_number @@ -42,21 +42,21 @@ class BatchResult(db.Model): for row in results_df.to_dict("records") ] - session.bulk_save_objects(records) - session.commit() + db.session.bulk_save_objects(records) + db.session.commit() logger.info(f"Saved batch {batch_number} successfully.") return True except Exception as e: - session.rollback() + db.session.rollback() logger.error(f"Error saving batch {batch_number}: {str(e)}") return False @classmethod - def get_batch_status(cls, session: Session, batch_number: int): + def get_batch_status(cls, batch_number: int): """Return summary info about one batch.""" try: result = ( - session.query( + db.session.query( cls.batch_number, cls.total_batches, cls.processed_at, @@ -75,11 +75,11 @@ class BatchResult(db.Model): return None @classmethod - def get_all_batches(cls, session: Session): + def get_all_batches(cls): """Return summaries for all batches.""" try: results = ( - session.query( + db.session.query( cls.batch_number, cls.total_batches, cls.processed_at, diff --git a/app/models/customer_account_transaction_hx.py b/app/models/customer_account_transaction_hx.py new file mode 100644 index 0000000..ac3e006 --- /dev/null +++ b/app/models/customer_account_transaction_hx.py @@ -0,0 +1,100 @@ +from venv import logger +from sqlalchemy import Column, Integer, String, Float, DateTime, ForeignKey +from sqlalchemy.orm import relationship +from app.extensions import db +import pandas as pd + +class CustomerAccountTransactionHx(db.Model): + __tablename__ = "customer_account_transaction_hx" + + id = Column(Integer, primary_key=True, autoincrement=True) + accountid = Column(String(64), nullable=False, index=True) + trx_type = Column(String(50), nullable=False) + amount = Column(Float, nullable=False) + description = Column(String(255)) + customer_id = Column(String(64)) + trx_start_date = Column(DateTime, nullable=False) + trx_end_date = Column(DateTime) + is_salary_related = Column(Integer, default=0) + is_consistent_amount = Column(Integer, default=0) + is_salary_type = Column(Integer, default=0) + + + + @classmethod + def get_all(cls): + """Fetch all transactions.""" + return db.session.query(cls).all() + + @classmethod + def get_rows_count(cls): + """Return total number of transaction rows.""" + try: + count = db.session.query(db.func.count(cls.id)).scalar() + return count + except Exception as e: + logger.error(f"Error getting row count: {str(e)}") + return None + + @classmethod + def get_by_account(cls, accountid: str): + """Fetch transactions for a given account.""" + return db.session.query(cls).filter_by(accountid=accountid).all() + + @classmethod + def get_accounts(cls, limit=None): + """Fetch distinct account IDs.""" + query = db.session.query(cls.accountid).distinct() + if limit: + query = query.limit(limit) + return [row.accountid for row in query.all()] + + @classmethod + def insert_transaction(cls, **kwargs): + """Insert a new transaction.""" + trx = cls(**kwargs) + try: + db.session.add(trx) + db.session.commit() + except Exception as e: + logger.error(f"Error inserting transaction: {str(e)}") + return None + return trx + + @classmethod + def bulk_insert(cls, transactions: list[dict]): + """Insert multiple transactions at once.""" + objs = [cls(**trx) for trx in transactions] + + try: + db.session.bulk_save_objects(objs) + db.session.commit() + except Exception as e: + logger.error(f"Error in bulk insert: {str(e)}") + return None + return objs + + @classmethod + def get_transactions_df(cls, accountids: list[str] = None): + """Return a Pandas DataFrame for ML model preparation.""" + query = db.session.query(cls) + if accountids: + query = query.filter(cls.accountid.in_(accountids)) + rows = query.all() + + + df = pd.DataFrame([{ + "id": trx.id, + "accountid": trx.accountid, + "trx_type": trx.trx_type, + "amount": trx.amount, + "description": trx.description, + "customer_id": trx.customer_id, + "trx_start_date": trx.trx_start_date, + "trx_end_date": trx.trx_end_date, + "is_salary_related": trx.is_salary_related, + "is_consistent_amount": trx.is_consistent_amount, + "is_salary_type": trx.is_salary_type, + } for trx in rows]) + + return df diff --git a/app/models/db_operations.py b/app/models/db_operations.py index 0517dee..6a20389 100644 --- a/app/models/db_operations.py +++ b/app/models/db_operations.py @@ -3,7 +3,7 @@ Database operations module for salary analytics. """ from sqlalchemy import text -from ..config import BATCH_RESULTS_TABLE +from app.config import BATCH_RESULTS_TABLE from datetime import datetime from app.utils.logger import logger