DigiFi/digifi-Analytics
1
0
Fork 0
Code Issues Pull Requests 1 Releases Activity

[add]: database connection

Browse Source
This commit is contained in:
VivianDee
2025-09-10 12:02:36 +01:00
parent 95732a470d
commit 8236462b83
5 changed files with 336 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.log
+219
View File
@@ -188,3 +188,222 @@
2025-09-10 10:35:40,300 - INFO - Loading chunk starting at offset 900000
2025-09-10 10:36:05,763 - INFO - Loading chunk starting at offset 910000
2025-09-10 10:36:15,199 - INFO - Loading chunk starting at offset 920000
2025-09-10 10:36:46,036 - INFO - Loading chunk starting at offset 930000
2025-09-10 10:36:53,221 - INFO - Loading chunk starting at offset 940000
2025-09-10 10:37:06,754 - INFO - Loading chunk starting at offset 950000
2025-09-10 10:37:15,558 - INFO - Loading chunk starting at offset 960000
2025-09-10 10:37:21,798 - INFO - [2025-09-10 10:37:21] Detecting salary...
2025-09-10 10:37:22,954 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:37:22,955 - INFO - [2025-09-10 10:37:22] Salary detection complete
2025-09-10 10:37:29,088 - INFO - Loading chunk starting at offset 970000
2025-09-10 10:37:54,730 - INFO - Loading chunk starting at offset 980000
2025-09-10 10:38:13,667 - INFO - Loading chunk starting at offset 990000
2025-09-10 10:38:27,880 - INFO - Loading chunk starting at offset 1000000
2025-09-10 10:39:07,546 - INFO - Loading chunk starting at offset 1010000
2025-09-10 10:39:17,369 - INFO - Loading chunk starting at offset 1020000
2025-09-10 10:39:23,176 - INFO - [2025-09-10 10:39:23] Detecting salary...
2025-09-10 10:39:24,444 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:39:24,445 - INFO - [2025-09-10 10:39:24] Salary detection complete
2025-09-10 10:39:26,048 - INFO - Loading chunk starting at offset 1030000
2025-09-10 10:39:45,494 - INFO - Loading chunk starting at offset 1040000
2025-09-10 10:39:50,480 - INFO - Loading chunk starting at offset 1050000
2025-09-10 10:39:54,793 - INFO - Loading chunk starting at offset 1060000
2025-09-10 10:39:59,415 - INFO - Loading chunk starting at offset 1070000
2025-09-10 10:40:04,306 - INFO - Loading chunk starting at offset 1080000
2025-09-10 10:40:10,517 - INFO - Loading chunk starting at offset 1090000
2025-09-10 10:40:15,671 - INFO - Loading chunk starting at offset 1100000
2025-09-10 10:40:21,662 - INFO - Loading chunk starting at offset 1110000
2025-09-10 10:40:30,551 - INFO - Loading chunk starting at offset 1120000
2025-09-10 10:40:47,391 - INFO - Loading chunk starting at offset 1130000
2025-09-10 10:40:55,609 - INFO - Loading chunk starting at offset 1140000
2025-09-10 10:41:04,496 - INFO - Loading chunk starting at offset 1150000
2025-09-10 10:41:14,608 - INFO - Loading chunk starting at offset 1160000
2025-09-10 10:41:24,447 - INFO - [2025-09-10 10:41:24] Detecting salary...
2025-09-10 10:41:28,537 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:41:28,548 - INFO - [2025-09-10 10:41:28] Salary detection complete
2025-09-10 10:42:00,867 - INFO - Loading chunk starting at offset 1170000
2025-09-10 10:42:13,069 - INFO - Loading chunk starting at offset 1180000
2025-09-10 10:42:21,593 - INFO - Loading chunk starting at offset 1190000
2025-09-10 10:42:32,011 - INFO - Loading chunk starting at offset 1200000
2025-09-10 10:42:37,982 - INFO - Loading chunk starting at offset 1210000
2025-09-10 10:42:45,458 - INFO - Loading chunk starting at offset 1220000
2025-09-10 10:42:54,545 - INFO - Loading chunk starting at offset 1230000
2025-09-10 10:43:15,705 - INFO - Loading chunk starting at offset 1240000
2025-09-10 10:43:28,549 - INFO - [2025-09-10 10:43:28] Detecting salary...
2025-09-10 10:43:31,640 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:43:31,641 - INFO - [2025-09-10 10:43:31] Salary detection complete
2025-09-10 10:43:43,344 - INFO - Loading chunk starting at offset 1250000
2025-09-10 10:43:50,004 - INFO - Loading chunk starting at offset 1260000
2025-09-10 10:43:59,430 - INFO - Loading chunk starting at offset 1270000
2025-09-10 10:44:07,478 - INFO - Loading chunk starting at offset 1280000
2025-09-10 10:44:18,927 - INFO - Loading chunk starting at offset 1290000
2025-09-10 10:44:28,523 - INFO - Loading chunk starting at offset 1300000
2025-09-10 10:44:38,629 - INFO - Loading chunk starting at offset 1310000
2025-09-10 10:44:56,977 - INFO - Loading chunk starting at offset 1320000
2025-09-10 10:45:09,203 - INFO - Loading chunk starting at offset 1330000
2025-09-10 10:45:18,657 - INFO - Loading chunk starting at offset 1340000
2025-09-10 10:45:27,201 - INFO - Loading chunk starting at offset 1350000
2025-09-10 10:45:31,641 - INFO - [2025-09-10 10:45:31] Detecting salary...
2025-09-10 10:45:32,261 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:45:32,262 - INFO - [2025-09-10 10:45:32] Salary detection complete
2025-09-10 10:45:35,308 - INFO - Loading chunk starting at offset 1360000
2025-09-10 10:45:44,783 - INFO - Loading chunk starting at offset 1370000
2025-09-10 10:45:52,048 - INFO - Loading chunk starting at offset 1380000
2025-09-10 10:45:58,491 - INFO - Loading chunk starting at offset 1390000
2025-09-10 10:46:13,270 - INFO - Loading chunk starting at offset 1400000
2025-09-10 10:46:25,221 - INFO - Loading chunk starting at offset 1410000
2025-09-10 10:46:34,358 - INFO - Loading chunk starting at offset 1420000
2025-09-10 10:46:38,777 - INFO - Loading chunk starting at offset 1430000
2025-09-10 10:46:42,534 - INFO - Loading chunk starting at offset 1440000
2025-09-10 10:46:46,147 - INFO - Loading chunk starting at offset 1450000
2025-09-10 10:46:50,727 - INFO - Loading chunk starting at offset 1460000
2025-09-10 10:46:55,939 - INFO - Loading chunk starting at offset 1470000
2025-09-10 10:47:01,696 - INFO - Loading chunk starting at offset 1480000
2025-09-10 10:47:07,411 - INFO - Loading chunk starting at offset 1490000
2025-09-10 10:47:13,449 - INFO - Loading chunk starting at offset 1500000
2025-09-10 10:47:23,058 - INFO - Loading chunk starting at offset 1510000
2025-09-10 10:47:30,097 - INFO - Loading chunk starting at offset 1520000
2025-09-10 10:47:32,259 - INFO - [2025-09-10 10:47:32] Detecting salary...
2025-09-10 10:47:32,923 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:47:32,924 - INFO - [2025-09-10 10:47:32] Salary detection complete
2025-09-10 10:47:36,423 - INFO - Loading chunk starting at offset 1530000
2025-09-10 10:47:45,086 - INFO - Loading chunk starting at offset 1540000
2025-09-10 10:47:57,079 - INFO - Loading chunk starting at offset 1550000
2025-09-10 10:48:19,741 - INFO - Loading chunk starting at offset 1560000
2025-09-10 10:48:41,300 - INFO - Loading chunk starting at offset 1570000
2025-09-10 10:48:51,349 - INFO - Loading chunk starting at offset 1580000
2025-09-10 10:48:58,892 - INFO - Loading chunk starting at offset 1590000
2025-09-10 10:49:04,857 - INFO - Loading chunk starting at offset 1600000
2025-09-10 10:49:10,299 - INFO - Loading chunk starting at offset 1610000
2025-09-10 10:49:16,650 - INFO - Loading chunk starting at offset 1620000
2025-09-10 10:49:23,107 - INFO - Loading chunk starting at offset 1630000
2025-09-10 10:49:32,320 - INFO - Loading chunk starting at offset 1640000
2025-09-10 10:49:32,927 - INFO - [2025-09-10 10:49:32] Detecting salary...
2025-09-10 10:49:33,484 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:49:33,485 - INFO - [2025-09-10 10:49:33] Salary detection complete
2025-09-10 10:49:40,707 - INFO - Loading chunk starting at offset 1650000
2025-09-10 10:49:50,677 - INFO - Loading chunk starting at offset 1660000
2025-09-10 10:49:59,664 - INFO - Loading chunk starting at offset 1670000
2025-09-10 10:50:07,452 - INFO - Loading chunk starting at offset 1680000
2025-09-10 10:50:13,583 - INFO - Loading chunk starting at offset 1690000
2025-09-10 10:50:20,858 - INFO - Loading chunk starting at offset 1700000
2025-09-10 10:50:27,519 - INFO - Loading chunk starting at offset 1710000
2025-09-10 10:50:34,012 - INFO - Loading chunk starting at offset 1720000
2025-09-10 10:50:39,628 - INFO - Loading chunk starting at offset 1730000
2025-09-10 10:50:50,012 - INFO - Loading chunk starting at offset 1740000
2025-09-10 10:51:06,647 - INFO - Loading chunk starting at offset 1750000
2025-09-10 10:51:17,401 - INFO - Loading chunk starting at offset 1760000
2025-09-10 10:51:26,360 - INFO - Loading chunk starting at offset 1770000
2025-09-10 10:51:33,485 - INFO - [2025-09-10 10:51:33] Detecting salary...
2025-09-10 10:51:55,392 - INFO - Initializing pipeline...
2025-09-10 10:51:55,395 - INFO - [2025-09-10 10:51:55] Detecting salary...
2025-09-10 10:51:55,395 - INFO - Started autonomous salary detection loop.
2025-09-10 10:51:55,409 - INFO - Server running on hostname: 22bad35c69c3
2025-09-10 10:51:55,416 - INFO - Server IP address: 172.25.0.2
2025-09-10 10:51:55,424 - INFO - Server is accessible at:
2025-09-10 10:51:55,426 - INFO - - http://localhost:8000
2025-09-10 10:51:55,444 - INFO - - http://127.0.0.1:8000
2025-09-10 10:51:55,455 - INFO - - http://172.25.0.2:8000
2025-09-10 10:51:55,456 - INFO - Pipeline initialized successfully
2025-09-10 10:51:56,024 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:51:56,024 - INFO - [2025-09-10 10:51:56] Salary detection complete
2025-09-10 10:52:36,450 - INFO - Initializing SalaryAnalyticsPipeline
2025-09-10 10:52:36,451 - INFO - Starting data loading process
2025-09-10 10:52:36,452 - INFO - No database connection. Attempting to connect...
2025-09-10 10:52:36,452 - INFO - Attempting to connect to database...
2025-09-10 10:52:36,538 - ERROR - Error connecting to database: No module named 'oracledb'
2025-09-10 10:52:36,539 - ERROR - Failed to establish database connection
2025-09-10 10:52:36,539 - ERROR - Failed to load data
2025-09-10 10:52:36,540 - ERROR - Failed to load data
2025-09-10 10:52:36,541 - INFO - Load data endpoint failed after 0.09 seconds
2025-09-10 10:52:36,541 - ERROR - Error loading data: 500: Failed to load data
2025-09-10 10:52:36,542 - INFO - Load data endpoint failed after 0.09 seconds
2025-09-10 10:52:58,149 - INFO - Shutting down Salary Analytics API...
2025-09-10 10:53:10,697 - INFO - generated new fontManager
2025-09-10 10:53:16,039 - INFO - Initializing pipeline...
2025-09-10 10:53:16,041 - INFO - [2025-09-10 10:53:16] Detecting salary...
2025-09-10 10:53:16,042 - INFO - Started autonomous salary detection loop.
2025-09-10 10:53:16,055 - INFO - Server running on hostname: 1c5d2376fb2a
2025-09-10 10:53:16,056 - INFO - Server IP address: 172.25.0.2
2025-09-10 10:53:16,057 - INFO - Server is accessible at:
2025-09-10 10:53:16,058 - INFO - - http://localhost:8000
2025-09-10 10:53:16,059 - INFO - - http://127.0.0.1:8000
2025-09-10 10:53:16,060 - INFO - - http://172.25.0.2:8000
2025-09-10 10:53:16,062 - INFO - Pipeline initialized successfully
2025-09-10 10:53:16,812 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
"data": [],
"error": {},
"message": "AutoCall Add Salary Successful",
"status": true,
"statusCode": 200
}
2025-09-10 10:53:16,813 - INFO - [2025-09-10 10:53:16] Salary detection complete
2025-09-10 10:53:25,477 - INFO - Initializing SalaryAnalyticsPipeline
2025-09-10 10:53:25,482 - INFO - Starting data loading process
2025-09-10 10:53:25,483 - INFO - No database connection. Attempting to connect...
2025-09-10 10:53:25,484 - INFO - Attempting to connect to database...
2025-09-10 10:53:25,679 - ERROR - Error connecting to database: No module named 'oracledb'
2025-09-10 10:53:25,679 - ERROR - Failed to establish database connection
2025-09-10 10:53:25,680 - ERROR - Failed to load data
2025-09-10 10:53:25,680 - ERROR - Failed to load data
2025-09-10 10:53:25,681 - INFO - Load data endpoint failed after 0.20 seconds
2025-09-10 10:53:25,682 - ERROR - Error loading data: 500: Failed to load data
2025-09-10 10:53:25,682 - INFO - Load data endpoint failed after 0.20 seconds
2025-09-10 10:53:46,324 - INFO - Shutting down Salary Analytics API...
app/config.py
+8 -8
View File
@@ -34,19 +34,19 @@ DB_CONFIG = {
}
# DNS = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={DB_CONFIG['host']})(PORT={DB_CONFIG['port']}))(CONNECT_DATA=(SID={DB_CONFIG['sid']})))"
DNS = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={DB_CONFIG['host']})(PORT={DB_CONFIG['port']}))(CONNECT_DATA=(SID={DB_CONFIG['sid']})))"
# Database Connection
# SQLALCHEMY_DATABASE_URI_INTERNAL = (f"oracle+oracledb://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DNS}")
# SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI_FULL", SQLALCHEMY_DATABASE_URI_INTERNAL)
SQLALCHEMY_DATABASE_URI_INTERNAL = (f"oracle+oracledb://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DNS}")
SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI_FULL", SQLALCHEMY_DATABASE_URI_INTERNAL)
#SQLALCHEMY_DATABASE_URI_FULL = 'oracle+oracledb://FIRSTADVSTG:Pchanged_56789@10.2.110.30:1521/?service_name=firstadv'
# SQLALCHEMY_DATABASE_URI_FULL = 'oracle+oracledb://FIRSTADVSTG:Pchanged_56789@10.2.110.30:1521/?service_name=firstadv'
# SQLAlchemy Configuration
SQLALCHEMY_DATABASE_URI = (
f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@"
f"{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}"
)
# SQLALCHEMY_DATABASE_URI = (
# f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@"
# f"{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}"
# )
SQLALCHEMY_TRACK_MODIFICATIONS = False
app/models/batch_results.py
+8 -8
View File
@@ -24,7 +24,7 @@ class BatchResult(db.Model):
@classmethod
def save_batch(cls, session: Session, batch_number, total_batches, results_df, status="success"):
def save_batch(cls, batch_number, total_batches, results_df, status="success"):
"""Save batch results into DB using ORM bulk insert."""
try:
results_df["batch_number"] = batch_number
@@ -42,21 +42,21 @@ class BatchResult(db.Model):
for row in results_df.to_dict("records")
]
session.bulk_save_objects(records)
session.commit()
db.session.bulk_save_objects(records)
db.session.commit()
logger.info(f"Saved batch {batch_number} successfully.")
return True
except Exception as e:
session.rollback()
db.session.rollback()
logger.error(f"Error saving batch {batch_number}: {str(e)}")
return False
@classmethod
def get_batch_status(cls, session: Session, batch_number: int):
def get_batch_status(cls, batch_number: int):
"""Return summary info about one batch."""
try:
result = (
session.query(
db.session.query(
cls.batch_number,
cls.total_batches,
cls.processed_at,
@@ -75,11 +75,11 @@ class BatchResult(db.Model):
return None
@classmethod
def get_all_batches(cls, session: Session):
def get_all_batches(cls):
"""Return summaries for all batches."""
try:
results = (
session.query(
db.session.query(
cls.batch_number,
cls.total_batches,
cls.processed_at,
app/models/customer_account_transaction_hx.py
+100
View File
@@ -0,0 +1,100 @@
from venv import logger
from sqlalchemy import Column, Integer, String, Float, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from app.extensions import db
import pandas as pd
class CustomerAccountTransactionHx(db.Model):
__tablename__ = "customer_account_transaction_hx"
id = Column(Integer, primary_key=True, autoincrement=True)
accountid = Column(String(64), nullable=False, index=True)
trx_type = Column(String(50), nullable=False)
amount = Column(Float, nullable=False)
description = Column(String(255))
customer_id = Column(String(64))
trx_start_date = Column(DateTime, nullable=False)
trx_end_date = Column(DateTime)
is_salary_related = Column(Integer, default=0)
is_consistent_amount = Column(Integer, default=0)
is_salary_type = Column(Integer, default=0)
@classmethod
def get_all(cls):
"""Fetch all transactions."""
return db.session.query(cls).all()
@classmethod
def get_rows_count(cls):
"""Return total number of transaction rows."""
try:
count = db.session.query(db.func.count(cls.id)).scalar()
return count
except Exception as e:
logger.error(f"Error getting row count: {str(e)}")
return None
@classmethod
def get_by_account(cls, accountid: str):
"""Fetch transactions for a given account."""
return db.session.query(cls).filter_by(accountid=accountid).all()
@classmethod
def get_accounts(cls, limit=None):
"""Fetch distinct account IDs."""
query = db.session.query(cls.accountid).distinct()
if limit:
query = query.limit(limit)
return [row.accountid for row in query.all()]
@classmethod
def insert_transaction(cls, **kwargs):
"""Insert a new transaction."""
trx = cls(**kwargs)
try:
db.session.add(trx)
db.session.commit()
except Exception as e:
logger.error(f"Error inserting transaction: {str(e)}")
return None
return trx
@classmethod
def bulk_insert(cls, transactions: list[dict]):
"""Insert multiple transactions at once."""
objs = [cls(**trx) for trx in transactions]
try:
db.session.bulk_save_objects(objs)
db.session.commit()
except Exception as e:
logger.error(f"Error in bulk insert: {str(e)}")
return None
return objs
@classmethod
def get_transactions_df(cls, accountids: list[str] = None):
"""Return a Pandas DataFrame for ML model preparation."""
query = db.session.query(cls)
if accountids:
query = query.filter(cls.accountid.in_(accountids))
rows = query.all()
df = pd.DataFrame([{
"id": trx.id,
"accountid": trx.accountid,
"trx_type": trx.trx_type,
"amount": trx.amount,
"description": trx.description,
"customer_id": trx.customer_id,
"trx_start_date": trx.trx_start_date,
"trx_end_date": trx.trx_end_date,
"is_salary_related": trx.is_salary_related,
"is_consistent_amount": trx.is_consistent_amount,
"is_salary_type": trx.is_salary_type,
} for trx in rows])
return df
app/models/db_operations.py
+1 -1
View File
@@ -3,7 +3,7 @@ Database operations module for salary analytics.
"""
from sqlalchemy import text
from ..config import BATCH_RESULTS_TABLE
from app.config import BATCH_RESULTS_TABLE
from datetime import datetime
from app.utils.logger import logger