[add]: database connection
This commit is contained in:
VivianDee
@@ -188,3 +188,222 @@
|
|||||||
2025-09-10 10:35:40,300 - INFO - Loading chunk starting at offset 900000
|
2025-09-10 10:35:40,300 - INFO - Loading chunk starting at offset 900000
|
||||||
2025-09-10 10:36:05,763 - INFO - Loading chunk starting at offset 910000
|
2025-09-10 10:36:05,763 - INFO - Loading chunk starting at offset 910000
|
||||||
2025-09-10 10:36:15,199 - INFO - Loading chunk starting at offset 920000
|
2025-09-10 10:36:15,199 - INFO - Loading chunk starting at offset 920000
|
||||||
|
2025-09-10 10:36:46,036 - INFO - Loading chunk starting at offset 930000
|
||||||
|
2025-09-10 10:36:53,221 - INFO - Loading chunk starting at offset 940000
|
||||||
|
2025-09-10 10:37:06,754 - INFO - Loading chunk starting at offset 950000
|
||||||
|
2025-09-10 10:37:15,558 - INFO - Loading chunk starting at offset 960000
|
||||||
|
2025-09-10 10:37:21,798 - INFO - [2025-09-10 10:37:21] Detecting salary...
|
||||||
|
2025-09-10 10:37:22,954 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:37:22,955 - INFO - [2025-09-10 10:37:22] Salary detection complete
|
||||||
|
2025-09-10 10:37:29,088 - INFO - Loading chunk starting at offset 970000
|
||||||
|
2025-09-10 10:37:54,730 - INFO - Loading chunk starting at offset 980000
|
||||||
|
2025-09-10 10:38:13,667 - INFO - Loading chunk starting at offset 990000
|
||||||
|
2025-09-10 10:38:27,880 - INFO - Loading chunk starting at offset 1000000
|
||||||
|
2025-09-10 10:39:07,546 - INFO - Loading chunk starting at offset 1010000
|
||||||
|
2025-09-10 10:39:17,369 - INFO - Loading chunk starting at offset 1020000
|
||||||
|
2025-09-10 10:39:23,176 - INFO - [2025-09-10 10:39:23] Detecting salary...
|
||||||
|
2025-09-10 10:39:24,444 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:39:24,445 - INFO - [2025-09-10 10:39:24] Salary detection complete
|
||||||
|
2025-09-10 10:39:26,048 - INFO - Loading chunk starting at offset 1030000
|
||||||
|
2025-09-10 10:39:45,494 - INFO - Loading chunk starting at offset 1040000
|
||||||
|
2025-09-10 10:39:50,480 - INFO - Loading chunk starting at offset 1050000
|
||||||
|
2025-09-10 10:39:54,793 - INFO - Loading chunk starting at offset 1060000
|
||||||
|
2025-09-10 10:39:59,415 - INFO - Loading chunk starting at offset 1070000
|
||||||
|
2025-09-10 10:40:04,306 - INFO - Loading chunk starting at offset 1080000
|
||||||
|
2025-09-10 10:40:10,517 - INFO - Loading chunk starting at offset 1090000
|
||||||
|
2025-09-10 10:40:15,671 - INFO - Loading chunk starting at offset 1100000
|
||||||
|
2025-09-10 10:40:21,662 - INFO - Loading chunk starting at offset 1110000
|
||||||
|
2025-09-10 10:40:30,551 - INFO - Loading chunk starting at offset 1120000
|
||||||
|
2025-09-10 10:40:47,391 - INFO - Loading chunk starting at offset 1130000
|
||||||
|
2025-09-10 10:40:55,609 - INFO - Loading chunk starting at offset 1140000
|
||||||
|
2025-09-10 10:41:04,496 - INFO - Loading chunk starting at offset 1150000
|
||||||
|
2025-09-10 10:41:14,608 - INFO - Loading chunk starting at offset 1160000
|
||||||
|
2025-09-10 10:41:24,447 - INFO - [2025-09-10 10:41:24] Detecting salary...
|
||||||
|
2025-09-10 10:41:28,537 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:41:28,548 - INFO - [2025-09-10 10:41:28] Salary detection complete
|
||||||
|
2025-09-10 10:42:00,867 - INFO - Loading chunk starting at offset 1170000
|
||||||
|
2025-09-10 10:42:13,069 - INFO - Loading chunk starting at offset 1180000
|
||||||
|
2025-09-10 10:42:21,593 - INFO - Loading chunk starting at offset 1190000
|
||||||
|
2025-09-10 10:42:32,011 - INFO - Loading chunk starting at offset 1200000
|
||||||
|
2025-09-10 10:42:37,982 - INFO - Loading chunk starting at offset 1210000
|
||||||
|
2025-09-10 10:42:45,458 - INFO - Loading chunk starting at offset 1220000
|
||||||
|
2025-09-10 10:42:54,545 - INFO - Loading chunk starting at offset 1230000
|
||||||
|
2025-09-10 10:43:15,705 - INFO - Loading chunk starting at offset 1240000
|
||||||
|
2025-09-10 10:43:28,549 - INFO - [2025-09-10 10:43:28] Detecting salary...
|
||||||
|
2025-09-10 10:43:31,640 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:43:31,641 - INFO - [2025-09-10 10:43:31] Salary detection complete
|
||||||
|
2025-09-10 10:43:43,344 - INFO - Loading chunk starting at offset 1250000
|
||||||
|
2025-09-10 10:43:50,004 - INFO - Loading chunk starting at offset 1260000
|
||||||
|
2025-09-10 10:43:59,430 - INFO - Loading chunk starting at offset 1270000
|
||||||
|
2025-09-10 10:44:07,478 - INFO - Loading chunk starting at offset 1280000
|
||||||
|
2025-09-10 10:44:18,927 - INFO - Loading chunk starting at offset 1290000
|
||||||
|
2025-09-10 10:44:28,523 - INFO - Loading chunk starting at offset 1300000
|
||||||
|
2025-09-10 10:44:38,629 - INFO - Loading chunk starting at offset 1310000
|
||||||
|
2025-09-10 10:44:56,977 - INFO - Loading chunk starting at offset 1320000
|
||||||
|
2025-09-10 10:45:09,203 - INFO - Loading chunk starting at offset 1330000
|
||||||
|
2025-09-10 10:45:18,657 - INFO - Loading chunk starting at offset 1340000
|
||||||
|
2025-09-10 10:45:27,201 - INFO - Loading chunk starting at offset 1350000
|
||||||
|
2025-09-10 10:45:31,641 - INFO - [2025-09-10 10:45:31] Detecting salary...
|
||||||
|
2025-09-10 10:45:32,261 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:45:32,262 - INFO - [2025-09-10 10:45:32] Salary detection complete
|
||||||
|
2025-09-10 10:45:35,308 - INFO - Loading chunk starting at offset 1360000
|
||||||
|
2025-09-10 10:45:44,783 - INFO - Loading chunk starting at offset 1370000
|
||||||
|
2025-09-10 10:45:52,048 - INFO - Loading chunk starting at offset 1380000
|
||||||
|
2025-09-10 10:45:58,491 - INFO - Loading chunk starting at offset 1390000
|
||||||
|
2025-09-10 10:46:13,270 - INFO - Loading chunk starting at offset 1400000
|
||||||
|
2025-09-10 10:46:25,221 - INFO - Loading chunk starting at offset 1410000
|
||||||
|
2025-09-10 10:46:34,358 - INFO - Loading chunk starting at offset 1420000
|
||||||
|
2025-09-10 10:46:38,777 - INFO - Loading chunk starting at offset 1430000
|
||||||
|
2025-09-10 10:46:42,534 - INFO - Loading chunk starting at offset 1440000
|
||||||
|
2025-09-10 10:46:46,147 - INFO - Loading chunk starting at offset 1450000
|
||||||
|
2025-09-10 10:46:50,727 - INFO - Loading chunk starting at offset 1460000
|
||||||
|
2025-09-10 10:46:55,939 - INFO - Loading chunk starting at offset 1470000
|
||||||
|
2025-09-10 10:47:01,696 - INFO - Loading chunk starting at offset 1480000
|
||||||
|
2025-09-10 10:47:07,411 - INFO - Loading chunk starting at offset 1490000
|
||||||
|
2025-09-10 10:47:13,449 - INFO - Loading chunk starting at offset 1500000
|
||||||
|
2025-09-10 10:47:23,058 - INFO - Loading chunk starting at offset 1510000
|
||||||
|
2025-09-10 10:47:30,097 - INFO - Loading chunk starting at offset 1520000
|
||||||
|
2025-09-10 10:47:32,259 - INFO - [2025-09-10 10:47:32] Detecting salary...
|
||||||
|
2025-09-10 10:47:32,923 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:47:32,924 - INFO - [2025-09-10 10:47:32] Salary detection complete
|
||||||
|
2025-09-10 10:47:36,423 - INFO - Loading chunk starting at offset 1530000
|
||||||
|
2025-09-10 10:47:45,086 - INFO - Loading chunk starting at offset 1540000
|
||||||
|
2025-09-10 10:47:57,079 - INFO - Loading chunk starting at offset 1550000
|
||||||
|
2025-09-10 10:48:19,741 - INFO - Loading chunk starting at offset 1560000
|
||||||
|
2025-09-10 10:48:41,300 - INFO - Loading chunk starting at offset 1570000
|
||||||
|
2025-09-10 10:48:51,349 - INFO - Loading chunk starting at offset 1580000
|
||||||
|
2025-09-10 10:48:58,892 - INFO - Loading chunk starting at offset 1590000
|
||||||
|
2025-09-10 10:49:04,857 - INFO - Loading chunk starting at offset 1600000
|
||||||
|
2025-09-10 10:49:10,299 - INFO - Loading chunk starting at offset 1610000
|
||||||
|
2025-09-10 10:49:16,650 - INFO - Loading chunk starting at offset 1620000
|
||||||
|
2025-09-10 10:49:23,107 - INFO - Loading chunk starting at offset 1630000
|
||||||
|
2025-09-10 10:49:32,320 - INFO - Loading chunk starting at offset 1640000
|
||||||
|
2025-09-10 10:49:32,927 - INFO - [2025-09-10 10:49:32] Detecting salary...
|
||||||
|
2025-09-10 10:49:33,484 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:49:33,485 - INFO - [2025-09-10 10:49:33] Salary detection complete
|
||||||
|
2025-09-10 10:49:40,707 - INFO - Loading chunk starting at offset 1650000
|
||||||
|
2025-09-10 10:49:50,677 - INFO - Loading chunk starting at offset 1660000
|
||||||
|
2025-09-10 10:49:59,664 - INFO - Loading chunk starting at offset 1670000
|
||||||
|
2025-09-10 10:50:07,452 - INFO - Loading chunk starting at offset 1680000
|
||||||
|
2025-09-10 10:50:13,583 - INFO - Loading chunk starting at offset 1690000
|
||||||
|
2025-09-10 10:50:20,858 - INFO - Loading chunk starting at offset 1700000
|
||||||
|
2025-09-10 10:50:27,519 - INFO - Loading chunk starting at offset 1710000
|
||||||
|
2025-09-10 10:50:34,012 - INFO - Loading chunk starting at offset 1720000
|
||||||
|
2025-09-10 10:50:39,628 - INFO - Loading chunk starting at offset 1730000
|
||||||
|
2025-09-10 10:50:50,012 - INFO - Loading chunk starting at offset 1740000
|
||||||
|
2025-09-10 10:51:06,647 - INFO - Loading chunk starting at offset 1750000
|
||||||
|
2025-09-10 10:51:17,401 - INFO - Loading chunk starting at offset 1760000
|
||||||
|
2025-09-10 10:51:26,360 - INFO - Loading chunk starting at offset 1770000
|
||||||
|
2025-09-10 10:51:33,485 - INFO - [2025-09-10 10:51:33] Detecting salary...
|
||||||
|
2025-09-10 10:51:55,392 - INFO - Initializing pipeline...
|
||||||
|
2025-09-10 10:51:55,395 - INFO - [2025-09-10 10:51:55] Detecting salary...
|
||||||
|
2025-09-10 10:51:55,395 - INFO - Started autonomous salary detection loop.
|
||||||
|
2025-09-10 10:51:55,409 - INFO - Server running on hostname: 22bad35c69c3
|
||||||
|
2025-09-10 10:51:55,416 - INFO - Server IP address: 172.25.0.2
|
||||||
|
2025-09-10 10:51:55,424 - INFO - Server is accessible at:
|
||||||
|
2025-09-10 10:51:55,426 - INFO - - http://localhost:8000
|
||||||
|
2025-09-10 10:51:55,444 - INFO - - http://127.0.0.1:8000
|
||||||
|
2025-09-10 10:51:55,455 - INFO - - http://172.25.0.2:8000
|
||||||
|
2025-09-10 10:51:55,456 - INFO - Pipeline initialized successfully
|
||||||
|
2025-09-10 10:51:56,024 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:51:56,024 - INFO - [2025-09-10 10:51:56] Salary detection complete
|
||||||
|
2025-09-10 10:52:36,450 - INFO - Initializing SalaryAnalyticsPipeline
|
||||||
|
2025-09-10 10:52:36,451 - INFO - Starting data loading process
|
||||||
|
2025-09-10 10:52:36,452 - INFO - No database connection. Attempting to connect...
|
||||||
|
2025-09-10 10:52:36,452 - INFO - Attempting to connect to database...
|
||||||
|
2025-09-10 10:52:36,538 - ERROR - Error connecting to database: No module named 'oracledb'
|
||||||
|
2025-09-10 10:52:36,539 - ERROR - Failed to establish database connection
|
||||||
|
2025-09-10 10:52:36,539 - ERROR - Failed to load data
|
||||||
|
2025-09-10 10:52:36,540 - ERROR - Failed to load data
|
||||||
|
2025-09-10 10:52:36,541 - INFO - Load data endpoint failed after 0.09 seconds
|
||||||
|
2025-09-10 10:52:36,541 - ERROR - Error loading data: 500: Failed to load data
|
||||||
|
2025-09-10 10:52:36,542 - INFO - Load data endpoint failed after 0.09 seconds
|
||||||
|
2025-09-10 10:52:58,149 - INFO - Shutting down Salary Analytics API...
|
||||||
|
2025-09-10 10:53:10,697 - INFO - generated new fontManager
|
||||||
|
2025-09-10 10:53:16,039 - INFO - Initializing pipeline...
|
||||||
|
2025-09-10 10:53:16,041 - INFO - [2025-09-10 10:53:16] Detecting salary...
|
||||||
|
2025-09-10 10:53:16,042 - INFO - Started autonomous salary detection loop.
|
||||||
|
2025-09-10 10:53:16,055 - INFO - Server running on hostname: 1c5d2376fb2a
|
||||||
|
2025-09-10 10:53:16,056 - INFO - Server IP address: 172.25.0.2
|
||||||
|
2025-09-10 10:53:16,057 - INFO - Server is accessible at:
|
||||||
|
2025-09-10 10:53:16,058 - INFO - - http://localhost:8000
|
||||||
|
2025-09-10 10:53:16,059 - INFO - - http://127.0.0.1:8000
|
||||||
|
2025-09-10 10:53:16,060 - INFO - - http://172.25.0.2:8000
|
||||||
|
2025-09-10 10:53:16,062 - INFO - Pipeline initialized successfully
|
||||||
|
2025-09-10 10:53:16,812 - INFO - POST http://www.simbrellang.net:5000/autocall/analytic-salary-detect status: 200, response: {
|
||||||
|
"data": [],
|
||||||
|
"error": {},
|
||||||
|
"message": "AutoCall Add Salary Successful",
|
||||||
|
"status": true,
|
||||||
|
"statusCode": 200
|
||||||
|
}
|
||||||
|
|
||||||
|
2025-09-10 10:53:16,813 - INFO - [2025-09-10 10:53:16] Salary detection complete
|
||||||
|
2025-09-10 10:53:25,477 - INFO - Initializing SalaryAnalyticsPipeline
|
||||||
|
2025-09-10 10:53:25,482 - INFO - Starting data loading process
|
||||||
|
2025-09-10 10:53:25,483 - INFO - No database connection. Attempting to connect...
|
||||||
|
2025-09-10 10:53:25,484 - INFO - Attempting to connect to database...
|
||||||
|
2025-09-10 10:53:25,679 - ERROR - Error connecting to database: No module named 'oracledb'
|
||||||
|
2025-09-10 10:53:25,679 - ERROR - Failed to establish database connection
|
||||||
|
2025-09-10 10:53:25,680 - ERROR - Failed to load data
|
||||||
|
2025-09-10 10:53:25,680 - ERROR - Failed to load data
|
||||||
|
2025-09-10 10:53:25,681 - INFO - Load data endpoint failed after 0.20 seconds
|
||||||
|
2025-09-10 10:53:25,682 - ERROR - Error loading data: 500: Failed to load data
|
||||||
|
2025-09-10 10:53:25,682 - INFO - Load data endpoint failed after 0.20 seconds
|
||||||
|
2025-09-10 10:53:46,324 - INFO - Shutting down Salary Analytics API...
|
||||||
|
|||||||
+8
-8
@@ -34,19 +34,19 @@ DB_CONFIG = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# DNS = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={DB_CONFIG['host']})(PORT={DB_CONFIG['port']}))(CONNECT_DATA=(SID={DB_CONFIG['sid']})))"
|
DNS = f"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST={DB_CONFIG['host']})(PORT={DB_CONFIG['port']}))(CONNECT_DATA=(SID={DB_CONFIG['sid']})))"
|
||||||
|
|
||||||
# Database Connection
|
# Database Connection
|
||||||
# SQLALCHEMY_DATABASE_URI_INTERNAL = (f"oracle+oracledb://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DNS}")
|
SQLALCHEMY_DATABASE_URI_INTERNAL = (f"oracle+oracledb://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DNS}")
|
||||||
# SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI_FULL", SQLALCHEMY_DATABASE_URI_INTERNAL)
|
SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI_FULL", SQLALCHEMY_DATABASE_URI_INTERNAL)
|
||||||
|
|
||||||
#SQLALCHEMY_DATABASE_URI_FULL = 'oracle+oracledb://FIRSTADVSTG:Pchanged_56789@10.2.110.30:1521/?service_name=firstadv'
|
# SQLALCHEMY_DATABASE_URI_FULL = 'oracle+oracledb://FIRSTADVSTG:Pchanged_56789@10.2.110.30:1521/?service_name=firstadv'
|
||||||
|
|
||||||
# SQLAlchemy Configuration
|
# SQLAlchemy Configuration
|
||||||
SQLALCHEMY_DATABASE_URI = (
|
# SQLALCHEMY_DATABASE_URI = (
|
||||||
f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@"
|
# f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@"
|
||||||
f"{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}"
|
# f"{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['name']}"
|
||||||
)
|
# )
|
||||||
|
|
||||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ class BatchResult(db.Model):
|
|||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def save_batch(cls, session: Session, batch_number, total_batches, results_df, status="success"):
|
def save_batch(cls, batch_number, total_batches, results_df, status="success"):
|
||||||
"""Save batch results into DB using ORM bulk insert."""
|
"""Save batch results into DB using ORM bulk insert."""
|
||||||
try:
|
try:
|
||||||
results_df["batch_number"] = batch_number
|
results_df["batch_number"] = batch_number
|
||||||
@@ -42,21 +42,21 @@ class BatchResult(db.Model):
|
|||||||
for row in results_df.to_dict("records")
|
for row in results_df.to_dict("records")
|
||||||
]
|
]
|
||||||
|
|
||||||
session.bulk_save_objects(records)
|
db.session.bulk_save_objects(records)
|
||||||
session.commit()
|
db.session.commit()
|
||||||
logger.info(f"Saved batch {batch_number} successfully.")
|
logger.info(f"Saved batch {batch_number} successfully.")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
session.rollback()
|
db.session.rollback()
|
||||||
logger.error(f"Error saving batch {batch_number}: {str(e)}")
|
logger.error(f"Error saving batch {batch_number}: {str(e)}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_batch_status(cls, session: Session, batch_number: int):
|
def get_batch_status(cls, batch_number: int):
|
||||||
"""Return summary info about one batch."""
|
"""Return summary info about one batch."""
|
||||||
try:
|
try:
|
||||||
result = (
|
result = (
|
||||||
session.query(
|
db.session.query(
|
||||||
cls.batch_number,
|
cls.batch_number,
|
||||||
cls.total_batches,
|
cls.total_batches,
|
||||||
cls.processed_at,
|
cls.processed_at,
|
||||||
@@ -75,11 +75,11 @@ class BatchResult(db.Model):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_all_batches(cls, session: Session):
|
def get_all_batches(cls):
|
||||||
"""Return summaries for all batches."""
|
"""Return summaries for all batches."""
|
||||||
try:
|
try:
|
||||||
results = (
|
results = (
|
||||||
session.query(
|
db.session.query(
|
||||||
cls.batch_number,
|
cls.batch_number,
|
||||||
cls.total_batches,
|
cls.total_batches,
|
||||||
cls.processed_at,
|
cls.processed_at,
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
from venv import logger
|
||||||
|
from sqlalchemy import Column, Integer, String, Float, DateTime, ForeignKey
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
from app.extensions import db
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
class CustomerAccountTransactionHx(db.Model):
|
||||||
|
__tablename__ = "customer_account_transaction_hx"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
accountid = Column(String(64), nullable=False, index=True)
|
||||||
|
trx_type = Column(String(50), nullable=False)
|
||||||
|
amount = Column(Float, nullable=False)
|
||||||
|
description = Column(String(255))
|
||||||
|
customer_id = Column(String(64))
|
||||||
|
trx_start_date = Column(DateTime, nullable=False)
|
||||||
|
trx_end_date = Column(DateTime)
|
||||||
|
is_salary_related = Column(Integer, default=0)
|
||||||
|
is_consistent_amount = Column(Integer, default=0)
|
||||||
|
is_salary_type = Column(Integer, default=0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_all(cls):
|
||||||
|
"""Fetch all transactions."""
|
||||||
|
return db.session.query(cls).all()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_rows_count(cls):
|
||||||
|
"""Return total number of transaction rows."""
|
||||||
|
try:
|
||||||
|
count = db.session.query(db.func.count(cls.id)).scalar()
|
||||||
|
return count
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting row count: {str(e)}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_by_account(cls, accountid: str):
|
||||||
|
"""Fetch transactions for a given account."""
|
||||||
|
return db.session.query(cls).filter_by(accountid=accountid).all()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_accounts(cls, limit=None):
|
||||||
|
"""Fetch distinct account IDs."""
|
||||||
|
query = db.session.query(cls.accountid).distinct()
|
||||||
|
if limit:
|
||||||
|
query = query.limit(limit)
|
||||||
|
return [row.accountid for row in query.all()]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def insert_transaction(cls, **kwargs):
|
||||||
|
"""Insert a new transaction."""
|
||||||
|
trx = cls(**kwargs)
|
||||||
|
try:
|
||||||
|
db.session.add(trx)
|
||||||
|
db.session.commit()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error inserting transaction: {str(e)}")
|
||||||
|
return None
|
||||||
|
return trx
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def bulk_insert(cls, transactions: list[dict]):
|
||||||
|
"""Insert multiple transactions at once."""
|
||||||
|
objs = [cls(**trx) for trx in transactions]
|
||||||
|
|
||||||
|
try:
|
||||||
|
db.session.bulk_save_objects(objs)
|
||||||
|
db.session.commit()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in bulk insert: {str(e)}")
|
||||||
|
return None
|
||||||
|
return objs
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_transactions_df(cls, accountids: list[str] = None):
|
||||||
|
"""Return a Pandas DataFrame for ML model preparation."""
|
||||||
|
query = db.session.query(cls)
|
||||||
|
if accountids:
|
||||||
|
query = query.filter(cls.accountid.in_(accountids))
|
||||||
|
rows = query.all()
|
||||||
|
|
||||||
|
|
||||||
|
df = pd.DataFrame([{
|
||||||
|
"id": trx.id,
|
||||||
|
"accountid": trx.accountid,
|
||||||
|
"trx_type": trx.trx_type,
|
||||||
|
"amount": trx.amount,
|
||||||
|
"description": trx.description,
|
||||||
|
"customer_id": trx.customer_id,
|
||||||
|
"trx_start_date": trx.trx_start_date,
|
||||||
|
"trx_end_date": trx.trx_end_date,
|
||||||
|
"is_salary_related": trx.is_salary_related,
|
||||||
|
"is_consistent_amount": trx.is_consistent_amount,
|
||||||
|
"is_salary_type": trx.is_salary_type,
|
||||||
|
} for trx in rows])
|
||||||
|
|
||||||
|
return df
|
||||||
@@ -3,7 +3,7 @@ Database operations module for salary analytics.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from ..config import BATCH_RESULTS_TABLE
|
from app.config import BATCH_RESULTS_TABLE
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from app.utils.logger import logger
|
from app.utils.logger import logger
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user