Enhance XLS upload functionality and update requirements. Added Flask, Flask-SQLAlchemy, and Alembic to requirements. Modified database schema in upload_xls.py for improved data handling and added SQLAlchemy configuration in config.py.

demo/upload_xls.py

@@ -32,23 +32,21 @@ def create_table_if_not_exists(conn):
         cur.execute("""
             CREATE TABLE IF NOT EXISTS analytics_raw_transactions (
                 id SERIAL PRIMARY KEY,
-                cif_id TEXT,
-                acid TEXT,
-                ref_num TEXT,
-                entry_usr TEXT,
-                tran_id TEXT,
-                tran_date TIMESTAMP NULL,
-                value_date TIMESTAMP NULL,
+                cust_id VARCHAR(10),
+                accountid VARCHAR(10),
+                tran_id VARCHAR(12),
                 entry_date TIMESTAMP NULL,
+                value_date TIMESTAMP NULL,
                 pstd_date TIMESTAMP NULL,
-                tran_subtype TEXT,
-                part_tran_type TEXT,
-                isreverse TEXT,
-                reverse TEXT,
-                tran_particular TEXT,
-                channel TEXT,
-                amount DECIMAL(20,2),
-                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                tran_date TIMESTAMP NULL,
+                tran_sub_ty VARCHAR(4),
+                part_tran_ty VARCHAR(4),
+                channel VARCHAR(32),
+                tran_amt DECIMAL(20,2),
+                balance DECIMAL(20,2),
+                isreverse VARCHAR(4),
+                reverse VARCHAR(4),
+                tran_particular VARCHAR(100)
             )
         """)
     conn.commit()
@@ -59,14 +57,21 @@ def upload_xls_to_db(xls_path):
         # Read XLS file
         df = pd.read_excel(xls_path, dtype=str)
         
-        # Convert date columns to datetime, errors='coerce' will set invalid parsing as NaT
-        for col in ["TRAN_DATE", "VALUE_DATE", "ENTRY_DATE", "PSTD_DATE"]:
+        # Convert date columns to datetime
+        date_cols = ["ENTRY_DATE", "VALUE_DATE", "PSTD_DATE", "TRAN_DATE"]
+        for col in date_cols:
             if col in df.columns:
                 df[col] = pd.to_datetime(df[col], errors='coerce')
+                df[col] = df[col].fillna(pd.Timestamp.now())
         
-        # Convert AMOUNT to numeric
-        if "AMOUNT" in df.columns:
-            df["AMOUNT"] = pd.to_numeric(df["AMOUNT"], errors='coerce')
+        # Convert numeric columns
+        for col in ["TRAN_AMT", "BALANCE"]:
+            if col in df.columns:
+                df[col] = pd.to_numeric(df[col].str.replace(",", ""), errors='coerce')
+        
+        # Truncate TRAN_PARTICULAR to 100 chars
+        if "TRAN_PARTICULAR" in df.columns:
+            df["TRAN_PARTICULAR"] = df["TRAN_PARTICULAR"].astype(str).str.slice(0, 100)
         
         # Connect to database
         conn = connect_to_db()
@@ -83,26 +88,25 @@ def upload_xls_to_db(xls_path):
                 cur,
                 """
                 INSERT INTO analytics_raw_transactions 
-                (cif_id, acid, ref_num, entry_usr, tran_id, tran_date, value_date, entry_date, pstd_date, tran_subtype, part_tran_type, isreverse, reverse, tran_particular, channel, amount)
+                (cust_id, accountid, tran_id, entry_date, value_date, pstd_date, tran_date, tran_sub_ty, part_tran_ty, channel, tran_amt, balance, isreverse, reverse, tran_particular)
                 VALUES %s
                 """,
                 [(
-                    row.get('CIF_ID'),
-                    row.get('ACID'),
-                    row.get('REF_NUM'),
-                    row.get('ENTRY_USR'),
+                    row.get('CUST_ID'),
+                    row.get('ACCOUNTID'),
                     row.get('TRAN_ID'),
-                    row.get('TRAN_DATE'),
-                    row.get('VALUE_DATE'),
                     row.get('ENTRY_DATE'),
+                    row.get('VALUE_DATE'),
                     row.get('PSTD_DATE'),
-                    row.get('TRAN_SUBTYPE'),
-                    row.get('PART_TRAN_TYPE'),
+                    row.get('TRAN_DATE'),
+                    row.get('TRAN_SUB_TY'),
+                    row.get('PART_TRAN_TY'),
+                    row.get('CHANNEL'),
+                    row.get('TRAN_AMT'),
+                    row.get('BALANCE'),
                     row.get('ISREVERSE'),
                     row.get('REVERSE'),
-                    row.get('TRAN_PARTICULAR'),
-                    row.get('CHANNEL'),
-                    row.get('AMOUNT')
+                    (row.get('TRAN_PARTICULAR') or '')[:100]
                 ) for row in data]
             )