Enhance XLS upload functionality and update requirements. Added Flask, Flask-SQLAlchemy, and Alembic to requirements. Modified database schema in upload_xls.py for improved data handling and added SQLAlchemy configuration in config.py.

2025-06-09 15:34:18 +01:00
parent f478a52a2f
commit c00bb71d2a
14 changed files with 427 additions and 33 deletions
@@ -0,0 +1,56 @@
+# Database Migration & Data Upload Guide
+
+This project uses **Flask-Migrate** (powered by Alembic) for managing database schema changes and a custom Flask CLI command for uploading XLS data.
+
+## Migration Workflow
+
+Follow these steps to manage database schema and upload data:
+
+1.  **Set Flask App Environment Variable:**
+    Open your terminal in the project root (`digifi-Analytics/`) and set the `FLASK_APP` variable:
+
+    ```bash
+    # For Windows
+    set FLASK_APP=run.py
+
+    # For macOS/Linux
+    export FLASK_APP=run.py
+    ```
+
+2.  **Initialize Migration Repository (First-time setup only):**
+    This command sets up the `migrations/` directory structure. You've likely already run this, so you might get a "Directory already exists" message, which is fine.
+
+    ```bash
+    flask --app run.py db init
+    ```
+
+3.  **Generate a New Migration Script:**
+    After making changes to `salary_analytics/app/models.py` (e.g., adding a new table or column), use this command to create a migration file. This file will contain the `upgrade()` and `downgrade()` logic.
+
+    ```bash
+    flask --app run.py db migrate -m "Descriptive message for your changes"
+    ```
+
+4.  **Review the Generated Migration:**
+    Open the newly created `.py` file in `migrations/versions/` (e.g., `XXXXXXXXXXXX_your_message.py`) and review the `upgrade()` and `downgrade()` functions. Ensure they accurately reflect your intended schema changes.
+
+5.  **Apply Migrations to the Database:**
+    This command executes the pending migration scripts, updating your database schema. **Always apply migrations after generating them.**
+
+    ```bash
+    flask --app run.py db upgrade
+    ```
+
+## Uploading XLS Data
+
+Once your `analytics_raw_transactions` table is created via migration, you can upload your XLS files using the custom command:
+
+```bash
+flask --app run.py upload-xls <path_to_your_xls_file>
+```
+
+**Example:**
+
+```bash
+flask --app run.py upload-xls data/transactions.xls
+```
@@ -0,0 +1,44 @@
+[alembic]
+script_location = migrations
+sqlalchemy.url = 
+revision_environment = false
+version_locations = %(script_location)s/versions
+output_encoding = utf-8
+
+[post_write_hooks]
+hooks = 
+hook_default_ops = 
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname = 
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = console
+qualname = sqlalchemy
+
+[logger_alembic]
+level = DEBUG
+handlers = console
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S 
@@ -0,0 +1,89 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import Base
+# target_metadata = Base.metadata
+from flask import current_app
+from salary_analytics.app.extensions import db
+
+config.set_main_option('sqlalchemy.url',
+                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+target_metadata = db.metadata 
+
+# Function to filter objects for autogenerate
+def include_object(object, name, type_, reflected, compare_to):
+    """
+    Control which objects are included in the autogenerate process.
+    Only include objects that are part of the Flask app's declared models.
+    """
+    if type_ == "table" and reflected and name not in target_metadata.tables:
+        return False # Exclude tables that are in DB but not in models
+    return True # Include everything else (e.g., our modeled tables, columns, etc.)
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a database to begin with.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        include_object=include_object
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata,
+            include_object=include_object
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online() 
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}
@@ -0,0 +1,46 @@
+"""Create analytics_raw_transactions table
+
+Revision ID: c2b2cdbc8022
+Revises: 
+Create Date: 2025-06-09 15:19:36.017861
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c2b2cdbc8022'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('analytics_raw_transactions',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('cust_id', sa.String(length=10), nullable=True),
+    sa.Column('accountid', sa.String(length=10), nullable=True),
+    sa.Column('tran_id', sa.String(length=12), nullable=True),
+    sa.Column('entry_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('value_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('pstd_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('tran_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('tran_sub_ty', sa.String(length=4), nullable=True),
+    sa.Column('part_tran_ty', sa.String(length=4), nullable=True),
+    sa.Column('channel', sa.String(length=32), nullable=True),
+    sa.Column('tran_amt', sa.Numeric(precision=20, scale=2), nullable=True),
+    sa.Column('balance', sa.Numeric(precision=20, scale=2), nullable=True),
+    sa.Column('isreverse', sa.String(length=4), nullable=True),
+    sa.Column('reverse', sa.String(length=4), nullable=True),
+    sa.Column('tran_particular', sa.String(length=100), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('analytics_raw_transactions')
+    # ### end Alembic commands ###