17828 - initial EFT file processing models (#1273)

pay-api/migrations/versions/2023_09_28_456234145e5e_eft_processing.py

@@ -0,0 +1,76 @@
+"""eft_processing
+
+Revision ID: 456234145e5e
+Revises: 3a21a14b4137
+Create Date: 2023-09-28 13:11:49.061949
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+# revision identifiers, used by Alembic.
+revision = '456234145e5e'
+down_revision = '3a21a14b4137'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    process_status_codes_table = op.create_table('eft_process_status_codes',
+                                                 sa.Column('code', sa.String(length=20), nullable=False),
+                                                 sa.Column('description', sa.String(length=100), nullable=False),
+                                                 sa.PrimaryKeyConstraint('code')
+                                                 )
+
+    op.bulk_insert(
+        process_status_codes_table,
+        [
+            {'code': 'COMPLETED', 'description': 'Record or File was able to be fully processed.'},
+            {'code': 'INPROGRESS', 'description': 'Record or File processing in progress.'},
+            {'code': 'FAILED', 'description': 'Record or File failed to process.'},
+            {'code': 'PARTIAL', 'description': 'Record or File was partially processed as there were some errors.'}
+        ]
+    )
+
+    op.create_table('eft_files',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('completed_on', sa.DateTime(), nullable=True),
+                    sa.Column('created_on', sa.DateTime(), nullable=False),
+                    sa.Column('deposit_from_date', sa.DateTime(), nullable=True),
+                    sa.Column('deposit_to_date', sa.DateTime(), nullable=True),
+                    sa.Column('file_creation_date', sa.DateTime(), nullable=True),
+                    sa.Column('file_ref', sa.String(), nullable=False),
+                    sa.Column('number_of_details', sa.Integer(), nullable=True),
+                    sa.Column('status_code', sa.String(length=20), nullable=False),
+                    sa.Column('total_deposit_cents', sa.BigInteger(), nullable=True),
+                    sa.ForeignKeyConstraint(['status_code'], ['eft_process_status_codes.code']),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+
+    op.create_table('eft_transactions',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('file_id', sa.Integer(), nullable=False),
+                    sa.Column('created_on', sa.DateTime(), nullable=False),
+                    sa.Column('completed_on', sa.DateTime(), nullable=True),
+                    sa.Column('line_number', sa.Integer(), nullable=False),
+                    sa.Column('line_type', sa.String(20), nullable=False),
+                    sa.Column('batch_number', sa.String(10), nullable=True),
+                    sa.Column('sequence_number', sa.String(3), nullable=True),
+                    sa.Column('jv_type', sa.String(1), nullable=True),
+                    sa.Column('jv_number', sa.String(10), nullable=True),
+                    sa.Column('batch_number', sa.String, nullable=True),
+                    sa.Column('last_updated_on', sa.DateTime(), nullable=False),
+                    sa.Column('status_code', sa.String(length=20), nullable=False),
+                    sa.Column('error_messages', postgresql.ARRAY(sa.String(150), dimensions=1), nullable=True),
+                    sa.ForeignKeyConstraint(['status_code'], ['eft_process_status_codes.code']),
+                    sa.ForeignKeyConstraint(['file_id'], ['eft_files.id']),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+
+
+def downgrade():
+    op.drop_table('eft_transactions')
+    op.drop_table('eft_files')
+    op.drop_table('eft_process_status_codes')

pay-api/src/pay_api/models/__init__.py

@@ -26,6 +26,9 @@
 from .db import db, ma  # noqa: I001
 from .disbursement_status_code import DisbursementStatusCode
 from .distribution_code import DistributionCode, DistributionCodeLink
+from .eft_file import EFTFile
+from .eft_process_status_code import EFTProcessStatusCode
+from .eft_transaction import EFTTransaction
 from .ejv_file import EjvFile
 from .ejv_header import EjvHeader
 from .ejv_invoice_link import EjvInvoiceLink

pay-api/src/pay_api/models/eft_file.py

@@ -0,0 +1,64 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model to handle EFT file processing."""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey
+
+from pay_api.utils.enums import EFTProcessStatus
+from .base_model import BaseModel
+from .db import db
+
+
+class EFTFile(BaseModel):  # pylint: disable=too-many-instance-attributes
+    """This class manages the file data for EFT transactions."""
+
+    __tablename__ = 'eft_files'
+    # this mapper is used so that new and old versions of the service can be run simultaneously,
+    # making rolling upgrades easier
+    # This is used by SQLAlchemy to explicitly define which fields we're interested
+    # so it doesn't freak out and say it can't map the structure if other fields are present.
+    # This could occur from a failed deploy or during an upgrade.
+    # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous
+    # and can interfere with Alembic upgrades.
+    #
+    # NOTE: please keep mapper names in alpha-order, easier to track that way
+    #       Exception, id is always first, _fields first
+    __mapper_args__ = {
+        'include_properties': [
+            'id',
+            'completed_on',
+            'created_on',
+            'deposit_from_date',
+            'deposit_to_date',
+            'file_creation_date',
+            'file_ref',
+            'number_of_details',
+            'status_code',
+            'total_deposit_cents'
+        ]
+    }
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    created_on = db.Column('created_on', db.DateTime, nullable=False, default=datetime.now)
+    completed_on = db.Column('completed_on', db.DateTime, nullable=True)
+    deposit_from_date = db.Column('deposit_from_date', db.DateTime, nullable=True)
+    deposit_to_date = db.Column('deposit_to_date', db.DateTime, nullable=True)
+    file_creation_date = db.Column('file_creation_date', db.DateTime, nullable=True)
+    number_of_details = db.Column('number_of_details', db.Integer, nullable=True)
+    total_deposit_cents = db.Column('total_deposit_cents', db.BigInteger, nullable=True)
+    file_ref = db.Column('file_ref', db.String, nullable=False, index=True)
+    status_code = db.Column(db.String, ForeignKey('eft_process_status_codes.code'),
+                            default=EFTProcessStatus.IN_PROGRESS.value, nullable=False)

pay-api/src/pay_api/models/eft_process_status_code.py

@@ -0,0 +1,51 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model to capture the state of EFT data being processed."""
+
+from .code_table import CodeTable
+from .db import db, ma
+
+
+class EFTProcessStatusCode(db.Model, CodeTable):
+    """This class manages all of the base data about a EFT Process statuss code."""
+
+    __tablename__ = 'eft_process_status_codes'
+    # this mapper is used so that new and old versions of the service can be run simultaneously,
+    # making rolling upgrades easier
+    # This is used by SQLAlchemy to explicitly define which fields we're interested
+    # so it doesn't freak out and say it can't map the structure if other fields are present.
+    # This could occur from a failed deploy or during an upgrade.
+    # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous
+    # and can interfere with Alembic upgrades.
+    #
+    # NOTE: please keep mapper names in alpha-order, easier to track that way
+    #       Exception, id is always first, _fields first
+    __mapper_args__ = {
+        'include_properties': [
+            'code',
+            'description'
+        ]
+    }
+
+    code = db.Column(db.String(20), primary_key=True)
+    description = db.Column('description', db.String(100), nullable=False)
+
+
+class EFTProcessStatusCodeSchema(ma.ModelSchema):  # pylint: disable=too-many-ancestors
+    """Main schema used to serialize the Status Code."""
+
+    class Meta:  # pylint: disable=too-few-public-methods
+        """Returns all the fields from the SQLAlchemy class."""
+
+        model = EFTProcessStatusCode

pay-api/src/pay_api/models/eft_transaction.py

@@ -0,0 +1,69 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model to handle EFT file processing."""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, String
+from sqlalchemy.dialects.postgresql import ARRAY
+
+from .base_model import BaseModel
+from .db import db
+
+
+class EFTTransaction(BaseModel):  # pylint: disable=too-many-instance-attributes
+    """This class manages the file data for EFT transactions."""
+
+    __tablename__ = 'eft_transactions'
+    # this mapper is used so that new and old versions of the service can be run simultaneously,
+    # making rolling upgrades easier
+    # This is used by SQLAlchemy to explicitly define which fields we're interested
+    # so it doesn't freak out and say it can't map the structure if other fields are present.
+    # This could occur from a failed deploy or during an upgrade.
+    # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous
+    # and can interfere with Alembic upgrades.
+    #
+    # NOTE: please keep mapper names in alpha-order, easier to track that way
+    #       Exception, id is always first, _fields first
+    __mapper_args__ = {
+        'include_properties': [
+            'id',
+            'batch_number',
+            'completed_on',
+            'created_on',
+            'error_messages',
+            'file_id',
+            'last_updated_on',
+            'line_number',
+            'line_type',
+            'jv_type',
+            'jv_number',
+            'sequence_number',
+            'status_code'
+        ]
+    }
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    batch_number = db.Column('batch_number', db.String(10), nullable=True)
+    completed_on = db.Column('completed_on', db.DateTime, nullable=True)
+    created_on = db.Column('created_on', db.DateTime, nullable=False, default=datetime.now)
+    error_messages = db.Column(ARRAY(String, dimensions=1), nullable=True)
+    file_id = db.Column(db.Integer, ForeignKey('eft_files.id'), nullable=False, index=True)
+    last_updated_on = db.Column('last_updated_on', db.DateTime, nullable=False, default=datetime.now)
+    line_number = db.Column('line_number', db.Integer, nullable=False)
+    line_type = db.Column('line_type', db.String(), nullable=False)
+    jv_type = db.Column('jv_type', db.String(1), nullable=True)
+    jv_number = db.Column('jv_number', db.String(10), nullable=True)
+    sequence_number = db.Column('sequence_number', db.String(3), nullable=True)
+    status_code = db.Column(db.String, ForeignKey('eft_process_status_codes.code'), nullable=False)

pay-api/src/pay_api/utils/enums.py

@@ -293,3 +293,20 @@ class CfsReceiptStatus(Enum):
     """Routing Slip Receipt Status."""
 
     REV = 'REV'
+
+
+class EFTProcessStatus(Enum):
+    """EFT Process Status."""
+
+    COMPLETED = 'COMPLETED'
+    IN_PROGRESS = 'INPROGRESS'
+    FAILED = 'FAILED'
+    PARTIAL = 'PARTIAL'
+
+
+class EFTFileLineType(Enum):
+    """EFT File (TDI17) Line types."""
+
+    HEADER = 'HEADER'
+    TRANSACTION = 'TRANSACTION'
+    TRAILER = 'TRAILER'

pay-api/tests/unit/models/test_eft_file.py

@@ -0,0 +1,71 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests to assure the EFT File model.
+
+Test-Suite to ensure that the EFT File model is working as expected.
+"""
+from datetime import datetime
+
+from pay_api.models.eft_file import EFTFile as EFTFileModel
+from pay_api.utils.enums import EFTProcessStatus
+
+
+def test_eft_file_defaults(session):
+    """Assert eft file defaults are stored."""
+    eft_file = EFTFileModel()
+    eft_file.file_ref = 'test.txt'
+    eft_file.save()
+
+    assert eft_file.id is not None
+    assert eft_file.created_on is not None
+    assert eft_file.status_code == EFTProcessStatus.IN_PROGRESS.value
+
+
+def test_eft_file_all_attributes(session):
+    """Assert all eft file attributes are stored."""
+    eft_file = EFTFileModel()
+
+    file_creation = datetime(2023, 9, 30, 1, 0)
+    created_on = datetime(2023, 9, 30, 10, 0)
+    completed_on = datetime(2023, 9, 30, 11, 0)
+    deposit_from_date = datetime(2023, 9, 28)
+    deposit_to_date = datetime(2023, 9, 29)
+    number_of_details = 10
+    total_deposit_cents = 125000
+
+    eft_file.file_creation_date = file_creation
+    eft_file.created_on = created_on
+    eft_file.completed_on = completed_on
+    eft_file.deposit_from_date = deposit_from_date
+    eft_file.deposit_to_date = deposit_to_date
+    eft_file.number_of_details = number_of_details
+    eft_file.total_deposit_cents = total_deposit_cents
+    eft_file.file_ref = 'test.txt'
+    eft_file.status_code = EFTProcessStatus.COMPLETED.value
+    eft_file.save()
+
+    assert eft_file.id is not None
+    eft_file = EFTFileModel.find_by_id(eft_file.id)
+
+    assert eft_file is not None
+    assert eft_file.status_code == EFTProcessStatus.COMPLETED.value
+    assert eft_file.file_creation_date == file_creation
+    assert eft_file.created_on == created_on
+    assert eft_file.completed_on == completed_on
+    assert eft_file.deposit_from_date == deposit_from_date
+    assert eft_file.deposit_to_date == deposit_to_date
+    assert eft_file.number_of_details == number_of_details
+    assert eft_file.total_deposit_cents == total_deposit_cents
+    assert eft_file.file_ref == 'test.txt'