From 99647a8b08d52c0e2c157bf5a36d4c19c931a1d6 Mon Sep 17 00:00:00 2001 From: Odysseus Chiu Date: Fri, 6 Oct 2023 09:58:05 -0700 Subject: [PATCH] 17828 - initial EFT file processing models (#1273) --- .../2023_09_28_456234145e5e_eft_processing.py | 76 +++++++++++++ pay-api/src/pay_api/models/__init__.py | 3 + pay-api/src/pay_api/models/eft_file.py | 64 +++++++++++ .../pay_api/models/eft_process_status_code.py | 51 +++++++++ pay-api/src/pay_api/models/eft_transaction.py | 69 ++++++++++++ pay-api/src/pay_api/utils/enums.py | 17 +++ pay-api/tests/unit/models/test_eft_file.py | 71 ++++++++++++ .../tests/unit/models/test_eft_transaction.py | 102 ++++++++++++++++++ 8 files changed, 453 insertions(+) create mode 100644 pay-api/migrations/versions/2023_09_28_456234145e5e_eft_processing.py create mode 100644 pay-api/src/pay_api/models/eft_file.py create mode 100644 pay-api/src/pay_api/models/eft_process_status_code.py create mode 100644 pay-api/src/pay_api/models/eft_transaction.py create mode 100644 pay-api/tests/unit/models/test_eft_file.py create mode 100644 pay-api/tests/unit/models/test_eft_transaction.py diff --git a/pay-api/migrations/versions/2023_09_28_456234145e5e_eft_processing.py b/pay-api/migrations/versions/2023_09_28_456234145e5e_eft_processing.py new file mode 100644 index 000000000..0bd18d990 --- /dev/null +++ b/pay-api/migrations/versions/2023_09_28_456234145e5e_eft_processing.py @@ -0,0 +1,76 @@ +"""eft_processing + +Revision ID: 456234145e5e +Revises: 3a21a14b4137 +Create Date: 2023-09-28 13:11:49.061949 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision = '456234145e5e' +down_revision = '3a21a14b4137' +branch_labels = None +depends_on = None + + +def upgrade(): + process_status_codes_table = op.create_table('eft_process_status_codes', + sa.Column('code', sa.String(length=20), nullable=False), + sa.Column('description', sa.String(length=100), nullable=False), + sa.PrimaryKeyConstraint('code') + ) + + op.bulk_insert( + process_status_codes_table, + [ + {'code': 'COMPLETED', 'description': 'Record or File was able to be fully processed.'}, + {'code': 'INPROGRESS', 'description': 'Record or File processing in progress.'}, + {'code': 'FAILED', 'description': 'Record or File failed to process.'}, + {'code': 'PARTIAL', 'description': 'Record or File was partially processed as there were some errors.'} + ] + ) + + op.create_table('eft_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('completed_on', sa.DateTime(), nullable=True), + sa.Column('created_on', sa.DateTime(), nullable=False), + sa.Column('deposit_from_date', sa.DateTime(), nullable=True), + sa.Column('deposit_to_date', sa.DateTime(), nullable=True), + sa.Column('file_creation_date', sa.DateTime(), nullable=True), + sa.Column('file_ref', sa.String(), nullable=False), + sa.Column('number_of_details', sa.Integer(), nullable=True), + sa.Column('status_code', sa.String(length=20), nullable=False), + sa.Column('total_deposit_cents', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['status_code'], ['eft_process_status_codes.code']), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table('eft_transactions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('created_on', sa.DateTime(), nullable=False), + sa.Column('completed_on', sa.DateTime(), nullable=True), + sa.Column('line_number', sa.Integer(), nullable=False), + sa.Column('line_type', sa.String(20), nullable=False), + sa.Column('batch_number', sa.String(10), nullable=True), + sa.Column('sequence_number', sa.String(3), nullable=True), + sa.Column('jv_type', sa.String(1), nullable=True), + sa.Column('jv_number', sa.String(10), nullable=True), + sa.Column('batch_number', sa.String, nullable=True), + sa.Column('last_updated_on', sa.DateTime(), nullable=False), + sa.Column('status_code', sa.String(length=20), nullable=False), + sa.Column('error_messages', postgresql.ARRAY(sa.String(150), dimensions=1), nullable=True), + sa.ForeignKeyConstraint(['status_code'], ['eft_process_status_codes.code']), + sa.ForeignKeyConstraint(['file_id'], ['eft_files.id']), + sa.PrimaryKeyConstraint('id') + ) + + +def downgrade(): + op.drop_table('eft_transactions') + op.drop_table('eft_files') + op.drop_table('eft_process_status_codes') diff --git a/pay-api/src/pay_api/models/__init__.py b/pay-api/src/pay_api/models/__init__.py index a75c708f3..a45190062 100755 --- a/pay-api/src/pay_api/models/__init__.py +++ b/pay-api/src/pay_api/models/__init__.py @@ -26,6 +26,9 @@ from .db import db, ma # noqa: I001 from .disbursement_status_code import DisbursementStatusCode from .distribution_code import DistributionCode, DistributionCodeLink +from .eft_file import EFTFile +from .eft_process_status_code import EFTProcessStatusCode +from .eft_transaction import EFTTransaction from .ejv_file import EjvFile from .ejv_header import EjvHeader from .ejv_invoice_link import EjvInvoiceLink diff --git a/pay-api/src/pay_api/models/eft_file.py b/pay-api/src/pay_api/models/eft_file.py new file mode 100644 index 000000000..14a971f9d --- /dev/null +++ b/pay-api/src/pay_api/models/eft_file.py @@ -0,0 +1,64 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Model to handle EFT file processing.""" + +from datetime import datetime + +from sqlalchemy import ForeignKey + +from pay_api.utils.enums import EFTProcessStatus +from .base_model import BaseModel +from .db import db + + +class EFTFile(BaseModel): # pylint: disable=too-many-instance-attributes + """This class manages the file data for EFT transactions.""" + + __tablename__ = 'eft_files' + # this mapper is used so that new and old versions of the service can be run simultaneously, + # making rolling upgrades easier + # This is used by SQLAlchemy to explicitly define which fields we're interested + # so it doesn't freak out and say it can't map the structure if other fields are present. + # This could occur from a failed deploy or during an upgrade. + # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous + # and can interfere with Alembic upgrades. + # + # NOTE: please keep mapper names in alpha-order, easier to track that way + # Exception, id is always first, _fields first + __mapper_args__ = { + 'include_properties': [ + 'id', + 'completed_on', + 'created_on', + 'deposit_from_date', + 'deposit_to_date', + 'file_creation_date', + 'file_ref', + 'number_of_details', + 'status_code', + 'total_deposit_cents' + ] + } + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + created_on = db.Column('created_on', db.DateTime, nullable=False, default=datetime.now) + completed_on = db.Column('completed_on', db.DateTime, nullable=True) + deposit_from_date = db.Column('deposit_from_date', db.DateTime, nullable=True) + deposit_to_date = db.Column('deposit_to_date', db.DateTime, nullable=True) + file_creation_date = db.Column('file_creation_date', db.DateTime, nullable=True) + number_of_details = db.Column('number_of_details', db.Integer, nullable=True) + total_deposit_cents = db.Column('total_deposit_cents', db.BigInteger, nullable=True) + file_ref = db.Column('file_ref', db.String, nullable=False, index=True) + status_code = db.Column(db.String, ForeignKey('eft_process_status_codes.code'), + default=EFTProcessStatus.IN_PROGRESS.value, nullable=False) diff --git a/pay-api/src/pay_api/models/eft_process_status_code.py b/pay-api/src/pay_api/models/eft_process_status_code.py new file mode 100644 index 000000000..fd8c4e6a0 --- /dev/null +++ b/pay-api/src/pay_api/models/eft_process_status_code.py @@ -0,0 +1,51 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Model to capture the state of EFT data being processed.""" + +from .code_table import CodeTable +from .db import db, ma + + +class EFTProcessStatusCode(db.Model, CodeTable): + """This class manages all of the base data about a EFT Process statuss code.""" + + __tablename__ = 'eft_process_status_codes' + # this mapper is used so that new and old versions of the service can be run simultaneously, + # making rolling upgrades easier + # This is used by SQLAlchemy to explicitly define which fields we're interested + # so it doesn't freak out and say it can't map the structure if other fields are present. + # This could occur from a failed deploy or during an upgrade. + # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous + # and can interfere with Alembic upgrades. + # + # NOTE: please keep mapper names in alpha-order, easier to track that way + # Exception, id is always first, _fields first + __mapper_args__ = { + 'include_properties': [ + 'code', + 'description' + ] + } + + code = db.Column(db.String(20), primary_key=True) + description = db.Column('description', db.String(100), nullable=False) + + +class EFTProcessStatusCodeSchema(ma.ModelSchema): # pylint: disable=too-many-ancestors + """Main schema used to serialize the Status Code.""" + + class Meta: # pylint: disable=too-few-public-methods + """Returns all the fields from the SQLAlchemy class.""" + + model = EFTProcessStatusCode diff --git a/pay-api/src/pay_api/models/eft_transaction.py b/pay-api/src/pay_api/models/eft_transaction.py new file mode 100644 index 000000000..9bd4b1bb6 --- /dev/null +++ b/pay-api/src/pay_api/models/eft_transaction.py @@ -0,0 +1,69 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Model to handle EFT file processing.""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, String +from sqlalchemy.dialects.postgresql import ARRAY + +from .base_model import BaseModel +from .db import db + + +class EFTTransaction(BaseModel): # pylint: disable=too-many-instance-attributes + """This class manages the file data for EFT transactions.""" + + __tablename__ = 'eft_transactions' + # this mapper is used so that new and old versions of the service can be run simultaneously, + # making rolling upgrades easier + # This is used by SQLAlchemy to explicitly define which fields we're interested + # so it doesn't freak out and say it can't map the structure if other fields are present. + # This could occur from a failed deploy or during an upgrade. + # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous + # and can interfere with Alembic upgrades. + # + # NOTE: please keep mapper names in alpha-order, easier to track that way + # Exception, id is always first, _fields first + __mapper_args__ = { + 'include_properties': [ + 'id', + 'batch_number', + 'completed_on', + 'created_on', + 'error_messages', + 'file_id', + 'last_updated_on', + 'line_number', + 'line_type', + 'jv_type', + 'jv_number', + 'sequence_number', + 'status_code' + ] + } + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + batch_number = db.Column('batch_number', db.String(10), nullable=True) + completed_on = db.Column('completed_on', db.DateTime, nullable=True) + created_on = db.Column('created_on', db.DateTime, nullable=False, default=datetime.now) + error_messages = db.Column(ARRAY(String, dimensions=1), nullable=True) + file_id = db.Column(db.Integer, ForeignKey('eft_files.id'), nullable=False, index=True) + last_updated_on = db.Column('last_updated_on', db.DateTime, nullable=False, default=datetime.now) + line_number = db.Column('line_number', db.Integer, nullable=False) + line_type = db.Column('line_type', db.String(), nullable=False) + jv_type = db.Column('jv_type', db.String(1), nullable=True) + jv_number = db.Column('jv_number', db.String(10), nullable=True) + sequence_number = db.Column('sequence_number', db.String(3), nullable=True) + status_code = db.Column(db.String, ForeignKey('eft_process_status_codes.code'), nullable=False) diff --git a/pay-api/src/pay_api/utils/enums.py b/pay-api/src/pay_api/utils/enums.py index 1927c2e3c..77c8b5e42 100644 --- a/pay-api/src/pay_api/utils/enums.py +++ b/pay-api/src/pay_api/utils/enums.py @@ -293,3 +293,20 @@ class CfsReceiptStatus(Enum): """Routing Slip Receipt Status.""" REV = 'REV' + + +class EFTProcessStatus(Enum): + """EFT Process Status.""" + + COMPLETED = 'COMPLETED' + IN_PROGRESS = 'INPROGRESS' + FAILED = 'FAILED' + PARTIAL = 'PARTIAL' + + +class EFTFileLineType(Enum): + """EFT File (TDI17) Line types.""" + + HEADER = 'HEADER' + TRANSACTION = 'TRANSACTION' + TRAILER = 'TRAILER' diff --git a/pay-api/tests/unit/models/test_eft_file.py b/pay-api/tests/unit/models/test_eft_file.py new file mode 100644 index 000000000..ac7402431 --- /dev/null +++ b/pay-api/tests/unit/models/test_eft_file.py @@ -0,0 +1,71 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the EFT File model. + +Test-Suite to ensure that the EFT File model is working as expected. +""" +from datetime import datetime + +from pay_api.models.eft_file import EFTFile as EFTFileModel +from pay_api.utils.enums import EFTProcessStatus + + +def test_eft_file_defaults(session): + """Assert eft file defaults are stored.""" + eft_file = EFTFileModel() + eft_file.file_ref = 'test.txt' + eft_file.save() + + assert eft_file.id is not None + assert eft_file.created_on is not None + assert eft_file.status_code == EFTProcessStatus.IN_PROGRESS.value + + +def test_eft_file_all_attributes(session): + """Assert all eft file attributes are stored.""" + eft_file = EFTFileModel() + + file_creation = datetime(2023, 9, 30, 1, 0) + created_on = datetime(2023, 9, 30, 10, 0) + completed_on = datetime(2023, 9, 30, 11, 0) + deposit_from_date = datetime(2023, 9, 28) + deposit_to_date = datetime(2023, 9, 29) + number_of_details = 10 + total_deposit_cents = 125000 + + eft_file.file_creation_date = file_creation + eft_file.created_on = created_on + eft_file.completed_on = completed_on + eft_file.deposit_from_date = deposit_from_date + eft_file.deposit_to_date = deposit_to_date + eft_file.number_of_details = number_of_details + eft_file.total_deposit_cents = total_deposit_cents + eft_file.file_ref = 'test.txt' + eft_file.status_code = EFTProcessStatus.COMPLETED.value + eft_file.save() + + assert eft_file.id is not None + eft_file = EFTFileModel.find_by_id(eft_file.id) + + assert eft_file is not None + assert eft_file.status_code == EFTProcessStatus.COMPLETED.value + assert eft_file.file_creation_date == file_creation + assert eft_file.created_on == created_on + assert eft_file.completed_on == completed_on + assert eft_file.deposit_from_date == deposit_from_date + assert eft_file.deposit_to_date == deposit_to_date + assert eft_file.number_of_details == number_of_details + assert eft_file.total_deposit_cents == total_deposit_cents + assert eft_file.file_ref == 'test.txt' diff --git a/pay-api/tests/unit/models/test_eft_transaction.py b/pay-api/tests/unit/models/test_eft_transaction.py new file mode 100644 index 000000000..127601ab8 --- /dev/null +++ b/pay-api/tests/unit/models/test_eft_transaction.py @@ -0,0 +1,102 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the EFT Transaction model. + +Test-Suite to ensure that the EFT Transaction model is working as expected. +""" +from datetime import datetime + +from pay_api.models import db +from pay_api.models.eft_file import EFTFile as EFTFileModel +from pay_api.models.eft_transaction import EFTTransaction as EFTTransactionModel +from pay_api.utils.enums import EFTFileLineType, EFTProcessStatus + + +def test_eft_transaction_defaults(session): + """Assert eft transaction defaults are stored.""" + eft_file = EFTFileModel() + eft_file.file_ref = 'test.txt' + eft_file.save() + + assert eft_file.id is not None + + eft_transaction = EFTTransactionModel() + eft_transaction.file_id = eft_file.id + eft_transaction.line_number = 1 + eft_transaction.line_type = EFTFileLineType.HEADER.value + eft_transaction.status_code = EFTProcessStatus.FAILED.value + eft_transaction.save() + + eft_transaction = db.session.query(EFTTransactionModel).filter( + EFTTransactionModel.id == eft_transaction.id).one_or_none() + + assert eft_transaction.id is not None + assert eft_transaction.created_on is not None + assert eft_transaction.last_updated_on is not None + assert eft_transaction.created_on.replace(microsecond=0) == eft_transaction.last_updated_on.replace(microsecond=0) + assert eft_transaction.status_code == EFTProcessStatus.FAILED.value + assert eft_transaction.file_id == eft_file.id + assert eft_transaction.line_number == 1 + assert eft_transaction.line_type == EFTFileLineType.HEADER.value + + +def test_eft_file_all_attributes(session): + """Assert all eft transaction attributes are stored.""" + eft_file = EFTFileModel() + eft_file.file_ref = 'test.txt' + eft_file.save() + + assert eft_file.id is not None + + completed_on = datetime(2023, 9, 30, 10, 0) + error_messages = ['message 1', 'message 2'] + batch_number = '123456789' + jv_type = 'I' + jv_number = '5678910' + sequence_number = '001' + + eft_transaction = EFTTransactionModel() + eft_transaction.file_id = eft_file.id + eft_transaction.batch_number = batch_number + eft_transaction.sequence_number = sequence_number + eft_transaction.jv_type = jv_type + eft_transaction.jv_number = jv_number + eft_transaction.line_number = 2 + eft_transaction.line_type = EFTFileLineType.TRANSACTION.value + eft_transaction.status_code = EFTProcessStatus.COMPLETED.value + eft_transaction.completed_on = completed_on + eft_transaction.error_messages = error_messages + eft_transaction.save() + + assert eft_transaction.id is not None + + eft_transaction = eft_transaction.find_by_id(eft_transaction.id) + + assert eft_transaction is not None + assert eft_transaction.created_on is not None + assert eft_transaction.last_updated_on is not None + assert eft_transaction.created_on.replace(microsecond=0) == eft_transaction.last_updated_on.replace(microsecond=0) + assert eft_transaction.status_code == EFTProcessStatus.COMPLETED.value + assert eft_transaction.file_id == eft_file.id + assert eft_transaction.sequence_number == sequence_number + assert eft_transaction.batch_number == batch_number + assert eft_transaction.jv_type == jv_type + assert eft_transaction.jv_number == jv_number + assert eft_transaction.line_number == 2 + assert eft_transaction.line_type == EFTFileLineType.TRANSACTION.value + assert eft_transaction.completed_on == completed_on + assert eft_transaction.error_messages == error_messages + assert eft_transaction.error_messages[0] == 'message 1' + assert eft_transaction.error_messages[1] == 'message 2'