From 1fd77dff83aea5ab6484bc8bf7a5a9dbaa47f844 Mon Sep 17 00:00:00 2001 From: tstabrawa <59430211+tstabrawa@users.noreply.github.com> Date: Sun, 14 Apr 2024 16:27:55 -0500 Subject: [PATCH] Update mapping for new Schwab CSV format Remove account ID parsing, custom_header checking, skipping extra lines, etc. Tweak field names. Update tests. --- csv2ofx/mappings/schwabchecking.py | 58 +++---------------- .../schwab-checking-baltest-case1.ofx | 2 +- .../schwab-checking-baltest-case2.ofx | 2 +- .../schwab-checking-baltest-case3.ofx | 2 +- .../schwab-checking-baltest-case4.ofx | 2 +- .../schwab-checking-baltest-case5.ofx | 2 +- .../schwab-checking-baltest-case6.ofx | 2 +- .../schwab-checking-baltest-case7.ofx | 2 +- data/converted/schwab-checking-msmoney.ofx | 2 +- data/converted/schwab-checking.ofx | 2 +- data/test/schwab-checking-baltest-case1.csv | 13 ++--- data/test/schwab-checking-baltest-case2.csv | 13 ++--- data/test/schwab-checking-baltest-case3.csv | 13 ++--- data/test/schwab-checking-baltest-case4.csv | 13 ++--- data/test/schwab-checking-baltest-case5.csv | 13 ++--- data/test/schwab-checking-baltest-case6.csv | 13 ++--- data/test/schwab-checking-baltest-case7.csv | 9 +-- data/test/schwab-checking.csv | 13 ++--- 18 files changed, 54 insertions(+), 122 deletions(-) diff --git a/csv2ofx/mappings/schwabchecking.py b/csv2ofx/mappings/schwabchecking.py index b6565b2..489ab05 100644 --- a/csv2ofx/mappings/schwabchecking.py +++ b/csv2ofx/mappings/schwabchecking.py @@ -4,64 +4,20 @@ from operator import itemgetter import re -# Default value if account_id not found in header rows. -schwab_account_id = "12345" - - -def is_real(transaction): - # Filter-out header line & lines that only have one column - type = transaction.get('Type') - return type and (type != "Type") - - -def schwab_filter(transaction): - if is_real(transaction): - return True - - # Sanity check header row in CSV matches custom_header - line = transaction.get(mapping["custom_header"][0]) - if line in mapping["custom_header"]: - for column in mapping["custom_header"]: - found_col = transaction.get(column) - if found_col != column: - raise ValueError("Header row mismatch (expected: '" + column + - "', found: '" + found_col + "')") - - # Look for account_id in one of the lines being filtered out - global schwab_account_id - match = re.search(r'^Transactions for .* account (\.*\d*) as of', line) - if match: - schwab_account_id = match.group(1) - return False - - -def get_account_id(transaction): - global schwab_account_id - return schwab_account_id - - mapping = { - # Technically Schwab Bank CSVs do have a header, but we want to extract - # the account_id from the line that comes before the header, so we use - # custom_header instead of parsing it from the second line of the CSV. - 'custom_header': ["Date","Type","Check #","Description","Withdrawal (-)", - "Deposit (+)","RunningBalance"], - 'has_header': False, - 'filter': schwab_filter, + 'has_header': True, + 'filter': lambda tr: True if tr.get('Status') == 'Posted' else False, 'is_split': False, 'bank': 'Charles Schwab Bank, N.A.', 'bank_id': '121202211', - 'account_id': get_account_id, + 'account_id': '12345', # Change to your actual account number if desired 'currency': 'USD', 'account': 'Charles Schwab Checking', - # Schwab Bank includes three lines of text in their CSV which otherwise - # break date parsing. Filtering by lines with no "Type" (and faking out - # the date for those lines) avoids the problem. - 'date': lambda tr: tr.get('Date') if is_real(tr) else "1-1-1970", - 'check_num': itemgetter('Check #'), + 'date': itemgetter('Date'), + 'check_num': itemgetter('CheckNumber'), 'payee': itemgetter('Description'), 'desc': itemgetter('Description'), - 'type': lambda tr: 'debit' if tr.get('Withdrawal (-)') != '' else 'credit', - 'amount': lambda tr: tr.get('Deposit (+)') or tr.get('Withdrawal (-)'), + 'type': lambda tr: 'debit' if tr.get('Withdrawal') != '' else 'credit', + 'amount': lambda tr: tr.get('Deposit') or tr.get('Withdrawal'), 'balance': itemgetter('RunningBalance'), } diff --git a/data/converted/schwab-checking-baltest-case1.ofx b/data/converted/schwab-checking-baltest-case1.ofx index 0d3239c..3a258c1 100644 --- a/data/converted/schwab-checking-baltest-case1.ofx +++ b/data/converted/schwab-checking-baltest-case1.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-baltest-case2.ofx b/data/converted/schwab-checking-baltest-case2.ofx index 129c277..fca0bc8 100644 --- a/data/converted/schwab-checking-baltest-case2.ofx +++ b/data/converted/schwab-checking-baltest-case2.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-baltest-case3.ofx b/data/converted/schwab-checking-baltest-case3.ofx index e57a286..dc611ef 100644 --- a/data/converted/schwab-checking-baltest-case3.ofx +++ b/data/converted/schwab-checking-baltest-case3.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-baltest-case4.ofx b/data/converted/schwab-checking-baltest-case4.ofx index cb25db8..263faff 100644 --- a/data/converted/schwab-checking-baltest-case4.ofx +++ b/data/converted/schwab-checking-baltest-case4.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-baltest-case5.ofx b/data/converted/schwab-checking-baltest-case5.ofx index 666498c..24282de 100644 --- a/data/converted/schwab-checking-baltest-case5.ofx +++ b/data/converted/schwab-checking-baltest-case5.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-baltest-case6.ofx b/data/converted/schwab-checking-baltest-case6.ofx index 6a650d9..e4e27ef 100644 --- a/data/converted/schwab-checking-baltest-case6.ofx +++ b/data/converted/schwab-checking-baltest-case6.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-baltest-case7.ofx b/data/converted/schwab-checking-baltest-case7.ofx index b4f1393..b5bff07 100644 --- a/data/converted/schwab-checking-baltest-case7.ofx +++ b/data/converted/schwab-checking-baltest-case7.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking-msmoney.ofx b/data/converted/schwab-checking-msmoney.ofx index 8c46ef5..cc00bd3 100644 --- a/data/converted/schwab-checking-msmoney.ofx +++ b/data/converted/schwab-checking-msmoney.ofx @@ -29,7 +29,7 @@ NEWFILEUID:NONE USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/converted/schwab-checking.ofx b/data/converted/schwab-checking.ofx index 0d3239c..3a258c1 100644 --- a/data/converted/schwab-checking.ofx +++ b/data/converted/schwab-checking.ofx @@ -22,7 +22,7 @@ ENCODING:UTF-8 USD 121202211 - ...123 + 12345 CHECKING diff --git a/data/test/schwab-checking-baltest-case1.csv b/data/test/schwab-checking-baltest-case1.csv index 3e1fee4..dbba925 100644 --- a/data/test/schwab-checking-baltest-case1.csv +++ b/data/test/schwab-checking-baltest-case1.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" -"08/14/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/09/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" -"08/04/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"08/14/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/09/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" +"08/04/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" diff --git a/data/test/schwab-checking-baltest-case2.csv b/data/test/schwab-checking-baltest-case2.csv index 9707ae0..64c461e 100644 --- a/data/test/schwab-checking-baltest-case2.csv +++ b/data/test/schwab-checking-baltest-case2.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" -"08/17/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/04/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" -"08/09/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"08/17/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/04/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"08/09/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" diff --git a/data/test/schwab-checking-baltest-case3.csv b/data/test/schwab-checking-baltest-case3.csv index 085ab42..410aec8 100644 --- a/data/test/schwab-checking-baltest-case3.csv +++ b/data/test/schwab-checking-baltest-case3.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/04/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" -"08/09/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" -"08/17/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/04/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"08/09/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" +"08/17/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" diff --git a/data/test/schwab-checking-baltest-case4.csv b/data/test/schwab-checking-baltest-case4.csv index ea16b7f..d51c70d 100644 --- a/data/test/schwab-checking-baltest-case4.csv +++ b/data/test/schwab-checking-baltest-case4.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" -"08/17/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/09/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" -"08/04/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"08/17/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/09/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" +"08/04/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" diff --git a/data/test/schwab-checking-baltest-case5.csv b/data/test/schwab-checking-baltest-case5.csv index c1ae840..ddad54c 100644 --- a/data/test/schwab-checking-baltest-case5.csv +++ b/data/test/schwab-checking-baltest-case5.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" -"08/17/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" -"08/17/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"08/17/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" +"08/17/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" diff --git a/data/test/schwab-checking-baltest-case6.csv b/data/test/schwab-checking-baltest-case6.csv index 6b1125b..1d694ad 100644 --- a/data/test/schwab-checking-baltest-case6.csv +++ b/data/test/schwab-checking-baltest-case6.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" -"08/17/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/17/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" -"08/17/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"08/17/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/17/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" +"08/17/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" diff --git a/data/test/schwab-checking-baltest-case7.csv b/data/test/schwab-checking-baltest-case7.csv index a6a04fd..2223a67 100644 --- a/data/test/schwab-checking-baltest-case7.csv +++ b/data/test/schwab-checking-baltest-case7.csv @@ -1,6 +1,3 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","CHECK","558","Check Paid #558","$20.00","","$858.47" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","CHECK","558","Check Paid #558","$20.00","","$858.47" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" diff --git a/data/test/schwab-checking.csv b/data/test/schwab-checking.csv index 3e1fee4..dbba925 100644 --- a/data/test/schwab-checking.csv +++ b/data/test/schwab-checking.csv @@ -1,8 +1,5 @@ -"Transactions for Checking account ...123 as of 08/20/2022 04:04:41 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions are not reflected within this sort criterion." -"Posted Transactions" -"08/17/2022","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" -"08/14/2022","ATM","","BMO HARRIS BANK","$103.00","","$858.47" -"08/09/2022","CHECK","558","Check Paid #558","$75.00","","$961.47" -"08/04/2022","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47" +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"08/17/2022","Posted","DEPOSIT","","Deposit Mobile Banking","","$20.00","$878.47" +"08/14/2022","Posted","ATM","","BMO HARRIS BANK","$103.00","","$858.47" +"08/09/2022","Posted","CHECK","558","Check Paid #558","$75.00","","$961.47" +"08/04/2022","Posted","ACH","","PAYPAL INST XFER 220803~ Tran: ACHDW","$57.27","","$1,036.47"