From 37b9b03d750dc92424c7eb10a2e926ecd7a5483e Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 16 Nov 2023 19:00:58 -0800 Subject: [PATCH 01/90] notes --- oldabe/money_in.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 93af750..346facf 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -326,6 +326,17 @@ def write_append_itemized_payments(itemized_payments): writer.writerow(astuple(row)) +# TODO - when we write debts +# 1) read all from the file and transform into a hash, where the key is a +# unique identifier constructed from the email + payment file and the value +# is a debt object +# 2) take the newly created/modified debt objects (all in one list) and iterate +# through - searching the full hash for each new debt object - modify if it +# was found, and add to the hash if not found +# 3) convert the hash into a list, ordered by created_at field, then write to +# the debts file (completely replacing existing contents) + + def write_valuation(valuation): rounded_valuation = f"{valuation:.2f}" valuation_file = os.path.join(ABE_ROOT, VALUATION_FILE) @@ -438,6 +449,7 @@ def process_payments(instruments, attributions): # first, process instruments (i.e. pay fees) transactions = distribute_payment(payment, instruments) new_transactions += transactions + # TODO - may need to calculate this differently with debts in the mix amount_paid_out = sum(t.amount for t in transactions) # deduct the amount paid out to instruments before # processing it for attributions From 6bd6af21504f7acaf34a48680c5fe9503d6cb80c Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Sep 2023 20:41:41 -0700 Subject: [PATCH 02/90] update requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9eb4951..c0670a2 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -requirements = ['click'] +requirements = [] test_requirements = [ 'pytest', @@ -12,7 +12,7 @@ 'coveralls', ] -dev_requirements = ['flake8', 'bump2version', 'sphinx', 'pre-commit', 'black'] +dev_requirements = ['flake8', 'black'] setup_requirements = ['pytest-runner'] From 3852b18c6be6a3794eca6048f79c522cc56f5449 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Sep 2023 20:41:52 -0700 Subject: [PATCH 03/90] commit recent wip around debts, "advances", etc. --- oldabe/models.py | 6 ++ oldabe/money_in.py | 185 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 164 insertions(+), 27 deletions(-) diff --git a/oldabe/models.py b/oldabe/models.py index e95a810..30678b1 100644 --- a/oldabe/models.py +++ b/oldabe/models.py @@ -23,6 +23,12 @@ class Debt: commit_hash: str = None created_at: datetime = field(default_factory=datetime.utcnow) + def is_fulfilled(self): + return self.amount_paid == self.amount + + def amount_remaining(self): + return self.amount - self.amount_paid + @dataclass class Payment: diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 346facf..c4bf20b 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -8,12 +8,15 @@ import subprocess from .models import Attribution, Payment, ItemizedPayment, Transaction -ABE_ROOT = 'abe' +# ABE_ROOT = 'abe' +ABE_ROOT = '.' PAYMENTS_DIR = os.path.join(ABE_ROOT, 'payments') NONATTRIBUTABLE_PAYMENTS_DIR = os.path.join( ABE_ROOT, 'payments', 'nonattributable' ) +UNPAYABLE_CONTRIBUTORS_FILE = 'unpayable_contributors.txt' TRANSACTIONS_FILE = 'transactions.txt' +DEBTS_FILE = 'debts.txt' ITEMIZED_PAYMENTS_FILE = 'itemized_payments.txt' PRICE_FILE = 'price.txt' VALUATION_FILE = 'valuation.txt' @@ -265,24 +268,10 @@ def get_rounding_difference(attributions): return difference -def renormalize(attributions, incoming_attribution): - """ - The incoming attribution is determined as a proportion of the total - posterior valuation. As the existing attributions total to 1 and don't - account for it, they must be proportionately scaled so that their new total - added to the incoming attribution once again totals to one, i.e. is - "renormalized." This effectively dilutes the attributions by the magnitude - of the incoming attribution. - """ - target_proportion = Decimal("1") - incoming_attribution.share +def renormalize(attributions, target_proportion): for email in attributions: # renormalize to reflect dilution attributions[email] *= target_proportion - # add incoming share to existing investor or record new investor - existing_attribution = attributions.get(incoming_attribution.email, None) - attributions[incoming_attribution.email] = ( - existing_attribution if existing_attribution else 0 - ) + incoming_attribution.share def correct_rounding_error(attributions, incoming_attribution): @@ -349,8 +338,25 @@ def dilute_attributions(incoming_attribution, attributions): """ Incorporate a fresh attributive share by diluting existing attributions, and correcting any rounding error that may arise from this. + + The incoming attribution is determined as a proportion of the total + posterior valuation. As the existing attributions total to 1 and don't + account for it, they must be proportionately scaled so that their new total + added to the incoming attribution once again totals to one, i.e. is + "renormalized." This effectively dilutes the attributions by the magnitude + of the incoming attribution. """ - renormalize(attributions, incoming_attribution) + target_proportion = Decimal("1") - incoming_attribution.share + for email in attributions: + # renormalize to reflect dilution + attributions[email] *= target_proportion + + # add incoming share to existing investor or record new investor + existing_attribution = attributions.get(incoming_attribution.email, None) + attributions[incoming_attribution.email] = ( + existing_attribution if existing_attribution else 0 + ) + incoming_attribution.share + correct_rounding_error(attributions, incoming_attribution) @@ -371,24 +377,146 @@ def get_git_revision_short_hash() -> str: ) -def distribute_payment(payment, attributions): - """ - Generate transactions to contributors from a (new) payment. +def read_debts(): + debts_file = os.path.join(ABE_ROOT, DEBTS_FILE) + debts = [] + with open(debts_file) as f: + for ( + email, + amount, + amount_paid, + payment_file, + commit_hash, + created_at, + ) in csv.reader(f): + debts.append(Debt(email, amount, amount_paid, payment_file, commit_hash, created_at)) - We consult the attribution file and determine how much is owed - to each contributor based on the current percentages, generating a - fresh entry in the transactions file for each contributor. + return debts + + +def _is_debt_fulfilled(debt): + return debt.amount_paid != debt.amount + + +def get_payable_debts(unpayable_contributors): + debts = read_debts() + debts = [d for d in debts + if not d.is_fulfilled() + and d.email not in unpayable_contributors] + return debts + + +def pay_debts(payable_debts, payment): + # returns debts, transactions + # go through debts in chronological order + # pay each as much as possible, stopping when either money runs out, or no further debts + updated_debts = [] + transactions = [] + for debt in sorted(payable_debts, key=lambda x: x.created_at): + payable_amount = min(payment.amount, debt.amount_remaining()) + if payable_amount < ACCOUNTING_ZERO: + break + debt.amount_paid += payable_amount + payment.amount -= payable_amount + transaction = Transaction(debt.email, payable_amount, payment.file, debt.commit_hash) + transactions.append(transaction) + updated_debts.append(debt) + + return updated_debts, transactions + + +def get_unpayable_contributors(): + unpayable_contributors_file = os.path.join(ABE_ROOT, UNPAYABLE_CONTRIBUTORS_FILE) + contributors = [] + with open(unpayable_contributors_file) as f: + for contributor in f: + contributor = contributor.strip() + if contributor: + contributors.append(contributor) + return contributors + + +def create_debts(remaining_amount, unpayable_contributors, attributions, payment_file): + unpayable_attributions = {email: share for email in attributions if email in unpayable_contributors} + debts = [] + commit_hash = get_git_revision_short_hash() + for email, share in unpayable_attributions.items(): + debt_amount = share * remaining_amount + debt = Debt(email, debt_amount, payment_file=payment_file, commit_hash=commit_hash)) + debts.append(debt) + + return debts + + +def write_debts(): + # 1. Build a hash of all the processed debts, generating an id for each (based on email and payment file) + # 2. read the existing debts file, row by row + # 3. if the debt in the row is in the "processed" hash, then write the processed version instead of the input version + # and remove it from the hash, otherwise write the input version + # 4. write the debts that remain in the processed hash + pass + + +def distribute_remaining_amount(remaining_amount, attributions, payment, unpayable_contributors): + """ + After paying off debts, distribute remaining amount to payable contributors + in the attributions file, by renormalizing after excluding unpayable + contributors. """ commit_hash = get_git_revision_short_hash() + target_proportion = 1 / (1 - sum(attributions[email] for email in unpayable_contributors)) + remainder_attributions = {} + for email in attributions: + # renormalize to reflect dilution + remainder_attributions[email] = attributions[email] * target_proportion # figure out how much each person in the attributions file is owed from # this payment, generating a transaction for each stakeholder. transactions = generate_transactions( - payment.amount, attributions, payment.file, commit_hash + remaining_amount, remainder_attributions, payment.file, commit_hash ) + return transactions +def distribute_payment(payment, attributions): + """ + Generate transactions to contributors from a (new) payment. + + We consult the attribution file and determine how much is owed + to each contributor based on the current percentages, generating a + fresh entry in the transactions file for each contributor. + """ + + # 1. check payable outstanding debts + # 2. pay them off in chronological order (maybe partially) + # 3. (if leftover) identify unpayable people in the relevant attributions file + # 4. record debt for each of them according to their attribution + # 5. distribute according to renormalized (remainder) attributions + # a. find out how much each person is owed according to attributions + # b. renormalize and find out how much we are about to pay them + # c. create transactions for the difference between b and their total advance (which could be zero), if above zero + # d. find out who has advances and decrement all of these by the amount we are about to pay them, aggregating this amount + # e. apply the total decremented advances to everyone who has zero advances, after renormalizing over them + # f. create transactions for these payout amounts in e + # g. create advances equal to the difference between b and a + unpayable_contributors = get_unpayable_contributors() + payable_debts = get_payable_debts(unpayable_contributors) + updated_debts, debt_transactions = pay_debts(payable_debts, payment) + remaining_amount = payment.amount - sum(t.amount for t in debt_transactions) + fresh_debts = create_debts(remaining_amount, unpayable_contributors, attributions, payment.payment_file) + + debts = updated_debts + fresh_debts + + transactions = (distribute_remaining_amount(remaining_amount, + attributions, + payment, + unpayable_contributors) + if remaining_amount else []) + + return debts, transactions + + def handle_investment( payment, new_itemized_payments, attributions, price, prior_valuation ): @@ -447,7 +575,7 @@ def process_payments(instruments, attributions): unprocessed_payments = _get_unprocessed_payments() for payment in unprocessed_payments: # first, process instruments (i.e. pay fees) - transactions = distribute_payment(payment, instruments) + debts, transactions = distribute_payment(payment, instruments) new_transactions += transactions # TODO - may need to calculate this differently with debts in the mix amount_paid_out = sum(t.amount for t in transactions) @@ -465,7 +593,7 @@ def process_payments(instruments, attributions): valuation = handle_investment( payment, new_itemized_payments, attributions, price, valuation ) - return new_transactions, valuation, new_itemized_payments + return debts, new_transactions, valuation, new_itemized_payments def process_payments_and_record_updates(): @@ -478,6 +606,7 @@ def process_payments_and_record_updates(): attributions = read_attributions(ATTRIBUTIONS_FILE) ( + debts, transactions, posterior_valuation, new_itemized_payments, @@ -486,6 +615,7 @@ def process_payments_and_record_updates(): # we only write the changes to disk at the end # so that if any errors are encountered, no # changes are made. + write_debts(debts) write_append_transactions(transactions) write_attributions(attributions) write_valuation(posterior_valuation) @@ -502,4 +632,5 @@ def main(): if __name__ == "__main__": - main() + # main() + print(get_unpayable_contributors()) From 1a6108fdb33e60cf2c2c89947920fd27db3cb11b Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 5 Oct 2023 19:21:08 -0700 Subject: [PATCH 04/90] Implement logic for writing debts (commit wip from last week) --- oldabe/models.py | 3 +++ oldabe/money_in.py | 63 +++++++++++++++++++++++++++++++++------------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/oldabe/models.py b/oldabe/models.py index 30678b1..fab99b4 100644 --- a/oldabe/models.py +++ b/oldabe/models.py @@ -23,6 +23,9 @@ class Debt: commit_hash: str = None created_at: datetime = field(default_factory=datetime.utcnow) + def key(self): + return (self.email, self.payment_file) + def is_fulfilled(self): return self.amount_paid == self.amount diff --git a/oldabe/money_in.py b/oldabe/money_in.py index c4bf20b..a837736 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -407,9 +407,12 @@ def get_payable_debts(unpayable_contributors): def pay_debts(payable_debts, payment): - # returns debts, transactions - # go through debts in chronological order - # pay each as much as possible, stopping when either money runs out, or no further debts + """ + Go through debts in chronological order, and pay each as much as possible, + stopping when either the money runs out, or there are no further debts. + Returns the updated debts reflecting fresh payments to be made this time, + and transactions representing those fresh payments. + """ updated_debts = [] transactions = [] for debt in sorted(payable_debts, key=lambda x: x.created_at): @@ -437,6 +440,9 @@ def get_unpayable_contributors(): def create_debts(remaining_amount, unpayable_contributors, attributions, payment_file): + """ + Create fresh debts (to unpayable contributors). + """ unpayable_attributions = {email: share for email in attributions if email in unpayable_contributors} debts = [] commit_hash = get_git_revision_short_hash() @@ -448,13 +454,31 @@ def create_debts(remaining_amount, unpayable_contributors, attributions, payment return debts -def write_debts(): - # 1. Build a hash of all the processed debts, generating an id for each (based on email and payment file) - # 2. read the existing debts file, row by row - # 3. if the debt in the row is in the "processed" hash, then write the processed version instead of the input version - # and remove it from the hash, otherwise write the input version - # 4. write the debts that remain in the processed hash - pass +def write_debts(processed_debts): + """ + 1. Build a hash of all the processed debts, generating an id for each + (based on email and payment file). + 2. read the existing debts file, row by row. + 3. if the debt in the row is in the "processed" hash, then write the + processed version instead of the input version and remove it from the + hash, otherwise write the input version. + 4. write the debts that remain in the processed hash. + """ + existing_debts = read_debts() + processed_debts_hash = {debt.key(): debt for debt in processed_debts} + debts_file = os.path.join(ABE_ROOT, DEBTS_FILE) + writer = csv.writer(f) + with open(debts_file, 'w') as f: + for existing_debt in existing_debts: + # if the existing debt has been processed, write the processed version + # otherwise re-write the existing version + if processed_debt := processed_debts_hash.get(existing_debt.key()): + writer.writerow(astuple(processed_debt)) + del processed_debts_hash[processed_debt.key()] + else: + writer.writerow(astuple(existing_debt)) + for debt in processed_debts_hash.values(): + writer.writerow(astuple(debt)) def distribute_remaining_amount(remaining_amount, attributions, payment, unpayable_contributors): @@ -504,15 +528,20 @@ def distribute_payment(payment, attributions): payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) remaining_amount = payment.amount - sum(t.amount for t in debt_transactions) - fresh_debts = create_debts(remaining_amount, unpayable_contributors, attributions, payment.payment_file) - debts = updated_debts + fresh_debts + fresh_debts = [] + equity_transactions = [] + if remaining_amount > ACCOUNTING_ZERO: + fresh_debts = create_debts(remaining_amount, unpayable_contributors, attributions, payment.payment_file) - transactions = (distribute_remaining_amount(remaining_amount, - attributions, - payment, - unpayable_contributors) - if remaining_amount else []) + equity_transactions = (distribute_remaining_amount(remaining_amount, + attributions, + payment, + unpayable_contributors) + if remaining_amount else []) + + debts = updated_debts + fresh_debts + transactions = equity_transactions + debt_transactions return debts, transactions From ac907faa386bef100be2db2bf5c49a5deafa712b Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 16 Nov 2023 18:50:02 -0800 Subject: [PATCH 05/90] commit wip --- oldabe/models.py | 5 +++++ oldabe/money_in.py | 22 +++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/oldabe/models.py b/oldabe/models.py index fab99b4..45ae797 100644 --- a/oldabe/models.py +++ b/oldabe/models.py @@ -33,6 +33,11 @@ def amount_remaining(self): return self.amount - self.amount_paid +@dataclass +class Advance: + pass + + @dataclass class Payment: email: str = None diff --git a/oldabe/money_in.py b/oldabe/money_in.py index a837736..572d6ce 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -517,13 +517,21 @@ def distribute_payment(payment, attributions): # 3. (if leftover) identify unpayable people in the relevant attributions file # 4. record debt for each of them according to their attribution # 5. distribute according to renormalized (remainder) attributions - # a. find out how much each person is owed according to attributions - # b. renormalize and find out how much we are about to pay them - # c. create transactions for the difference between b and their total advance (which could be zero), if above zero - # d. find out who has advances and decrement all of these by the amount we are about to pay them, aggregating this amount - # e. apply the total decremented advances to everyone who has zero advances, after renormalizing over them - # f. create transactions for these payout amounts in e - # g. create advances equal to the difference between b and a + # a. find out the total amount each person would be owed (if everyone were payable) according to attributions + # b. then renormalize (by excluding unpayable people) and find out how much we are actually about to pay them + # c. create transactions for the difference between b and their total advance, if above zero + # d. create advances for all payable people equal to the difference between b and a + # e. among those we are about to pay, find out who has advances and decrement all of these by the amount + # we are about to pay them, aggregating this amount + # f. apply the total decremented advances to + # i. all those who have zero advances (including any who _now_ have zero advances), if any, + # after renormalizing over them, recording an advance to them for each such payment(!), + # since we consider this payment to exceed their attributive share this time around + # ii. all payable people, otherwise (i.e. everyone has advances), once again recording an + # advance for each such payment in this case + # f2. apply the total decremented advances to all payable contributors, + # recording an advance for each of them. + # g. create transactions for these payout amounts in f unpayable_contributors = get_unpayable_contributors() payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) From c511e95268be4e7fb12eee832e95da3296d19ce8 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 29 Nov 2023 19:33:20 -0800 Subject: [PATCH 06/90] commit wip from last time --- oldabe/money_in.py | 108 +++++++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 33 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 572d6ce..65c58ca 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -175,8 +175,8 @@ def generate_transactions(amount, attributions, payment_file, commit_hash): assert amount > 0 assert attributions transactions = [] - for email, share in attributions.items(): - t = Transaction(email, amount * share, payment_file, commit_hash) + for email, amount_owed in get_amounts_owed(amount, attributions): + t = Transaction(email, amount_owed, payment_file, commit_hash) transactions.append(t) return transactions @@ -268,9 +268,10 @@ def get_rounding_difference(attributions): return difference -def renormalize(attributions, target_proportion): +def normalize(attributions): + total_share = sum(share for _, share in attributions.items()) + target_proportion = Decimal("1") / total_share for email in attributions: - # renormalize to reflect dilution attributions[email] *= target_proportion @@ -443,7 +444,10 @@ def create_debts(remaining_amount, unpayable_contributors, attributions, payment """ Create fresh debts (to unpayable contributors). """ - unpayable_attributions = {email: share for email in attributions if email in unpayable_contributors} + unpayable_attributions = {email: share + for email, share + in attributions.items() + if email in unpayable_contributors} debts = [] commit_hash = get_git_revision_short_hash() for email, share in unpayable_attributions.items(): @@ -481,25 +485,78 @@ def write_debts(processed_debts): writer.writerow(astuple(debt)) -def distribute_remaining_amount(remaining_amount, attributions, payment, unpayable_contributors): - """ - After paying off debts, distribute remaining amount to payable contributors - in the attributions file, by renormalizing after excluding unpayable - contributors. - """ - commit_hash = get_git_revision_short_hash() - target_proportion = 1 / (1 - sum(attributions[email] for email in unpayable_contributors)) +def renormalize(attributions, excluded_contributors): + target_proportion = 1 / (1 - sum(attributions[email] for email in excluded_contributors)) remainder_attributions = {} for email in attributions: # renormalize to reflect dilution remainder_attributions[email] = attributions[email] * target_proportion + return remainder_attributions + + +def get_amounts_owed(total_amount, attributions): + return {email: share * total_amount + for email, share in attributions.items()} + - # figure out how much each person in the attributions file is owed from - # this payment, generating a transaction for each stakeholder. +def distribute_remaining_amount(remaining_amount, truncated_payable_attributions, payment): + """ + After paying off debts, distribute remaining amount to payable contributors + in the attributions file, by renormalizing after excluding unpayable + contributors. + """ + # $15 pre-existing advance + # $10 amount they are owed + # $20 amount we are about to pay them + # A: + # pay them $5 + # draw down $15 advance to zero + # pot is at $15 + # B: + # increase advance by $10 (i.e. b - a) + # draw down $25 to $5 + # $20 remains in the pot + # but we give it to them and increase their advance to $35 + # Next time (assume no unpayable contributors): + # B: + # advance = $35 + # a = $10 + # b = $10 + # draw $35 down to $25 + # $10 remains in the pot + # $5 (say) is allocated to them + # increase advance by $5 to $30 + # a. find out the total amount each person would be owed + # (if everyone were payable) according to attributions + # b. then renormalize (by excluding unpayable people) and + # find out how much we are actually about to pay them + # d. create advances for all payable people equal to + # the difference between b and a + # c. record "initial owed amount" for the difference between b and + # their total advance, if above zero + # e. among those we are about to pay, find out who has advances and + # decrement all of these by the amount + # we are about to pay them, aggregating this amount + # f. apply the total decremented advances to all payable contributors, + # recording an advance for each of them. + # g. create transactions for these payout amounts in f + # + # + # Goals: make sure advances eventually decrease + # and that the numbers look fair + + amounts_owed = get_amounts_owed(remaining_amount, truncated_payable_attributions) + remainder_attributions = normalize(truncated_payable_attributions) + advances transactions = generate_transactions( remaining_amount, remainder_attributions, payment.file, commit_hash ) + + + commit_hash = get_git_revision_short_hash() + + return transactions @@ -517,21 +574,6 @@ def distribute_payment(payment, attributions): # 3. (if leftover) identify unpayable people in the relevant attributions file # 4. record debt for each of them according to their attribution # 5. distribute according to renormalized (remainder) attributions - # a. find out the total amount each person would be owed (if everyone were payable) according to attributions - # b. then renormalize (by excluding unpayable people) and find out how much we are actually about to pay them - # c. create transactions for the difference between b and their total advance, if above zero - # d. create advances for all payable people equal to the difference between b and a - # e. among those we are about to pay, find out who has advances and decrement all of these by the amount - # we are about to pay them, aggregating this amount - # f. apply the total decremented advances to - # i. all those who have zero advances (including any who _now_ have zero advances), if any, - # after renormalizing over them, recording an advance to them for each such payment(!), - # since we consider this payment to exceed their attributive share this time around - # ii. all payable people, otherwise (i.e. everyone has advances), once again recording an - # advance for each such payment in this case - # f2. apply the total decremented advances to all payable contributors, - # recording an advance for each of them. - # g. create transactions for these payout amounts in f unpayable_contributors = get_unpayable_contributors() payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) @@ -540,12 +582,12 @@ def distribute_payment(payment, attributions): fresh_debts = [] equity_transactions = [] if remaining_amount > ACCOUNTING_ZERO: + # TODO: pass truncated unpayable attributions instead of (unpayable contributors, attributions) fresh_debts = create_debts(remaining_amount, unpayable_contributors, attributions, payment.payment_file) equity_transactions = (distribute_remaining_amount(remaining_amount, - attributions, - payment, - unpayable_contributors) + truncated_payable_attributions, + payment) if remaining_amount else []) debts = updated_debts + fresh_debts From fe1813180b6b300f184f5496142ea37aef9f6883 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 13 Dec 2023 21:20:08 -0700 Subject: [PATCH 07/90] WIP transitioning to the new simplified advances scheme (in the doc) --- oldabe/money_in.py | 90 +++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 54 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 65c58ca..30051c5 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -440,19 +440,17 @@ def get_unpayable_contributors(): return contributors -def create_debts(remaining_amount, unpayable_contributors, attributions, payment_file): +def create_debts(amounts_owed, unpayable_contributors, payment_file): """ Create fresh debts (to unpayable contributors). """ - unpayable_attributions = {email: share - for email, share - in attributions.items() + unpayable_amounts_owed = {email: share + for email, share in amounts_owed.items() if email in unpayable_contributors} debts = [] commit_hash = get_git_revision_short_hash() - for email, share in unpayable_attributions.items(): - debt_amount = share * remaining_amount - debt = Debt(email, debt_amount, payment_file=payment_file, commit_hash=commit_hash)) + for email, amount_owed in unpayable_amounts_owed.items(): + debt = Debt(email, amount_owed, payment_file=payment_file, commit_hash=commit_hash)) debts.append(debt) return debts @@ -505,46 +503,6 @@ def distribute_remaining_amount(remaining_amount, truncated_payable_attributions in the attributions file, by renormalizing after excluding unpayable contributors. """ - # $15 pre-existing advance - # $10 amount they are owed - # $20 amount we are about to pay them - # A: - # pay them $5 - # draw down $15 advance to zero - # pot is at $15 - # B: - # increase advance by $10 (i.e. b - a) - # draw down $25 to $5 - # $20 remains in the pot - # but we give it to them and increase their advance to $35 - # Next time (assume no unpayable contributors): - # B: - # advance = $35 - # a = $10 - # b = $10 - # draw $35 down to $25 - # $10 remains in the pot - # $5 (say) is allocated to them - # increase advance by $5 to $30 - # a. find out the total amount each person would be owed - # (if everyone were payable) according to attributions - # b. then renormalize (by excluding unpayable people) and - # find out how much we are actually about to pay them - # d. create advances for all payable people equal to - # the difference between b and a - # c. record "initial owed amount" for the difference between b and - # their total advance, if above zero - # e. among those we are about to pay, find out who has advances and - # decrement all of these by the amount - # we are about to pay them, aggregating this amount - # f. apply the total decremented advances to all payable contributors, - # recording an advance for each of them. - # g. create transactions for these payout amounts in f - # - # - # Goals: make sure advances eventually decrease - # and that the numbers look fair - amounts_owed = get_amounts_owed(remaining_amount, truncated_payable_attributions) remainder_attributions = normalize(truncated_payable_attributions) advances @@ -552,14 +510,17 @@ def distribute_remaining_amount(remaining_amount, truncated_payable_attributions remaining_amount, remainder_attributions, payment.file, commit_hash ) - - commit_hash = get_git_revision_short_hash() - return transactions +def draw_down_advances(amounts_owed, unpayable_contributors): + payable_amounts_owed = {email: share + for email, share in amounts_owed.items() + if email not in unpayable_contributors} + + def distribute_payment(payment, attributions): """ Generate transactions to contributors from a (new) payment. @@ -577,13 +538,34 @@ def distribute_payment(payment, attributions): unpayable_contributors = get_unpayable_contributors() payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) - remaining_amount = payment.amount - sum(t.amount for t in debt_transactions) + # The "available" amount is what is left over after paying off debts + available_amount = payment.amount - sum(t.amount for t in debt_transactions) fresh_debts = [] equity_transactions = [] - if remaining_amount > ACCOUNTING_ZERO: - # TODO: pass truncated unpayable attributions instead of (unpayable contributors, attributions) - fresh_debts = create_debts(remaining_amount, unpayable_contributors, attributions, payment.payment_file) + if available_amount > ACCOUNTING_ZERO: + amounts_owed = {email: share * available_amount + for email, share + in attributions.items()} + fresh_debts = create_debts(amounts_owed, + unpayable_contributors, + payment.payment_file) + redistribution_pot = sum(d.amount for d in fresh_debts) + # TODO - complete draw_down_advances - it should get the + # advances objects for each person and mutate them, then return + # the advances that have been mutated so they can be recorded + # - also needs to return the sum of amount that was drawn down + # so it can be added to the pot. - also needs to update amounts_owed + # for each person with what remains after advances were drawn down + + # TODO - decide how to represent Advances + draw_down_advances(amounts_owed, unpayable_contributors) + + # put the sum of fresh debts in the pot + # draw down advances using the available amount divided according to + # truncated payable attributions + # put the sum of drawn down advances in the pot + # finally, redistribute the pot over all contributors in truncated payable attributions equity_transactions = (distribute_remaining_amount(remaining_amount, truncated_payable_attributions, From 0c6b3be4bf7291d0ee50d20ea2f8d9a2ab328e05 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Dec 2023 21:56:33 -0700 Subject: [PATCH 08/90] draw down advances and collect them into the redistribution pot --- oldabe/models.py | 7 +++- oldabe/money_in.py | 86 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/oldabe/models.py b/oldabe/models.py index 45ae797..0067f21 100644 --- a/oldabe/models.py +++ b/oldabe/models.py @@ -35,7 +35,12 @@ def amount_remaining(self): @dataclass class Advance: - pass + email: str = None + amount: Decimal = 0 + + payment_file: str = None + commit_hash: str = None + created_at: datetime = field(default_factory=datetime.utcnow) @dataclass diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 30051c5..5e1192b 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import csv +from collections import defaultdict from decimal import Decimal, getcontext from dataclasses import astuple import re @@ -17,6 +18,7 @@ UNPAYABLE_CONTRIBUTORS_FILE = 'unpayable_contributors.txt' TRANSACTIONS_FILE = 'transactions.txt' DEBTS_FILE = 'debts.txt' +ADVANCES_FILE = 'advances.txt' ITEMIZED_PAYMENTS_FILE = 'itemized_payments.txt' PRICE_FILE = 'price.txt' VALUATION_FILE = 'valuation.txt' @@ -395,6 +397,22 @@ def read_debts(): return debts +def read_advances(): + advances_file = os.path.join(ABE_ROOT, ADVANCES_FILE) + advances = defaultdict(list) + with open(advances_file) as f: + for ( + email, + amount, + payment_file, + commit_hash, + created_at, + ) in csv.reader(f): + advances[email].append(Advance(email, amount, payment_file, commit_hash, created_at)) + + return advances + + def _is_debt_fulfilled(debt): return debt.amount_paid != debt.amount @@ -444,13 +462,13 @@ def create_debts(amounts_owed, unpayable_contributors, payment_file): """ Create fresh debts (to unpayable contributors). """ - unpayable_amounts_owed = {email: share - for email, share in amounts_owed.items() + unpayable_amounts_owed = {email: amount + for email, amount in amounts_owed.items() if email in unpayable_contributors} debts = [] commit_hash = get_git_revision_short_hash() - for email, amount_owed in unpayable_amounts_owed.items(): - debt = Debt(email, amount_owed, payment_file=payment_file, commit_hash=commit_hash)) + for email, amount in unpayable_amounts_owed.items(): + debt = Debt(email, amount, payment_file=payment_file, commit_hash=commit_hash)) debts.append(debt) return debts @@ -516,8 +534,14 @@ def distribute_remaining_amount(remaining_amount, truncated_payable_attributions def draw_down_advances(amounts_owed, unpayable_contributors): - payable_amounts_owed = {email: share - for email, share in amounts_owed.items() + """ + Two things: + 1. Draw down the advances (i.e. create negative advances) + 2. Update payable amounts owed, reporting what we are actually + going to pay people at this stage. + """ + payable_amounts_owed = {email: amount + for email, amount in amounts_owed.items() if email not in unpayable_contributors} @@ -534,7 +558,7 @@ def distribute_payment(payment, attributions): # 2. pay them off in chronological order (maybe partially) # 3. (if leftover) identify unpayable people in the relevant attributions file # 4. record debt for each of them according to their attribution - # 5. distribute according to renormalized (remainder) attributions + commit_hash = get_git_revision_short_hash() unpayable_contributors = get_unpayable_contributors() payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) @@ -551,22 +575,40 @@ def distribute_payment(payment, attributions): unpayable_contributors, payment.payment_file) redistribution_pot = sum(d.amount for d in fresh_debts) - # TODO - complete draw_down_advances - it should get the - # advances objects for each person and mutate them, then return - # the advances that have been mutated so they can be recorded - # - also needs to return the sum of amount that was drawn down - # so it can be added to the pot. - also needs to update amounts_owed - # for each person with what remains after advances were drawn down - - # TODO - decide how to represent Advances - draw_down_advances(amounts_owed, unpayable_contributors) - - # put the sum of fresh debts in the pot - # draw down advances using the available amount divided according to - # truncated payable attributions - # put the sum of drawn down advances in the pot - # finally, redistribute the pot over all contributors in truncated payable attributions + # just retain payable people and their amounts owed + amounts_owed = {email: amount + for email, amount in amounts_owed.items() + if email not in unpayable_contributors} + + all_advances = read_advances() + advance_totals = {email: sum(a.amount for a in advances) + for email, advances + in all_advances.items()} + negative_advances = [] + for email, advance_total in advance_totals.items(): + amount_owed = amounts_owed.get(email, 0) + drawdown_amount = max(advance_total - amount_owed, 0) + if drawdown_amount: + negative_advance = Advance(email=email, + amount=-drawdown_amount # note minus sign + payment_file=payment.payment_file + commit_hash=commit_hash) + negative_advances.append(negative_advance) + amounts_owed[email] -= drawdown_amount + + # note that these are drawn down amounts and therefore have negative amounts + # and that's why we take the absolute value here + redistribution_pot += sum(abs(a.amount) for a in negative_advances) + + # TODO: finally, redistribute the pot over all payable contributors + # go through attributions for payable people and redistribute pot accordingly + # create advances for each of those amounts, and add the amount to amounts_owed + # later, generate transactions from the final amounts in amounts_owed + + # this will now need to be updated to reflect the above comment + # will be almost the same as before except that remaining_amount will be + # redistribution_pot instead. equity_transactions = (distribute_remaining_amount(remaining_amount, truncated_payable_attributions, payment) From 986afe2c576267a4ea4d008d80ecdb9d852b4de1 Mon Sep 17 00:00:00 2001 From: apromessi Date: Sun, 14 Jan 2024 17:21:04 -0700 Subject: [PATCH 09/90] notes --- oldabe/models.py | 5 ++++- oldabe/money_in.py | 24 +++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/oldabe/models.py b/oldabe/models.py index 0067f21..59c51df 100644 --- a/oldabe/models.py +++ b/oldabe/models.py @@ -33,11 +33,14 @@ def amount_remaining(self): return self.amount - self.amount_paid +# Individual advances can have a positive or negative amount (to +# indicate an actual advance payment, or a drawn down advance). +# To find the current advance amount for a given contributor, +# sum all of their existing Advance objects. @dataclass class Advance: email: str = None amount: Decimal = 0 - payment_file: str = None commit_hash: str = None created_at: datetime = field(default_factory=datetime.utcnow) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 5e1192b..a1d1d24 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -29,6 +29,7 @@ ACCOUNTING_ZERO = Decimal("0.01") +# TODO - move to utils def parse_percentage(value): """ Translates values expressed in percentage format (75.234) into @@ -46,6 +47,7 @@ def parse_percentage(value): return value +# TODO - move to utils def serialize_proportion(value): """ Translates values expressed in decimal format (0.75234) into @@ -258,6 +260,7 @@ def _get_attributions_total(attributions): return sum(attributions.values()) +# TODO - move to utils def get_rounding_difference(attributions): """ Get the difference of the total of the attributions from 1, which is @@ -277,6 +280,7 @@ def normalize(attributions): attributions[email] *= target_proportion +# TODO - move to utils def correct_rounding_error(attributions, incoming_attribution): """Due to finite precision, the Decimal module will round up or down on the last decimal place. This could result in the aggregate value not @@ -371,6 +375,7 @@ def inflate_valuation(valuation, amount): return valuation + amount +# TODO - move to utils def get_git_revision_short_hash() -> str: """From https://stackoverflow.com/a/21901260""" return ( @@ -413,8 +418,14 @@ def read_advances(): return advances -def _is_debt_fulfilled(debt): - return debt.amount_paid != debt.amount +# get_advance_total_by_contributor +# get_advance_sum_by_contributor/email/user +def get_sum_of_advances_by_contributor(): + all_advances = read_advances() + advance_totals = {email: sum(a.amount for a in advances) + for email, advances + in all_advances.items()} + return advance_totals def get_payable_debts(unpayable_contributors): @@ -581,10 +592,9 @@ def distribute_payment(payment, attributions): for email, amount in amounts_owed.items() if email not in unpayable_contributors} - all_advances = read_advances() - advance_totals = {email: sum(a.amount for a in advances) - for email, advances - in all_advances.items()} + # use the amount owed to each contributor to draw down any advances + # they may already have and then decrement their amount owed accordingly + advance_totals = get_sum_of_advances_by_contributor() negative_advances = [] for email, advance_total in advance_totals.items(): amount_owed = amounts_owed.get(email, 0) @@ -609,7 +619,7 @@ def distribute_payment(payment, attributions): # this will now need to be updated to reflect the above comment # will be almost the same as before except that remaining_amount will be # redistribution_pot instead. - equity_transactions = (distribute_remaining_amount(remaining_amount, + equity_transactions = (distribute_remaining_amount(redistribution_pot, truncated_payable_attributions, payment) if remaining_amount else []) From c47e5447686a197850722cf39fd279c4146d6210 Mon Sep 17 00:00:00 2001 From: apromessi Date: Sun, 14 Jan 2024 17:42:07 -0700 Subject: [PATCH 10/90] implement redistribute_pot --- oldabe/money_in.py | 54 +++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index a1d1d24..fc0085a 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -526,22 +526,23 @@ def get_amounts_owed(total_amount, attributions): for email, share in attributions.items()} -def distribute_remaining_amount(remaining_amount, truncated_payable_attributions, payment): - """ - After paying off debts, distribute remaining amount to payable contributors - in the attributions file, by renormalizing after excluding unpayable - contributors. - """ - amounts_owed = get_amounts_owed(remaining_amount, truncated_payable_attributions) - remainder_attributions = normalize(truncated_payable_attributions) - advances - transactions = generate_transactions( - remaining_amount, remainder_attributions, payment.file, commit_hash +# TODO (turn into doc string): finally, redistribute the pot over all payable contributors +# go through attributions for payable people and redistribute pot accordingly +# create advances for each of those amounts, and add the amount to amounts_owed +def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment_file, amounts_owed): + fresh_advances = [] + normalized_payable_attributions = normalize( + {email: share for email, share in attributions.items() if email not in unpayable_contributors} ) + for email, share in normalized_payable_attributions.items(): + advance_amount = redistribution_pot * share + fresh_advances.append(Advance(email=email, + amount=advance_amount, + payment_file=payment_file, + commit_hash=get_git_revision_short_hash())) + amounts_owed[email] += advance_amount - commit_hash = get_git_revision_short_hash() - - return transactions + return fresh_advances def draw_down_advances(amounts_owed, unpayable_contributors): @@ -601,8 +602,8 @@ def distribute_payment(payment, attributions): drawdown_amount = max(advance_total - amount_owed, 0) if drawdown_amount: negative_advance = Advance(email=email, - amount=-drawdown_amount # note minus sign - payment_file=payment.payment_file + amount=-drawdown_amount, # note minus sign + payment_file=payment.payment_file, commit_hash=commit_hash) negative_advances.append(negative_advance) amounts_owed[email] -= drawdown_amount @@ -611,19 +612,14 @@ def distribute_payment(payment, attributions): # and that's why we take the absolute value here redistribution_pot += sum(abs(a.amount) for a in negative_advances) - # TODO: finally, redistribute the pot over all payable contributors - # go through attributions for payable people and redistribute pot accordingly - # create advances for each of those amounts, and add the amount to amounts_owed - # later, generate transactions from the final amounts in amounts_owed - - # this will now need to be updated to reflect the above comment - # will be almost the same as before except that remaining_amount will be - # redistribution_pot instead. - equity_transactions = (distribute_remaining_amount(redistribution_pot, - truncated_payable_attributions, - payment) - if remaining_amount else []) - + # redistribute the pot over all payable contributors - produce fresh advances and add to amounts owed + fresh_advances = redistribute_pot(redistribution_pot, + attributions, + unpayable_contributors, + payment.payment_file, + amounts_owed) + # TODO - generate transactions from the final amounts in amounts_owed + debts = updated_debts + fresh_debts transactions = equity_transactions + debt_transactions From c4f8364357a905863b42e0a628eb9091bbac22a6 Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 18 Jan 2024 19:21:38 -0700 Subject: [PATCH 11/90] doc string for redistribute_pot --- oldabe/money_in.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index fc0085a..538239b 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -479,7 +479,7 @@ def create_debts(amounts_owed, unpayable_contributors, payment_file): debts = [] commit_hash = get_git_revision_short_hash() for email, amount in unpayable_amounts_owed.items(): - debt = Debt(email, amount, payment_file=payment_file, commit_hash=commit_hash)) + debt = Debt(email, amount, payment_file=payment_file, commit_hash=commit_hash) debts.append(debt) return debts @@ -526,10 +526,11 @@ def get_amounts_owed(total_amount, attributions): for email, share in attributions.items()} -# TODO (turn into doc string): finally, redistribute the pot over all payable contributors -# go through attributions for payable people and redistribute pot accordingly -# create advances for each of those amounts, and add the amount to amounts_owed def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment_file, amounts_owed): + # Redistribute the pot of remaining money over all payable contributors, according to attributions + # share (normalized to 100%). Create advances for those amounts (because they are in excess + # of the amount owed to each contributor from the original payment) and add the amounts to the + # amounts_owed dictionary to keep track of the full amount we are about to pay everyone. fresh_advances = [] normalized_payable_attributions = normalize( {email: share for email, share in attributions.items() if email not in unpayable_contributors} From 28a257b66abf5fbfab5bba3b793c0f06887366d4 Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 18 Jan 2024 19:26:24 -0700 Subject: [PATCH 12/90] delete alternate function names --- oldabe/money_in.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 538239b..6d5c511 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -418,8 +418,6 @@ def read_advances(): return advances -# get_advance_total_by_contributor -# get_advance_sum_by_contributor/email/user def get_sum_of_advances_by_contributor(): all_advances = read_advances() advance_totals = {email: sum(a.amount for a in advances) From f5516b116f3532f2bfd996bda722bf78d282d531 Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 25 Jan 2024 20:30:51 -0700 Subject: [PATCH 13/90] update docstrings --- oldabe/money_in.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 6d5c511..a318663 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -419,6 +419,12 @@ def read_advances(): def get_sum_of_advances_by_contributor(): + """ + Sum all Advance objects for each contributor to get the total amount + that they currently have in advances and have not yet drawn down. + Return a dictionary with the contributor's email as the key and the + their advance amount as the value. + """ all_advances = read_advances() advance_totals = {email: sum(a.amount for a in advances) for email, advances @@ -525,10 +531,12 @@ def get_amounts_owed(total_amount, attributions): def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment_file, amounts_owed): - # Redistribute the pot of remaining money over all payable contributors, according to attributions - # share (normalized to 100%). Create advances for those amounts (because they are in excess - # of the amount owed to each contributor from the original payment) and add the amounts to the - # amounts_owed dictionary to keep track of the full amount we are about to pay everyone. + """ + Redistribute the pot of remaining money over all payable contributors, according to attributions + share (normalized to 100%). Create advances for those amounts (because they are in excess + of the amount owed to each contributor from the original payment) and add the amounts to the + amounts_owed dictionary to keep track of the full amount we are about to pay everyone. + """ fresh_advances = [] normalized_payable_attributions = normalize( {email: share for email, share in attributions.items() if email not in unpayable_contributors} From 21a066e9c8700d49ba9df0ee152f6b1a746203ec Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 25 Jan 2024 21:02:34 -0700 Subject: [PATCH 14/90] update naming to split amounts_owed into amounts_payable and amounts_unpayable, delete unused function for drawing down advances --- oldabe/money_in.py | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index a318663..0a6935a 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -477,12 +477,12 @@ def create_debts(amounts_owed, unpayable_contributors, payment_file): """ Create fresh debts (to unpayable contributors). """ - unpayable_amounts_owed = {email: amount - for email, amount in amounts_owed.items() - if email in unpayable_contributors} + amounts_unpayable = {email: amount + for email, amount in amounts_owed.items() + if email in unpayable_contributors} debts = [] commit_hash = get_git_revision_short_hash() - for email, amount in unpayable_amounts_owed.items(): + for email, amount in amounts_unpayable.items(): debt = Debt(email, amount, payment_file=payment_file, commit_hash=commit_hash) debts.append(debt) @@ -530,12 +530,12 @@ def get_amounts_owed(total_amount, attributions): for email, share in attributions.items()} -def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment_file, amounts_owed): +def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment_file, amounts_payable): """ Redistribute the pot of remaining money over all payable contributors, according to attributions share (normalized to 100%). Create advances for those amounts (because they are in excess of the amount owed to each contributor from the original payment) and add the amounts to the - amounts_owed dictionary to keep track of the full amount we are about to pay everyone. + amounts_payable dictionary to keep track of the full amount we are about to pay everyone. """ fresh_advances = [] normalized_payable_attributions = normalize( @@ -547,23 +547,11 @@ def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, p amount=advance_amount, payment_file=payment_file, commit_hash=get_git_revision_short_hash())) - amounts_owed[email] += advance_amount + amounts_payable[email] += advance_amount return fresh_advances -def draw_down_advances(amounts_owed, unpayable_contributors): - """ - Two things: - 1. Draw down the advances (i.e. create negative advances) - 2. Update payable amounts owed, reporting what we are actually - going to pay people at this stage. - """ - payable_amounts_owed = {email: amount - for email, amount in amounts_owed.items() - if email not in unpayable_contributors} - - def distribute_payment(payment, attributions): """ Generate transactions to contributors from a (new) payment. @@ -587,33 +575,31 @@ def distribute_payment(payment, attributions): fresh_debts = [] equity_transactions = [] if available_amount > ACCOUNTING_ZERO: - amounts_owed = {email: share * available_amount - for email, share - in attributions.items()} + amounts_owed = get_amounts_owed(available_amount, attributions) fresh_debts = create_debts(amounts_owed, unpayable_contributors, payment.payment_file) redistribution_pot = sum(d.amount for d in fresh_debts) # just retain payable people and their amounts owed - amounts_owed = {email: amount - for email, amount in amounts_owed.items() - if email not in unpayable_contributors} + amounts_payable = {email: amount + for email, amount in amounts_owed.items() + if email not in unpayable_contributors} # use the amount owed to each contributor to draw down any advances # they may already have and then decrement their amount owed accordingly advance_totals = get_sum_of_advances_by_contributor() negative_advances = [] for email, advance_total in advance_totals.items(): - amount_owed = amounts_owed.get(email, 0) - drawdown_amount = max(advance_total - amount_owed, 0) + amount_payable = amounts_payable.get(email, 0) + drawdown_amount = max(advance_total - amount_payable, 0) if drawdown_amount: negative_advance = Advance(email=email, amount=-drawdown_amount, # note minus sign payment_file=payment.payment_file, commit_hash=commit_hash) negative_advances.append(negative_advance) - amounts_owed[email] -= drawdown_amount + amounts_payable[email] -= drawdown_amount # note that these are drawn down amounts and therefore have negative amounts # and that's why we take the absolute value here @@ -624,8 +610,8 @@ def distribute_payment(payment, attributions): attributions, unpayable_contributors, payment.payment_file, - amounts_owed) - # TODO - generate transactions from the final amounts in amounts_owed + amounts_payable) + # TODO - generate transactions from the final amounts in amounts_payable debts = updated_debts + fresh_debts transactions = equity_transactions + debt_transactions From 167241ff6e66df3376e64163965e9c9bf6db93e5 Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 29 Feb 2024 20:34:22 -0800 Subject: [PATCH 15/90] create new equity transactions from running tallies in amounts_payable --- oldabe/money_in.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 0a6935a..4b5f2f0 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -605,13 +605,19 @@ def distribute_payment(payment, attributions): # and that's why we take the absolute value here redistribution_pot += sum(abs(a.amount) for a in negative_advances) - # redistribute the pot over all payable contributors - produce fresh advances and add to amounts owed + # redistribute the pot over all payable contributors - produce fresh advances and add to amounts payable fresh_advances = redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment.payment_file, amounts_payable) - # TODO - generate transactions from the final amounts in amounts_payable + + for email, amount in amounts_payable.items(): + new_equity_transaction = Transaction(email=email, + amount=amount, + payment_file=payment.payment_file, + commit_hash=commit_hash) + equity_transactions.append(new_equity_transaction) debts = updated_debts + fresh_debts transactions = equity_transactions + debt_transactions From 74018f058c94446db0b4c3d1fe09ac9b790d89f8 Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 29 Feb 2024 20:41:54 -0800 Subject: [PATCH 16/90] return newly created advances from distribute payment function, add note for next steps --- oldabe/money_in.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 4b5f2f0..146796f 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -574,6 +574,8 @@ def distribute_payment(payment, attributions): fresh_debts = [] equity_transactions = [] + negative_advances = [] + fresh_advances = [] if available_amount > ACCOUNTING_ZERO: amounts_owed = get_amounts_owed(available_amount, attributions) fresh_debts = create_debts(amounts_owed, @@ -589,7 +591,6 @@ def distribute_payment(payment, attributions): # use the amount owed to each contributor to draw down any advances # they may already have and then decrement their amount owed accordingly advance_totals = get_sum_of_advances_by_contributor() - negative_advances = [] for email, advance_total in advance_totals.items(): amount_payable = amounts_payable.get(email, 0) drawdown_amount = max(advance_total - amount_payable, 0) @@ -621,8 +622,9 @@ def distribute_payment(payment, attributions): debts = updated_debts + fresh_debts transactions = equity_transactions + debt_transactions + advances = negative_advances + fresh_advances - return debts, transactions + return debts, transactions, advances def handle_investment( @@ -669,6 +671,11 @@ def _create_itemized_payment(payment, fee_amount): ) +# TODO - next step - see how the updates we've made for debts/advances +# change things in the following process_payments function (return values +# for distribute_payment, calculating amount_paid_out, make sure to +# return any new advances so they can be written in the final step) +# TODO - create new function for writing newly created advances to the file def process_payments(instruments, attributions): """ Process new payments by paying out instruments and then, from the amount @@ -683,7 +690,7 @@ def process_payments(instruments, attributions): unprocessed_payments = _get_unprocessed_payments() for payment in unprocessed_payments: # first, process instruments (i.e. pay fees) - debts, transactions = distribute_payment(payment, instruments) + debts, transactions, advances = distribute_payment(payment, instruments) new_transactions += transactions # TODO - may need to calculate this differently with debts in the mix amount_paid_out = sum(t.amount for t in transactions) From 09105ead99bb33131d62bc389d46fcb97561ce95 Mon Sep 17 00:00:00 2001 From: apromessi Date: Thu, 29 Feb 2024 20:44:28 -0800 Subject: [PATCH 17/90] add note upon a note --- oldabe/money_in.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 146796f..b36a983 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -321,7 +321,7 @@ def write_append_itemized_payments(itemized_payments): for row in itemized_payments: writer.writerow(astuple(row)) - +# [[ OLD NOTE - still relevant?? ]] # TODO - when we write debts # 1) read all from the file and transform into a hash, where the key is a # unique identifier constructed from the email + payment file and the value From aefa6d5369e300844e0488b3e31d8fd8c697c338 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 20:42:34 -0700 Subject: [PATCH 18/90] Address todos Ensure advances are returned from core functions and handled in the dispatching functions. Write advances at the end, like we do all the other objects. --- oldabe/money_in.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index b36a983..7f553db 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -321,6 +321,15 @@ def write_append_itemized_payments(itemized_payments): for row in itemized_payments: writer.writerow(astuple(row)) + +def write_append_advances(advances): + advances_file = os.path.join(ABE_ROOT, ADVANCES_FILE) + with open(advances_file, 'a') as f: + writer = csv.writer(f) + for row in advances: + writer.writerow(astuple(row)) + + # [[ OLD NOTE - still relevant?? ]] # TODO - when we write debts # 1) read all from the file and transform into a hash, where the key is a @@ -619,7 +628,7 @@ def distribute_payment(payment, attributions): payment_file=payment.payment_file, commit_hash=commit_hash) equity_transactions.append(new_equity_transaction) - + debts = updated_debts + fresh_debts transactions = equity_transactions + debt_transactions advances = negative_advances + fresh_advances @@ -665,17 +674,12 @@ def _create_itemized_payment(payment, fee_amount): return ItemizedPayment( payment.email, fee_amount, - payment.amount, + payment.amount, # already has fees deducted payment.attributable, payment.file, ) -# TODO - next step - see how the updates we've made for debts/advances -# change things in the following process_payments function (return values -# for distribute_payment, calculating amount_paid_out, make sure to -# return any new advances so they can be written in the final step) -# TODO - create new function for writing newly created advances to the file def process_payments(instruments, attributions): """ Process new payments by paying out instruments and then, from the amount @@ -685,6 +689,8 @@ def process_payments(instruments, attributions): """ price = read_price() valuation = read_valuation() + new_debts = [] + new_advances = [] new_transactions = [] new_itemized_payments = [] unprocessed_payments = _get_unprocessed_payments() @@ -692,23 +698,27 @@ def process_payments(instruments, attributions): # first, process instruments (i.e. pay fees) debts, transactions, advances = distribute_payment(payment, instruments) new_transactions += transactions - # TODO - may need to calculate this differently with debts in the mix - amount_paid_out = sum(t.amount for t in transactions) + new_debts += debts + new_advances += advances + fees_paid_out = sum(t.amount for t in transactions) # deduct the amount paid out to instruments before # processing it for attributions - payment.amount -= amount_paid_out + payment.amount -= fees_paid_out new_itemized_payments.append( - _create_itemized_payment(payment, amount_paid_out) + _create_itemized_payment(payment, fees_paid_out) ) # next, process attributions - using the amount owed to the project # (which is the amount leftover after paying instruments/fees) if payment.amount > ACCOUNTING_ZERO: - new_transactions += distribute_payment(payment, attributions) + debts, transactions, advances = distribute_payment(payment, attributions) + new_transactions += transactions + new_debts += debts + new_advances += advances if payment.attributable: valuation = handle_investment( payment, new_itemized_payments, attributions, price, valuation ) - return debts, new_transactions, valuation, new_itemized_payments + return new_debts, new_transactions, valuation, new_itemized_payments, new_advances def process_payments_and_record_updates(): @@ -725,6 +735,7 @@ def process_payments_and_record_updates(): transactions, posterior_valuation, new_itemized_payments, + advances, ) = process_payments(instruments, attributions) # we only write the changes to disk at the end @@ -735,6 +746,7 @@ def process_payments_and_record_updates(): write_attributions(attributions) write_valuation(posterior_valuation) write_append_itemized_payments(new_itemized_payments) + write_append_advances(advances) def main(): From 9c036386166e1306f51210d2e099de0255992a3c Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:06:45 -0700 Subject: [PATCH 19/90] Record any changes to advances after execution --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index a10f6d6..2b89b71 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -26,7 +26,7 @@ echo "... done." echo "Committing updated transactions and attributions back to repo..." git config --global user.email "abe@drym.org" git config --global user.name "Old Abe" -git add abe/transactions.txt abe/attributions.txt abe/valuation.txt abe/itemized_payments.txt +git add abe/transactions.txt abe/attributions.txt abe/valuation.txt abe/itemized_payments.txt abe/advances.txt set +e From 29d77047e190492bf9a22d0d16482f916ed80d35 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:11:47 -0700 Subject: [PATCH 20/90] debugging.. --- oldabe/money_in.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 7f553db..7bd1e3c 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -574,8 +574,10 @@ def distribute_payment(payment, attributions): # 2. pay them off in chronological order (maybe partially) # 3. (if leftover) identify unpayable people in the relevant attributions file # 4. record debt for each of them according to their attribution + print("Listing directory files...") + print(os.listdir(ABE_ROOT)) commit_hash = get_git_revision_short_hash() - unpayable_contributors = get_unpayable_contributors() + unpayable_contributors = [] # get_unpayable_contributors() payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) # The "available" amount is what is left over after paying off debts From 4db66a20cd89fd7a71a7f1071c191f5f6bf4abb0 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:25:06 -0700 Subject: [PATCH 21/90] remove the override that we had for debugging for some reason --- oldabe/money_in.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 7bd1e3c..ebebc47 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -761,5 +761,4 @@ def main(): if __name__ == "__main__": - # main() - print(get_unpayable_contributors()) + main() From 7074aae2ce9a1a61dc51c7034179d699bc31c2af Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:29:55 -0700 Subject: [PATCH 22/90] Use abe folder as ABE_ROOT --- oldabe/money_in.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index ebebc47..32d1567 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -9,8 +9,8 @@ import subprocess from .models import Attribution, Payment, ItemizedPayment, Transaction -# ABE_ROOT = 'abe' -ABE_ROOT = '.' +ABE_ROOT = './abe' +# ABE_ROOT = '.' PAYMENTS_DIR = os.path.join(ABE_ROOT, 'payments') NONATTRIBUTABLE_PAYMENTS_DIR = os.path.join( ABE_ROOT, 'payments', 'nonattributable' From b16e1699fc788f204f7a1fbfbc56fe6ee4e13fff Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:35:55 -0700 Subject: [PATCH 23/90] use correct field name --- oldabe/money_in.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 32d1567..ddbe4ea 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -591,7 +591,7 @@ def distribute_payment(payment, attributions): amounts_owed = get_amounts_owed(available_amount, attributions) fresh_debts = create_debts(amounts_owed, unpayable_contributors, - payment.payment_file) + payment.file) redistribution_pot = sum(d.amount for d in fresh_debts) # just retain payable people and their amounts owed @@ -608,7 +608,7 @@ def distribute_payment(payment, attributions): if drawdown_amount: negative_advance = Advance(email=email, amount=-drawdown_amount, # note minus sign - payment_file=payment.payment_file, + payment_file=payment.file, commit_hash=commit_hash) negative_advances.append(negative_advance) amounts_payable[email] -= drawdown_amount @@ -621,13 +621,13 @@ def distribute_payment(payment, attributions): fresh_advances = redistribute_pot(redistribution_pot, attributions, unpayable_contributors, - payment.payment_file, + payment.file, amounts_payable) for email, amount in amounts_payable.items(): new_equity_transaction = Transaction(email=email, amount=amount, - payment_file=payment.payment_file, + payment_file=payment.file, commit_hash=commit_hash) equity_transactions.append(new_equity_transaction) From d2369db138f0bf68441e2d526c1556fefc1fd8a9 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:40:04 -0700 Subject: [PATCH 24/90] mutate dict defined in calling context --- oldabe/money_in.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index ddbe4ea..540473a 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -547,10 +547,9 @@ def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, p amounts_payable dictionary to keep track of the full amount we are about to pay everyone. """ fresh_advances = [] - normalized_payable_attributions = normalize( - {email: share for email, share in attributions.items() if email not in unpayable_contributors} - ) - for email, share in normalized_payable_attributions.items(): + payable_attributions = {email: share for email, share in attributions.items() if email not in unpayable_contributors} + normalize(payable_attributions) + for email, share in payable_attributions.items(): advance_amount = redistribution_pot * share fresh_advances.append(Advance(email=email, amount=advance_amount, From e86a993d5798655bac75aef71a71dbcea9ed37b3 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:40:19 -0700 Subject: [PATCH 25/90] import Advance model --- oldabe/money_in.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 540473a..3363e82 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -7,7 +7,7 @@ import re import os import subprocess -from .models import Attribution, Payment, ItemizedPayment, Transaction +from .models import Advance, Attribution, Payment, ItemizedPayment, Transaction ABE_ROOT = './abe' # ABE_ROOT = '.' From 4d73b513d898964e4d6a3c433dbff7a862bd9c0d Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 7 Mar 2024 21:43:39 -0700 Subject: [PATCH 26/90] actually open file to be written --- oldabe/money_in.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 3363e82..e6a6835 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -511,18 +511,19 @@ def write_debts(processed_debts): existing_debts = read_debts() processed_debts_hash = {debt.key(): debt for debt in processed_debts} debts_file = os.path.join(ABE_ROOT, DEBTS_FILE) - writer = csv.writer(f) with open(debts_file, 'w') as f: - for existing_debt in existing_debts: - # if the existing debt has been processed, write the processed version - # otherwise re-write the existing version - if processed_debt := processed_debts_hash.get(existing_debt.key()): - writer.writerow(astuple(processed_debt)) - del processed_debts_hash[processed_debt.key()] - else: - writer.writerow(astuple(existing_debt)) - for debt in processed_debts_hash.values(): - writer.writerow(astuple(debt)) + writer = csv.writer(f) + with open(debts_file, 'w') as f: + for existing_debt in existing_debts: + # if the existing debt has been processed, write the processed version + # otherwise re-write the existing version + if processed_debt := processed_debts_hash.get(existing_debt.key()): + writer.writerow(astuple(processed_debt)) + del processed_debts_hash[processed_debt.key()] + else: + writer.writerow(astuple(existing_debt)) + for debt in processed_debts_hash.values(): + writer.writerow(astuple(debt)) def renormalize(attributions, excluded_contributors): From 4c07d8c730634b7a25df39a6e611defa83da4d0a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 19:29:39 -0700 Subject: [PATCH 27/90] Actually read unpayable contributors file --- oldabe/money_in.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index e6a6835..152afdb 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -472,6 +472,10 @@ def pay_debts(payable_debts, payment): def get_unpayable_contributors(): + """ + Read the unpayable_contributors file to get the list of contributors who + are unpayable. + """ unpayable_contributors_file = os.path.join(ABE_ROOT, UNPAYABLE_CONTRIBUTORS_FILE) contributors = [] with open(unpayable_contributors_file) as f: @@ -577,7 +581,7 @@ def distribute_payment(payment, attributions): print("Listing directory files...") print(os.listdir(ABE_ROOT)) commit_hash = get_git_revision_short_hash() - unpayable_contributors = [] # get_unpayable_contributors() + unpayable_contributors = get_unpayable_contributors() payable_debts = get_payable_debts(unpayable_contributors) updated_debts, debt_transactions = pay_debts(payable_debts, payment) # The "available" amount is what is left over after paying off debts From b109d29b9e9799444dc0d25530a7ef32db9d7d62 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 19:57:20 -0700 Subject: [PATCH 28/90] Also commit updated debts after running job --- entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 2b89b71..444bc31 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -23,14 +23,14 @@ echo "... done." # Note that running this locally would cause your global # git config to be modified -echo "Committing updated transactions and attributions back to repo..." +echo "Committing updated accounting records back to repo..." git config --global user.email "abe@drym.org" git config --global user.name "Old Abe" -git add abe/transactions.txt abe/attributions.txt abe/valuation.txt abe/itemized_payments.txt abe/advances.txt +git add abe/transactions.txt abe/attributions.txt abe/valuation.txt abe/itemized_payments.txt abe/advances.txt abe/debts.txt set +e -git commit -m "Updated transactions and attributions" +git commit -m "Updated accounting records" git fetch git rebase origin/`git remote set-head origin -a | cut -d' ' -f4` git push origin `git remote set-head origin -a | cut -d' ' -f4` From 091eca4a035779065693d3a37615fdcb42ad6c11 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 19:57:45 -0700 Subject: [PATCH 29/90] Strategically re-enable error forwarding to fail on python errors Looks like we had disabled these for testing some time ago, so, reenabling them now with some additional useful options. --- entrypoint.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 444bc31..f673c63 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,7 @@ # ensure that any errors encountered cause immediate # termination with a non-zero exit code -set -e +set -euo pipefail echo "PWD is: " echo $(pwd) @@ -28,16 +28,12 @@ git config --global user.email "abe@drym.org" git config --global user.name "Old Abe" git add abe/transactions.txt abe/attributions.txt abe/valuation.txt abe/itemized_payments.txt abe/advances.txt abe/debts.txt -set +e - git commit -m "Updated accounting records" git fetch git rebase origin/`git remote set-head origin -a | cut -d' ' -f4` git push origin `git remote set-head origin -a | cut -d' ' -f4` echo "... done." -set -e - echo "Running money_out script..." echo balances=$(python -m oldabe.money_out) >> $GITHUB_OUTPUT echo "... done." From 4cac2d3035f841da7840255114eec6dfa7f4aeea Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:04:26 -0700 Subject: [PATCH 30/90] Use bash instead of sh --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index f673c63..dae21b5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/sh -l +#!/bin/bash # ensure that any errors encountered cause immediate # termination with a non-zero exit code From 8c10dedb3cda94b5d77cf26305f15fcecee43703 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:08:04 -0700 Subject: [PATCH 31/90] try with quotes --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index dae21b5..54be575 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,7 +11,7 @@ echo "Running ls..." echo $(ls /) echo "... done." -export PYTHONPATH=/:$PYTHONPATH +export PYTHONPATH="/:$PYTHONPATH" echo "Running ls /github/workspace..." echo $(ls /github/workspace) From 12c70b942bfd450a5b3cddb095798cb285e2599d Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:10:13 -0700 Subject: [PATCH 32/90] remove -u option to avoid unbound PYTHONPATH --- entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 54be575..3765867 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,7 @@ # ensure that any errors encountered cause immediate # termination with a non-zero exit code -set -euo pipefail +set -eo pipefail echo "PWD is: " echo $(pwd) @@ -11,7 +11,7 @@ echo "Running ls..." echo $(ls /) echo "... done." -export PYTHONPATH="/:$PYTHONPATH" +export PYTHONPATH=/:$PYTHONPATH echo "Running ls /github/workspace..." echo $(ls /github/workspace) From 109555c3f44a7f4c4fb333a781cd7202520851d3 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:15:15 -0700 Subject: [PATCH 33/90] import missing Debt object --- oldabe/money_in.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 152afdb..114647b 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -7,7 +7,9 @@ import re import os import subprocess -from .models import Advance, Attribution, Payment, ItemizedPayment, Transaction +from .models import ( + Advance, Attribution, Debt, Payment, ItemizedPayment, Transaction +) ABE_ROOT = './abe' # ABE_ROOT = '.' From dbf2c0d7731786a3e8d0d6e387bc755d9ebb274a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:27:27 -0700 Subject: [PATCH 34/90] don't create advances that are effectively for zero amounts --- oldabe/money_in.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 114647b..7d26c37 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -611,7 +611,7 @@ def distribute_payment(payment, attributions): for email, advance_total in advance_totals.items(): amount_payable = amounts_payable.get(email, 0) drawdown_amount = max(advance_total - amount_payable, 0) - if drawdown_amount: + if drawdown_amount > ACCOUNTING_ZERO: negative_advance = Advance(email=email, amount=-drawdown_amount, # note minus sign payment_file=payment.file, From b978b080cbdfebe40e0c373a0fd939fb615bc34e Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:34:39 -0700 Subject: [PATCH 35/90] handle missing debts file --- oldabe/money_in.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 7d26c37..0193d0c 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -399,16 +399,19 @@ def get_git_revision_short_hash() -> str: def read_debts(): debts_file = os.path.join(ABE_ROOT, DEBTS_FILE) debts = [] - with open(debts_file) as f: - for ( - email, - amount, - amount_paid, - payment_file, - commit_hash, - created_at, - ) in csv.reader(f): - debts.append(Debt(email, amount, amount_paid, payment_file, commit_hash, created_at)) + try: + with open(debts_file) as f: + for ( + email, + amount, + amount_paid, + payment_file, + commit_hash, + created_at, + ) in csv.reader(f): + debts.append(Debt(email, amount, amount_paid, payment_file, commit_hash, created_at)) + except FileNotFoundError: + pass return debts From 61beb81fde5f3889c8423c43848615bb90177d39 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:36:32 -0700 Subject: [PATCH 36/90] handle missing advances file --- oldabe/money_in.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 0193d0c..5843b02 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -419,15 +419,18 @@ def read_debts(): def read_advances(): advances_file = os.path.join(ABE_ROOT, ADVANCES_FILE) advances = defaultdict(list) - with open(advances_file) as f: - for ( - email, - amount, - payment_file, - commit_hash, - created_at, - ) in csv.reader(f): - advances[email].append(Advance(email, amount, payment_file, commit_hash, created_at)) + try: + with open(advances_file) as f: + for ( + email, + amount, + payment_file, + commit_hash, + created_at, + ) in csv.reader(f): + advances[email].append(Advance(email, amount, payment_file, commit_hash, created_at)) + except FileNotFoundError: + pass return advances From 6c981ae9ca57c9a2bb3732458018ee2ea1f3f409 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 14 Mar 2024 20:48:53 -0700 Subject: [PATCH 37/90] don't redistribute pot if there's nothing in there --- oldabe/money_in.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 5843b02..eed359b 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -630,11 +630,12 @@ def distribute_payment(payment, attributions): redistribution_pot += sum(abs(a.amount) for a in negative_advances) # redistribute the pot over all payable contributors - produce fresh advances and add to amounts payable - fresh_advances = redistribute_pot(redistribution_pot, - attributions, - unpayable_contributors, - payment.file, - amounts_payable) + if redistribution_pot > ACCOUNTING_ZERO: + fresh_advances = redistribute_pot(redistribution_pot, + attributions, + unpayable_contributors, + payment.file, + amounts_payable) for email, amount in amounts_payable.items(): new_equity_transaction = Transaction(email=email, From 125ce555426163d55300e97a6b5006bc78c774bf Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Mar 2024 20:13:05 -0700 Subject: [PATCH 38/90] Add aggregate advances to outstanding balances issue for testing --- oldabe/money_out.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/oldabe/money_out.py b/oldabe/money_out.py index 0109395..41c7406 100755 --- a/oldabe/money_out.py +++ b/oldabe/money_out.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from .models import Transaction, Debt +from .models import Transaction, Debt, Advance from datetime import datetime import csv import os @@ -12,6 +12,7 @@ PAYOUTS_DIR = os.path.join(ABE_ROOT, 'payouts') TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt') DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt') +ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt') def read_transaction_amounts(): @@ -105,13 +106,47 @@ def prepare_debts_message(outstanding_debts: dict): return "\r\n".join(line.strip() for line in message.split('\n')).strip() -def combined_message(balances_message, debts_message): +def read_advance_amounts(): + advances = defaultdict(int) + with open(ADVANCES_FILE) as f: + for row in csv.reader(f): + a = Advance(*row) + a.amount = Decimal(a.amount) + advances[a.email] += a.amount + return advances + + +def prepare_advances_message(advances: dict): + """ A temporary message reporting aggregate advances, for testing purposes. + """ + if not advances: + return "There are no advances." + advances_table = "" + for name, advance in advances.items(): + advances_table += f"{name} | {advance:.2f}\n" + message = f""" + The current advances are: + + | Name | Advance | + | ---- | ------- | + {advances_table} + + **Total** = {sum(advances.values()):.2f} + """ + return "\r\n".join(line.strip() for line in message.split('\n')).strip() + + +def combined_message(balances_message, debts_message, advances_message): message = f""" {balances_message} ---------------------------- {debts_message} + + ---------------------------- + + {advances_message} """ return "\r\n".join(line.strip() for line in message.split('\n')).strip() @@ -129,7 +164,9 @@ def main(): balances_message = prepare_balances_message(balances) outstanding_debts = read_outstanding_debt_amounts() debts_message = prepare_debts_message(outstanding_debts) - print(combined_message(balances_message, debts_message)) + advances = read_advance_amounts() + advances_message = prepare_advances_message(advances) + print(combined_message(balances_message, debts_message, advances_message)) if __name__ == "__main__": From 66a09527a1a48475adfee5a4791c10bef17c51c7 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Mar 2024 20:23:42 -0700 Subject: [PATCH 39/90] parse amounts as `Decimal` --- oldabe/money_in.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index eed359b..e630d44 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -409,7 +409,7 @@ def read_debts(): commit_hash, created_at, ) in csv.reader(f): - debts.append(Debt(email, amount, amount_paid, payment_file, commit_hash, created_at)) + debts.append(Debt(email, Decimal(amount), Decimal(amount_paid), payment_file, commit_hash, created_at)) except FileNotFoundError: pass @@ -428,7 +428,7 @@ def read_advances(): commit_hash, created_at, ) in csv.reader(f): - advances[email].append(Advance(email, amount, payment_file, commit_hash, created_at)) + advances[email].append(Advance(email, Decimal(amount), payment_file, commit_hash, created_at)) except FileNotFoundError: pass From c5087c979f2c6b0d64d67b438ea6c35b4eaad010 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Mar 2024 20:24:10 -0700 Subject: [PATCH 40/90] add a comment for future refactoring --- oldabe/money_in.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index e630d44..6cf65f5 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -387,6 +387,8 @@ def inflate_valuation(valuation, amount): # TODO - move to utils +# and standardize the parsing from text into python objects +# e.g. Decimal and DateTime def get_git_revision_short_hash() -> str: """From https://stackoverflow.com/a/21901260""" return ( From aa6f4b32cc5ff8bfb855d0d5c630352e4392ef9b Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Mar 2024 20:38:36 -0700 Subject: [PATCH 41/90] only draw down advances for contributors in the relevant attributions file --- oldabe/money_in.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 6cf65f5..3668732 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -418,7 +418,7 @@ def read_debts(): return debts -def read_advances(): +def read_advances(attributions): advances_file = os.path.join(ABE_ROOT, ADVANCES_FILE) advances = defaultdict(list) try: @@ -430,21 +430,22 @@ def read_advances(): commit_hash, created_at, ) in csv.reader(f): - advances[email].append(Advance(email, Decimal(amount), payment_file, commit_hash, created_at)) + if email in attributions: + advances[email].append(Advance(email, Decimal(amount), payment_file, commit_hash, created_at)) except FileNotFoundError: pass return advances -def get_sum_of_advances_by_contributor(): +def get_sum_of_advances_by_contributor(attributions): """ Sum all Advance objects for each contributor to get the total amount that they currently have in advances and have not yet drawn down. Return a dictionary with the contributor's email as the key and the their advance amount as the value. """ - all_advances = read_advances() + all_advances = read_advances(attributions) advance_totals = {email: sum(a.amount for a in advances) for email, advances in all_advances.items()} @@ -614,8 +615,8 @@ def distribute_payment(payment, attributions): if email not in unpayable_contributors} # use the amount owed to each contributor to draw down any advances - # they may already have and then decrement their amount owed accordingly - advance_totals = get_sum_of_advances_by_contributor() + # they may already have and then decrement their amount payable accordingly + advance_totals = get_sum_of_advances_by_contributor(attributions) for email, advance_total in advance_totals.items(): amount_payable = amounts_payable.get(email, 0) drawdown_amount = max(advance_total - amount_payable, 0) From 71cf2039e2272e849d8b00792f515527055e0659 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Mar 2024 20:44:37 -0700 Subject: [PATCH 42/90] parse ItemizedPayment amounts as Decimal --- oldabe/money_in.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 3668732..511ae43 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -202,8 +202,8 @@ def get_existing_itemized_payments(): ) in csv.reader(f): itemized_payment = ItemizedPayment( email, - fee_amount, - project_amount, + Decimal(fee_amount), + Decimal(project_amount), attributable, payment_file, ) From 0715c370836f8dc023035568b6c29e3be4bcd6e3 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 21 Mar 2024 21:02:33 -0700 Subject: [PATCH 43/90] support partial drawdown of advances --- oldabe/money_in.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 511ae43..eeae286 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -619,7 +619,7 @@ def distribute_payment(payment, attributions): advance_totals = get_sum_of_advances_by_contributor(attributions) for email, advance_total in advance_totals.items(): amount_payable = amounts_payable.get(email, 0) - drawdown_amount = max(advance_total - amount_payable, 0) + drawdown_amount = min(advance_total, amount_payable) if drawdown_amount > ACCOUNTING_ZERO: negative_advance = Advance(email=email, amount=-drawdown_amount, # note minus sign From d424cb96ab11a68c2b89b453923344d61200cf2f Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Fri, 29 Mar 2024 18:43:18 -0700 Subject: [PATCH 44/90] don't pay debts unless contributor is in the relevant attributions --- oldabe/money_in.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index eeae286..4b91430 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -452,10 +452,11 @@ def get_sum_of_advances_by_contributor(attributions): return advance_totals -def get_payable_debts(unpayable_contributors): +def get_payable_debts(unpayable_contributors, attributions): debts = read_debts() debts = [d for d in debts if not d.is_fulfilled() + and d.email in attributions and d.email not in unpayable_contributors] return debts @@ -593,7 +594,7 @@ def distribute_payment(payment, attributions): print(os.listdir(ABE_ROOT)) commit_hash = get_git_revision_short_hash() unpayable_contributors = get_unpayable_contributors() - payable_debts = get_payable_debts(unpayable_contributors) + payable_debts = get_payable_debts(unpayable_contributors, attributions) updated_debts, debt_transactions = pay_debts(payable_debts, payment) # The "available" amount is what is left over after paying off debts available_amount = payment.amount - sum(t.amount for t in debt_transactions) From 32009ef9fe9ae4e818fcc17627f2ae190c37ead4 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Fri, 29 Mar 2024 19:09:08 -0700 Subject: [PATCH 45/90] print statements for debugging --- oldabe/money_in.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 4b91430..21dd822 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -603,8 +603,10 @@ def distribute_payment(payment, attributions): equity_transactions = [] negative_advances = [] fresh_advances = [] + print(f"available amount is {available_amount}") if available_amount > ACCOUNTING_ZERO: amounts_owed = get_amounts_owed(available_amount, attributions) + print(f"total of amounts owed is {sum(v for v in amounts_owed.values())}") fresh_debts = create_debts(amounts_owed, unpayable_contributors, payment.file) @@ -614,10 +616,12 @@ def distribute_payment(payment, attributions): amounts_payable = {email: amount for email, amount in amounts_owed.items() if email not in unpayable_contributors} + print(f"total of amounts payable is {sum(v for v in amounts_payable.values())}") # use the amount owed to each contributor to draw down any advances # they may already have and then decrement their amount payable accordingly advance_totals = get_sum_of_advances_by_contributor(attributions) + print(f"advance totals is {advance_totals}") for email, advance_total in advance_totals.items(): amount_payable = amounts_payable.get(email, 0) drawdown_amount = min(advance_total, amount_payable) @@ -632,6 +636,7 @@ def distribute_payment(payment, attributions): # note that these are drawn down amounts and therefore have negative amounts # and that's why we take the absolute value here redistribution_pot += sum(abs(a.amount) for a in negative_advances) + print(f"redistribution_pot is {redistribution_pot}") # redistribute the pot over all payable contributors - produce fresh advances and add to amounts payable if redistribution_pot > ACCOUNTING_ZERO: @@ -648,6 +653,8 @@ def distribute_payment(payment, attributions): commit_hash=commit_hash) equity_transactions.append(new_equity_transaction) + print(f"sum of equity transactions is {sum(t.amount for t in equity_transactions)}") + print(f"sum of debt transactions is {sum(t.amount for t in debt_transactions)}") debts = updated_debts + fresh_debts transactions = equity_transactions + debt_transactions advances = negative_advances + fresh_advances From 53f31d3fd4009a2475270833c472ce9071144bb3 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Fri, 29 Mar 2024 19:14:26 -0700 Subject: [PATCH 46/90] add a couple more debug statements --- oldabe/money_in.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 21dd822..1ec4955 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -597,6 +597,8 @@ def distribute_payment(payment, attributions): payable_debts = get_payable_debts(unpayable_contributors, attributions) updated_debts, debt_transactions = pay_debts(payable_debts, payment) # The "available" amount is what is left over after paying off debts + print(f"new payment amount is {payment.amount}") + print(f"sum of debt transactions is {sum(t.amount for t in debt_transactions)}") available_amount = payment.amount - sum(t.amount for t in debt_transactions) fresh_debts = [] From 5f3fe148353767b6f11afaaf6a8de60908d81317 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Fri, 29 Mar 2024 19:23:15 -0700 Subject: [PATCH 47/90] more debug statements --- oldabe/money_in.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 1ec4955..64c96e5 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -729,6 +729,10 @@ def process_payments(instruments, attributions): new_debts += debts new_advances += advances fees_paid_out = sum(t.amount for t in transactions) + print(f"fees paid out are {fees_paid_out}") + print(f"total transaction amounts {sum(t.amount for t in transactions)}") + print(f"total debt amounts {sum(d.amount for d in debts)}") + print(f"total advance amounts {sum(a.amount for a in advances)}") # deduct the amount paid out to instruments before # processing it for attributions payment.amount -= fees_paid_out From e522554bf84b7a6244d19b69671f057eed677c76 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Fri, 29 Mar 2024 19:29:08 -0700 Subject: [PATCH 48/90] more debug statements --- oldabe/money_in.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 64c96e5..0bf63af 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -725,6 +725,7 @@ def process_payments(instruments, attributions): for payment in unprocessed_payments: # first, process instruments (i.e. pay fees) debts, transactions, advances = distribute_payment(payment, instruments) + print(f"payment amount BEFORE deducting fees is {payment.amount}") new_transactions += transactions new_debts += debts new_advances += advances @@ -736,6 +737,7 @@ def process_payments(instruments, attributions): # deduct the amount paid out to instruments before # processing it for attributions payment.amount -= fees_paid_out + print(f"payment amount AFTER deducting fees is {payment.amount}") new_itemized_payments.append( _create_itemized_payment(payment, fees_paid_out) ) From 92c756b2375aa21239f109e5bb9b71835e79e4e6 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Fri, 29 Mar 2024 19:37:00 -0700 Subject: [PATCH 49/90] don't double-deduct paid off debts from available payment amount --- oldabe/money_in.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 0bf63af..afbd8f2 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -470,12 +470,13 @@ def pay_debts(payable_debts, payment): """ updated_debts = [] transactions = [] + available_amount = payment.amount for debt in sorted(payable_debts, key=lambda x: x.created_at): - payable_amount = min(payment.amount, debt.amount_remaining()) + payable_amount = min(available_amount, debt.amount_remaining()) if payable_amount < ACCOUNTING_ZERO: break debt.amount_paid += payable_amount - payment.amount -= payable_amount + available_amount -= payable_amount transaction = Transaction(debt.email, payable_amount, payment.file, debt.commit_hash) transactions.append(transaction) updated_debts.append(debt) From d62acd3f465d1ccfa356e3842b54258cdcb86fd7 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 11 Apr 2024 20:52:38 -0700 Subject: [PATCH 50/90] Start on integration tests using pyfakefs --- Makefile | 11 +++++++++-- oldabe/money_in.py | 6 ++++-- setup.py | 1 + tests/integration/__init__.py | 0 tests/integration/money_in_test.py | 31 ++++++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/money_in_test.py diff --git a/Makefile b/Makefile index 0c1d35f..5ce0315 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ DOCS-PATH=docs export PYTEST_DISABLE_PLUGIN_AUTOLOAD = 1 UNIT_TESTS_PATH = tests/unit +INTEGRATION_TESTS_PATH = tests/integration help: @echo "clean - remove all build, test, coverage and Python artifacts" @@ -23,6 +24,7 @@ help: @echo "lint - alias for lint-source" @echo "black - run black auto-formatting on all code" @echo "test-unit - run unit tests" + @echo "test-integration - run integration tests" @echo "test - run specified tests, e.g.:" @echo " make test DEST=tests/unit/my_module.py" @echo " (defaults to unit tests if none specified)" @@ -93,7 +95,12 @@ black: test-unit: python setup.py test --addopts $(UNIT_TESTS_PATH) -test-all: clean-test test-unit +# NOTE: does not work! Only works when this identical +# command is run directly at the command line +test-integration: + pytest $(INTEGRATION_TESTS_PATH) + +test-all: clean-test test-unit test-integration test: ifdef DEST @@ -146,4 +153,4 @@ sdist: clean python setup.py sdist ls -l dist -.PHONY: help build build-for-test docs clean clean-build clean-pyc clean-test lint-source lint-tests lint-all lint black test-unit test-all test test-stop test-debug test-matrix test-tldr test-wiki debug coverage cover-coveralls sdist +.PHONY: help build build-for-test docs clean clean-build clean-pyc clean-test lint-source lint-tests lint-all lint black test-unit test-integration test-all test test-stop test-debug test-matrix test-tldr test-wiki debug coverage cover-coveralls sdist diff --git a/oldabe/money_in.py b/oldabe/money_in.py index afbd8f2..03beab9 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -120,8 +120,10 @@ def read_attributions(attributions_filename, validate=True): attributions_file = os.path.join(ABE_ROOT, attributions_filename) with open(attributions_file) as f: for row in csv.reader(f): - email, percentage = row - attributions[email] = parse_percentage(percentage) + if row and row[0].strip(): + email, percentage = row + email = email.strip() + attributions[email] = parse_percentage(percentage) if validate: assert _get_attributions_total(attributions) == Decimal("1") return attributions diff --git a/setup.py b/setup.py index c0670a2..f9e74ea 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ 'pytest-pudb', 'pytest-sugar', 'pytest-tldr', + 'pyfakefs', 'tox', 'tox-gh-actions', 'coveralls', diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py new file mode 100644 index 0000000..7cfb10f --- /dev/null +++ b/tests/integration/money_in_test.py @@ -0,0 +1,31 @@ +import pytest +from unittest.mock import patch +from oldabe.money_in import ( + process_payments_and_record_updates, +) +import os + + +@patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') +def test_creates_transactions_on_payment(mock_git_rev, fs): + ff = fs.create_file("./abe/price.txt", contents="10") + fs.create_file("./abe/valuation.txt", contents="100000") + fs.create_file("./abe/instruments.txt", contents=""" + old abe,1 + DIA,5 + """) + fs.create_file("./abe/attributions.txt", contents=""" + sid,50 + jair,30 + ariana,20 + """) + # print(ff.name) + # print(ff.contents) + # print(ff.filesystem) + with open('abe/price.txt') as f: + assert f.read() == "10" + process_payments_and_record_updates() + with open('./abe/transactions.txt') as f: + assert 'sid' in f.read() + assert 'jair' in f.read() + assert 'ariana' in f.read() From 25d6c49b3a11dad1d594f4f42661f7e87d713d0d Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 18 Apr 2024 20:40:00 -0700 Subject: [PATCH 51/90] a couple of starter integration tests --- tests/integration/money_in_test.py | 44 ++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py index 7cfb10f..00f8f14 100644 --- a/tests/integration/money_in_test.py +++ b/tests/integration/money_in_test.py @@ -1,4 +1,6 @@ import pytest +from datetime import datetime +import time_machine from unittest.mock import patch from oldabe.money_in import ( process_payments_and_record_updates, @@ -7,8 +9,8 @@ @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') -def test_creates_transactions_on_payment(mock_git_rev, fs): - ff = fs.create_file("./abe/price.txt", contents="10") +def test_no_transactions_if_no_payments(mock_git_rev, fs): + fs.create_file("./abe/price.txt", contents="10") fs.create_file("./abe/valuation.txt", contents="100000") fs.create_file("./abe/instruments.txt", contents=""" old abe,1 @@ -19,13 +21,39 @@ def test_creates_transactions_on_payment(mock_git_rev, fs): jair,30 ariana,20 """) - # print(ff.name) - # print(ff.contents) - # print(ff.filesystem) with open('abe/price.txt') as f: assert f.read() == "10" process_payments_and_record_updates() with open('./abe/transactions.txt') as f: - assert 'sid' in f.read() - assert 'jair' in f.read() - assert 'ariana' in f.read() + assert f.read() == "" + + +@time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) +@patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') +def test_creates_transactions_on_payment(mock_git_rev, fs): + fs.create_file("./abe/price.txt", contents="10") + fs.create_file("./abe/valuation.txt", contents="100000") + fs.create_file("./abe/instruments.txt", contents=""" + old abe,1 + DIA,5 + """) + fs.create_file("./abe/attributions.txt", contents=""" + sid,50 + jair,30 + ariana,20 + """) + + fs.create_file("./abe/payments/1.txt", + contents="sam,036eaf6,100,dummydate") + process_payments_and_record_updates() + with open('./abe/transactions.txt') as f: + # TODO: figure out why it's writing 3 decimal places + # and decide on handling + assert f.read() == ( + "old abe,1.000,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.000,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,47.000000,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,28.200000,1.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,18.800000,1.txt,abcd123,1985-10-26 01:24:00\n" + ) + From a788cd3ab01df208a74284cf3aa8b8cd19b82778 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 18 Apr 2024 20:40:29 -0700 Subject: [PATCH 52/90] handle missing input files --- oldabe/money_in.py | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 03beab9..243f0f7 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -160,15 +160,18 @@ def find_unprocessed_payments(): """ recorded_payments = set() transactions_file = os.path.join(ABE_ROOT, TRANSACTIONS_FILE) - with open(transactions_file) as f: - for ( - _email, - _amount, - payment_file, - _commit_hash, - _created_at, - ) in csv.reader(f): - recorded_payments.add(payment_file) + try: + with open(transactions_file) as f: + for ( + _email, + _amount, + payment_file, + _commit_hash, + _created_at, + ) in csv.reader(f): + recorded_payments.add(payment_file) + except FileNotFoundError: + pass all_payments = get_all_payments() print("all payments") print(all_payments) @@ -493,11 +496,14 @@ def get_unpayable_contributors(): """ unpayable_contributors_file = os.path.join(ABE_ROOT, UNPAYABLE_CONTRIBUTORS_FILE) contributors = [] - with open(unpayable_contributors_file) as f: - for contributor in f: - contributor = contributor.strip() - if contributor: - contributors.append(contributor) + try: + with open(unpayable_contributors_file) as f: + for contributor in f: + contributor = contributor.strip() + if contributor: + contributors.append(contributor) + except FileNotFoundError: + pass return contributors @@ -692,15 +698,6 @@ def handle_investment( return posterior_valuation -def _get_unprocessed_payments(): - try: - unprocessed_payments = find_unprocessed_payments() - except FileNotFoundError: - unprocessed_payments = [] - print(unprocessed_payments) - return unprocessed_payments - - def _create_itemized_payment(payment, fee_amount): return ItemizedPayment( payment.email, @@ -724,7 +721,8 @@ def process_payments(instruments, attributions): new_advances = [] new_transactions = [] new_itemized_payments = [] - unprocessed_payments = _get_unprocessed_payments() + unprocessed_payments = find_unprocessed_payments() + print("Unprocessed payments", unprocessed_payments) for payment in unprocessed_payments: # first, process instruments (i.e. pay fees) debts, transactions, advances = distribute_payment(payment, instruments) @@ -766,6 +764,7 @@ def process_payments_and_record_updates(): """ instruments = read_attributions(INSTRUMENTS_FILE, validate=False) attributions = read_attributions(ATTRIBUTIONS_FILE) + print("Attributions are:", attributions) ( debts, @@ -775,6 +774,7 @@ def process_payments_and_record_updates(): advances, ) = process_payments(instruments, attributions) + print("Transactions are:", transactions) # we only write the changes to disk at the end # so that if any errors are encountered, no # changes are made. From 16fe1445886f587aba1726150f843b8375299d65 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 18 Apr 2024 20:49:08 -0700 Subject: [PATCH 53/90] declare `time_machine` dependency for tests --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f9e74ea..386ad35 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ 'pytest-sugar', 'pytest-tldr', 'pyfakefs', + 'time_machine', 'tox', 'tox-gh-actions', 'coveralls', From 1b7eee00e5a33bdf6cfdee5758b145c05baef67a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 25 Apr 2024 19:39:38 -0700 Subject: [PATCH 54/90] a couple of working integration tests --- oldabe/money_in.py | 9 ++++++--- tests/integration/fixtures.py | 17 ++++++++++++++++ tests/integration/money_in_test.py | 32 +++++------------------------- 3 files changed, 28 insertions(+), 30 deletions(-) create mode 100644 tests/integration/fixtures.py diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 243f0f7..a0b4735 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -133,11 +133,14 @@ def get_all_payments(): """ Reads payment files and returns all existing payment objects. """ - payments = [ - read_payment(f, attributable=True) + try: + payments = [ + read_payment(f, attributable=True) for f in os.listdir(PAYMENTS_DIR) if not os.path.isdir(os.path.join(PAYMENTS_DIR, f)) - ] + ] + except FileNotFoundError: + payments = [] try: payments += [ read_payment(f, attributable=False) diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py new file mode 100644 index 0000000..2f6c1e8 --- /dev/null +++ b/tests/integration/fixtures.py @@ -0,0 +1,17 @@ +import pytest + + +@pytest.fixture +def abe_fs(fs): + fs.create_file("./abe/price.txt", contents="10") + fs.create_file("./abe/valuation.txt", contents="100000") + fs.create_file("./abe/instruments.txt", contents=""" + old abe,1 + DIA,5 + """) + fs.create_file("./abe/attributions.txt", contents=""" + sid,50 + jair,30 + ariana,20 + """) + return fs diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py index 00f8f14..2fb893f 100644 --- a/tests/integration/money_in_test.py +++ b/tests/integration/money_in_test.py @@ -5,22 +5,12 @@ from oldabe.money_in import ( process_payments_and_record_updates, ) +from .fixtures import abe_fs import os @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') -def test_no_transactions_if_no_payments(mock_git_rev, fs): - fs.create_file("./abe/price.txt", contents="10") - fs.create_file("./abe/valuation.txt", contents="100000") - fs.create_file("./abe/instruments.txt", contents=""" - old abe,1 - DIA,5 - """) - fs.create_file("./abe/attributions.txt", contents=""" - sid,50 - jair,30 - ariana,20 - """) +def test_no_transactions_if_no_payments(mock_git_rev, abe_fs): with open('abe/price.txt') as f: assert f.read() == "10" process_payments_and_record_updates() @@ -30,21 +20,9 @@ def test_no_transactions_if_no_payments(mock_git_rev, fs): @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') -def test_creates_transactions_on_payment(mock_git_rev, fs): - fs.create_file("./abe/price.txt", contents="10") - fs.create_file("./abe/valuation.txt", contents="100000") - fs.create_file("./abe/instruments.txt", contents=""" - old abe,1 - DIA,5 - """) - fs.create_file("./abe/attributions.txt", contents=""" - sid,50 - jair,30 - ariana,20 - """) - - fs.create_file("./abe/payments/1.txt", - contents="sam,036eaf6,100,dummydate") +def test_creates_transactions_on_payment(mock_git_rev, abe_fs): + abe_fs.create_file("./abe/payments/1.txt", + contents="sam,036eaf6,100,dummydate") process_payments_and_record_updates() with open('./abe/transactions.txt') as f: # TODO: figure out why it's writing 3 decimal places From 8aba0ead0eedd50afd21cb106ba21a1970f6170e Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 25 Apr 2024 20:57:51 -0700 Subject: [PATCH 55/90] Organize integration tests into classes and add a couple more --- oldabe/money_in.py | 2 +- tests/integration/money_in_test.py | 101 ++++++++++++++++++++++------- 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index a0b4735..72e351f 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -696,7 +696,7 @@ def handle_investment( incoming_attribution = calculate_incoming_attribution( payment.email, incoming_investment, posterior_valuation ) - if incoming_attribution: + if incoming_attribution and incoming_attribution.share > ACCOUNTING_ZERO: dilute_attributions(incoming_attribution, attributions) return posterior_valuation diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py index 2fb893f..9264aa2 100644 --- a/tests/integration/money_in_test.py +++ b/tests/integration/money_in_test.py @@ -1,5 +1,6 @@ import pytest from datetime import datetime +from decimal import localcontext import time_machine from unittest.mock import patch from oldabe.money_in import ( @@ -9,29 +10,79 @@ import os -@patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') -def test_no_transactions_if_no_payments(mock_git_rev, abe_fs): - with open('abe/price.txt') as f: - assert f.read() == "10" - process_payments_and_record_updates() - with open('./abe/transactions.txt') as f: - assert f.read() == "" - - -@time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) -@patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') -def test_creates_transactions_on_payment(mock_git_rev, abe_fs): - abe_fs.create_file("./abe/payments/1.txt", - contents="sam,036eaf6,100,dummydate") - process_payments_and_record_updates() - with open('./abe/transactions.txt') as f: - # TODO: figure out why it's writing 3 decimal places - # and decide on handling - assert f.read() == ( - "old abe,1.000,1.txt,abcd123,1985-10-26 01:24:00\n" - "DIA,5.000,1.txt,abcd123,1985-10-26 01:24:00\n" - "sid,47.000000,1.txt,abcd123,1985-10-26 01:24:00\n" - "jair,28.200000,1.txt,abcd123,1985-10-26 01:24:00\n" - "ariana,18.800000,1.txt,abcd123,1985-10-26 01:24:00\n" - ) +class TestNoPayments: + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_no_transactions_generated(self, mock_git_rev, abe_fs): + with open('abe/price.txt') as f: + assert f.read() == "10" + process_payments_and_record_updates() + with open('./abe/transactions.txt') as f: + assert f.read() == "" + +class TestPaymentAbovePrice: + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_generates_transactions(self, mock_git_rev, abe_fs): + amount = 100 + abe_fs.create_file("./abe/payments/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + process_payments_and_record_updates() + with open('./abe/transactions.txt') as f: + # TODO: figure out why it's writing 3 decimal places + # and decide on handling + assert f.read() == ( + "old abe,1.000,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.000,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,47.000000,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,28.200000,1.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,18.800000,1.txt,abcd123,1985-10-26 01:24:00\n" + ) + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_dilutes_attributions(self, mock_git_rev, abe_fs): + with localcontext() as context: + context.prec = 2 + amount = 10000 + abe_fs.create_file("./abe/payments/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + process_payments_and_record_updates() + with open('./abe/attributions.txt') as f: + # TODO: figure out why it's writing 3 decimal places + # and decide on handling + assert f.read() == ( + "sid,46\n" + "jair,28\n" + "ariana,18\n" + "sam,8.5\n" + ) + + +class TestPaymentBelowPrice: + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_generates_transactions(self, mock_git_rev, abe_fs, fs): + with localcontext() as context: + context.prec = 2 + amount = 1 + abe_fs.create_file("./abe/payments/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + process_payments_and_record_updates() + with open('./abe/transactions.txt') as f: + # TODO: figure out why it's writing 3 decimal places + # and decide on handling + assert f.read() == ( + "old abe,0.010,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,0.050,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,0.47,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,0.28,1.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,0.19,1.txt,abcd123,1985-10-26 01:24:00\n" + ) + + +class UnpayableContributor: + def test_records_debt(self): + pass + def test_debt_paid_on_becoming_payable(self): + pass From b6c543005f3c0f1ca283f1a98f39b9d779fbbd57 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 2 May 2024 19:27:22 -0700 Subject: [PATCH 56/90] Set convenient decimal precision in another test Also, so it doesn't get buried, note a todo to look into later. --- tests/integration/money_in_test.py | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py index 9264aa2..2d0fb96 100644 --- a/tests/integration/money_in_test.py +++ b/tests/integration/money_in_test.py @@ -19,25 +19,29 @@ def test_no_transactions_generated(self, mock_git_rev, abe_fs): with open('./abe/transactions.txt') as f: assert f.read() == "" +# TODO: in some cases even though the value is e.g. 1, +# it's writing out 3 decimal places, like 1.000. We should +# figure out why this is happening (and whether it's OK) +# and decide on appropriate handling class TestPaymentAbovePrice: @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') def test_generates_transactions(self, mock_git_rev, abe_fs): - amount = 100 - abe_fs.create_file("./abe/payments/1.txt", - contents=f"sam,036eaf6,{amount},dummydate") - process_payments_and_record_updates() - with open('./abe/transactions.txt') as f: - # TODO: figure out why it's writing 3 decimal places - # and decide on handling - assert f.read() == ( - "old abe,1.000,1.txt,abcd123,1985-10-26 01:24:00\n" - "DIA,5.000,1.txt,abcd123,1985-10-26 01:24:00\n" - "sid,47.000000,1.txt,abcd123,1985-10-26 01:24:00\n" - "jair,28.200000,1.txt,abcd123,1985-10-26 01:24:00\n" - "ariana,18.800000,1.txt,abcd123,1985-10-26 01:24:00\n" - ) + with localcontext() as context: + context.prec = 2 + amount = 100 + abe_fs.create_file("./abe/payments/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + process_payments_and_record_updates() + with open('./abe/transactions.txt') as f: + assert f.read() == ( + "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,47,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,28,1.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,19,1.txt,abcd123,1985-10-26 01:24:00\n" + ) @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') @@ -49,8 +53,6 @@ def test_dilutes_attributions(self, mock_git_rev, abe_fs): contents=f"sam,036eaf6,{amount},dummydate") process_payments_and_record_updates() with open('./abe/attributions.txt') as f: - # TODO: figure out why it's writing 3 decimal places - # and decide on handling assert f.read() == ( "sid,46\n" "jair,28\n" @@ -62,7 +64,7 @@ def test_dilutes_attributions(self, mock_git_rev, abe_fs): class TestPaymentBelowPrice: @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') - def test_generates_transactions(self, mock_git_rev, abe_fs, fs): + def test_generates_transactions(self, mock_git_rev, abe_fs): with localcontext() as context: context.prec = 2 amount = 1 @@ -70,8 +72,6 @@ def test_generates_transactions(self, mock_git_rev, abe_fs, fs): contents=f"sam,036eaf6,{amount},dummydate") process_payments_and_record_updates() with open('./abe/transactions.txt') as f: - # TODO: figure out why it's writing 3 decimal places - # and decide on handling assert f.read() == ( "old abe,0.010,1.txt,abcd123,1985-10-26 01:24:00\n" "DIA,0.050,1.txt,abcd123,1985-10-26 01:24:00\n" From 1ce8abb305870712127bb854031ce4cd5d45aa01 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 2 May 2024 20:17:47 -0700 Subject: [PATCH 57/90] More integration tests Also stub out remaining tests we thought of. --- tests/integration/money_in_test.py | 75 ++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py index 2d0fb96..d9a781d 100644 --- a/tests/integration/money_in_test.py +++ b/tests/integration/money_in_test.py @@ -24,6 +24,7 @@ def test_no_transactions_generated(self, mock_git_rev, abe_fs): # figure out why this is happening (and whether it's OK) # and decide on appropriate handling + class TestPaymentAbovePrice: @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') @@ -81,8 +82,76 @@ def test_generates_transactions(self, mock_git_rev, abe_fs): ) -class UnpayableContributor: - def test_records_debt(self): +class TestNonAttributablePayment: + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_does_not_dilute_attributions(self, mock_git_rev, abe_fs): + with localcontext() as context: + context.prec = 2 + amount = 10000 + abe_fs.create_file("./abe/payments/nonattributable/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + process_payments_and_record_updates() + with open('./abe/attributions.txt') as f: + assert f.read() == ( + "sid,50\n" + "jair,30\n" + "ariana,20\n" + ) + + +class TestUnpayableContributor: + + def _call(self, abe_fs): + with localcontext() as context: + context.prec = 2 + amount = 100 + abe_fs.create_file("./abe/payments/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + abe_fs.create_file("./abe/unpayable_contributors.txt", + contents=f"ariana") + process_payments_and_record_updates() + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_generates_transactions(self, mock_git_rev, abe_fs): + self._call(abe_fs) + with open('./abe/transactions.txt') as f: + assert f.read() == ( + "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,58,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,35,1.txt,abcd123,1985-10-26 01:24:00\n" + ) + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_records_debt(self, mock_git_rev, abe_fs): + self._call(abe_fs) + with open('./abe/debts.txt') as f: + assert f.read() == ( + "ariana,19,0,1.txt,abcd123,1985-10-26 01:24:00\n" + ) + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_records_advances(self, mock_git_rev, abe_fs): + pass + + +class TestUnpayableContributorBecomesPayable: + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_debt_paid(self, mock_git_rev, abe_fs): pass - def test_debt_paid_on_becoming_payable(self): + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_transactions(self, mock_git_rev, abe_fs): + pass + + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) + @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') + def test_advances(self, mock_git_rev, abe_fs): pass From 74ba9958551095be0de5c37daaecc90423660e5a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 2 May 2024 20:18:40 -0700 Subject: [PATCH 58/90] Remove nested context manager for opening debts file This was working though unintended, but it was confusing FakeFilesystem in tests. --- oldabe/money_in.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/oldabe/money_in.py b/oldabe/money_in.py index 72e351f..f913fc7 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -541,17 +541,16 @@ def write_debts(processed_debts): debts_file = os.path.join(ABE_ROOT, DEBTS_FILE) with open(debts_file, 'w') as f: writer = csv.writer(f) - with open(debts_file, 'w') as f: - for existing_debt in existing_debts: - # if the existing debt has been processed, write the processed version - # otherwise re-write the existing version - if processed_debt := processed_debts_hash.get(existing_debt.key()): - writer.writerow(astuple(processed_debt)) - del processed_debts_hash[processed_debt.key()] - else: - writer.writerow(astuple(existing_debt)) - for debt in processed_debts_hash.values(): - writer.writerow(astuple(debt)) + for existing_debt in existing_debts: + # if the existing debt has been processed, write the processed version + # otherwise re-write the existing version + if processed_debt := processed_debts_hash.get(existing_debt.key()): + writer.writerow(astuple(processed_debt)) + del processed_debts_hash[processed_debt.key()] + else: + writer.writerow(astuple(existing_debt)) + for debt in processed_debts_hash.values(): + writer.writerow(astuple(debt)) def renormalize(attributions, excluded_contributors): From 0489c4fd086d8820d486c814b41bdcffb8861888 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Thu, 9 May 2024 20:03:41 -0700 Subject: [PATCH 59/90] Integration tests for remaining cases we identified --- tests/integration/money_in_test.py | 71 ++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/tests/integration/money_in_test.py b/tests/integration/money_in_test.py index d9a781d..73b2577 100644 --- a/tests/integration/money_in_test.py +++ b/tests/integration/money_in_test.py @@ -136,22 +136,85 @@ def test_records_debt(self, mock_git_rev, abe_fs): @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') def test_records_advances(self, mock_git_rev, abe_fs): - pass + # advances for payable people + # and none for unpayable + self._call(abe_fs) + with open('./abe/advances.txt') as f: + assert f.read() == ( + "sid,11,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n" + ) class TestUnpayableContributorBecomesPayable: + def _call(self, abe_fs): + with localcontext() as context: + context.prec = 2 + amount = 100 + abe_fs.create_file('./abe/transactions.txt', + contents=( + "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,58,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,35,1.txt,abcd123,1985-10-26 01:24:00\n" + )) + + abe_fs.create_file("./abe/debts.txt", + contents="ariana,19,0,1.txt,abcd123,1985-10-26 01:24:00\n") + abe_fs.create_file("./abe/advances.txt", + contents=( + "sid,11,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n" + )) + abe_fs.create_file("./abe/payments/1.txt", + contents=f"sam,036eaf6,{amount},dummydate") + abe_fs.create_file("./abe/payments/2.txt", + contents=f"sam,036eaf6,{amount},dummydate") + process_payments_and_record_updates() + @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') def test_debt_paid(self, mock_git_rev, abe_fs): - pass + self._call(abe_fs) + with open('./abe/debts.txt') as f: + assert f.read() == ( + "ariana,19,19,1.txt,abcd123,1985-10-26 01:24:00\n" + ) @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') def test_transactions(self, mock_git_rev, abe_fs): - pass + # here, because the two payment amounts are the same, + # it ends up correcting immediately. We might consider + # more tests where the second amount is larger, or + # where there are more debts + self._call(abe_fs) + with open('./abe/transactions.txt') as f: + assert f.read() == ( + "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,58,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,35,1.txt,abcd123,1985-10-26 01:24:00\n" + "old abe,1.0,2.txt,abcd123,1985-10-26 01:24:00\n" + "DIA,5.0,2.txt,abcd123,1985-10-26 01:24:00\n" + "sid,36,2.txt,abcd123,1985-10-26 01:24:00\n" + "jair,20,2.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,19,2.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,19,2.txt,abcd123,1985-10-26 01:24:00\n" + ) @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False) @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123') def test_advances(self, mock_git_rev, abe_fs): - pass + self._call(abe_fs) + with open('./abe/advances.txt') as f: + assert f.read() == ( + "sid,11,1.txt,abcd123,1985-10-26 01:24:00\n" + "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n" + "sid,-11,2.txt,abcd123,1985-10-26 01:24:00\n" + "jair,-6.8,2.txt,abcd123,1985-10-26 01:24:00\n" + "sid,9.0,2.txt,abcd123,1985-10-26 01:24:00\n" + "jair,5.4,2.txt,abcd123,1985-10-26 01:24:00\n" + "ariana,3.6,2.txt,abcd123,1985-10-26 01:24:00\n" + ) From 636ac58ef2227b1ab34438bb536de8f7cacc0888 Mon Sep 17 00:00:00 2001 From: apromessi Date: Tue, 14 May 2024 20:56:10 -0700 Subject: [PATCH 60/90] create new files for constants and utils --- oldabe/constants.py | 16 ++++ oldabe/money_in.py | 152 ++++-------------------------------- oldabe/money_out.py | 9 +-- oldabe/utils.py | 84 ++++++++++++++++++++ tests/unit/money_in_test.py | 100 +----------------------- tests/unit/utils_test.py | 126 ++++++++++++++++++++++++++++++ 6 files changed, 245 insertions(+), 242 deletions(-) create mode 100644 oldabe/constants.py create mode 100644 oldabe/utils.py create mode 100644 tests/unit/utils_test.py diff --git a/oldabe/constants.py b/oldabe/constants.py new file mode 100644 index 0000000..5a4040a --- /dev/null +++ b/oldabe/constants.py @@ -0,0 +1,16 @@ +import os + +ABE_ROOT = 'abe' +PAYOUTS_DIR = os.path.join(ABE_ROOT, 'payouts') +TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt') +DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt') +ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt') +NONATTRIBUTABLE_PAYMENTS_DIR = os.path.join( + ABE_ROOT, 'payments', 'nonattributable' +) +UNPAYABLE_CONTRIBUTORS_FILE = 'unpayable_contributors.txt' +ITEMIZED_PAYMENTS_FILE = 'itemized_payments.txt' +PRICE_FILE = 'price.txt' +VALUATION_FILE = 'valuation.txt' +ATTRIBUTIONS_FILE = 'attributions.txt' +INSTRUMENTS_FILE = 'instruments.txt' diff --git a/oldabe/money_in.py b/oldabe/money_in.py index f913fc7..b4eacf4 100755 --- a/oldabe/money_in.py +++ b/oldabe/money_in.py @@ -10,78 +10,24 @@ from .models import ( Advance, Attribution, Debt, Payment, ItemizedPayment, Transaction ) - -ABE_ROOT = './abe' -# ABE_ROOT = '.' -PAYMENTS_DIR = os.path.join(ABE_ROOT, 'payments') -NONATTRIBUTABLE_PAYMENTS_DIR = os.path.join( - ABE_ROOT, 'payments', 'nonattributable' +from .utils. import ( + parse_percentage, serialize_proportion, get_rounding_difference, + correct_rounding_error, get_git_revision_short_hash +) +from .constants import ( + ABE_ROOT, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE, ADVANCES_FILE + NONATTRIBUTABLE_PAYMENTS_DIR, UNPAYABLE_CONTRIBUTORS_FILE, + ITEMIZED_PAYMENTS_FILE, PRICE_FILE, VALUATION_FILE, ATTRIBUTIONS_FILE, + INSTRUMENTS_FILE ) -UNPAYABLE_CONTRIBUTORS_FILE = 'unpayable_contributors.txt' -TRANSACTIONS_FILE = 'transactions.txt' -DEBTS_FILE = 'debts.txt' -ADVANCES_FILE = 'advances.txt' -ITEMIZED_PAYMENTS_FILE = 'itemized_payments.txt' -PRICE_FILE = 'price.txt' -VALUATION_FILE = 'valuation.txt' -ATTRIBUTIONS_FILE = 'attributions.txt' -INSTRUMENTS_FILE = 'instruments.txt' + ROUNDING_TOLERANCE = Decimal("0.000001") ACCOUNTING_ZERO = Decimal("0.01") -# TODO - move to utils -def parse_percentage(value): - """ - Translates values expressed in percentage format (75.234) into - their decimal equivalents (0.75234). This effectively divides - the value by 100 without losing precision. - """ - value = re.sub("[^0-9.]", "", value) - value = "00" + value - if "." not in value: - value = value + ".0" - value = re.sub( - r"(?P
\d{2})\.(?P\d+)", r".\g
\g", value
-    )
-    value = Decimal(value)
-    return value
-
-
-# TODO - move to utils
-def serialize_proportion(value):
-    """
-    Translates values expressed in decimal format (0.75234) into
-    their percentage equivalents (75.234). This effectively multiplies
-    the value by 100 without losing precision.
-    """
-    # otherwise, decimal gets translated '2E-7.0'
-    value = format(value, "f")
-    if "." in value:
-        value = value + "00"
-    else:
-        value = value + ".00"
-    value = re.sub(
-        r"(?P
\d)\.(?P\d{2})(?P\d*)",
-        # match a number followed by a decimal point
-        # followed by at least two digits
-        r"\g
\g.\g",
-        # move the decimal point two places to the right
-        value,
-    )
-    # strip leading insignificant zeroes
-    value = value.lstrip("0")
-    # ensure there's a single leading zero if it is
-    # a decimal value less than 1
-    value = re.sub(r"^\.", r"0.", value)
-    if "." in value:
-        # strip trailing insignificant zeroes
-        value = value.rstrip("0")
-        # remove decimal point if whole number
-        value = re.sub(r"\.$", r"", value)
-    return value
-
+# TODO standardize the parsing from text into python objects
+# e.g. Decimal and DateTime
 
 def read_payment(payment_file, attributable=True):
     """
@@ -176,8 +122,6 @@ def find_unprocessed_payments():
     except FileNotFoundError:
         pass
     all_payments = get_all_payments()
-    print("all payments")
-    print(all_payments)
     return [p for p in all_payments if p.file not in recorded_payments]
 
 
@@ -196,7 +140,9 @@ def generate_transactions(amount, attributions, payment_file, commit_hash):
 
 
 def get_existing_itemized_payments():
-    # TODO
+    """
+    Reads itemized payment files and returns all itemized payment objects.
+    """
     itemized_payments = []
     itemized_payments_file = os.path.join(ABE_ROOT, ITEMIZED_PAYMENTS_FILE)
     try:
@@ -270,19 +216,6 @@ def _get_attributions_total(attributions):
     return sum(attributions.values())
 
 
-# TODO - move to utils
-def get_rounding_difference(attributions):
-    """
-    Get the difference of the total of the attributions from 1, which is
-    expected to occur due to finite precision. If the difference exceeds the
-    expected error tolerance, an error is signaled.
-    """
-    total = _get_attributions_total(attributions)
-    difference = total - Decimal("1")
-    assert abs(difference) <= ROUNDING_TOLERANCE
-    return difference
-
-
 def normalize(attributions):
     total_share = sum(share for _, share in attributions.items())
     target_proportion = Decimal("1") / total_share
@@ -290,17 +223,6 @@ def normalize(attributions):
         attributions[email] *= target_proportion
 
 
-# TODO - move to utils
-def correct_rounding_error(attributions, incoming_attribution):
-    """Due to finite precision, the Decimal module will round up or down
-    on the last decimal place. This could result in the aggregate value not
-    quite totaling to 1. This corrects that total by either adding or
-    subtracting the difference from the incoming attribution (by convention).
-    """
-    difference = get_rounding_difference(attributions)
-    attributions[incoming_attribution.email] -= difference
-
-
 def write_attributions(attributions):
     # don't write attributions if they aren't normalized
     assert _get_attributions_total(attributions) == Decimal("1")
@@ -340,18 +262,6 @@ def write_append_advances(advances):
             writer.writerow(astuple(row))
 
 
-# [[ OLD NOTE - still relevant?? ]]
-# TODO - when we write debts
-# 1) read all from the file and transform into a hash, where the key is a 
-#   unique identifier constructed from the email + payment file and the value
-#   is a debt object
-# 2) take the newly created/modified debt objects (all in one list) and iterate
-#   through - searching the full hash for each new debt object - modify if it
-#   was found, and add to the hash if not found
-# 3) convert the hash into a list, ordered by created_at field, then write to
-#   the debts file (completely replacing existing contents)
-
-
 def write_valuation(valuation):
     rounded_valuation = f"{valuation:.2f}"
     valuation_file = os.path.join(ABE_ROOT, VALUATION_FILE)
@@ -394,18 +304,6 @@ def inflate_valuation(valuation, amount):
     return valuation + amount
 
 
-# TODO - move to utils
-# and standardize the parsing from text into python objects
-# e.g. Decimal and DateTime
-def get_git_revision_short_hash() -> str:
-    """From https://stackoverflow.com/a/21901260"""
-    return (
-        subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'])
-        .decode('ascii')
-        .strip()
-    )
-
-
 def read_debts():
     debts_file = os.path.join(ABE_ROOT, DEBTS_FILE)
     debts = []
@@ -601,25 +499,19 @@ def distribute_payment(payment, attributions):
     # 2. pay them off in chronological order (maybe partially)
     # 3. (if leftover) identify unpayable people in the relevant attributions file
     # 4. record debt for each of them according to their attribution
-    print("Listing directory files...")
-    print(os.listdir(ABE_ROOT))
     commit_hash = get_git_revision_short_hash()
     unpayable_contributors = get_unpayable_contributors()
     payable_debts = get_payable_debts(unpayable_contributors, attributions)
     updated_debts, debt_transactions = pay_debts(payable_debts, payment)
     # The "available" amount is what is left over after paying off debts
-    print(f"new payment amount is {payment.amount}")
-    print(f"sum of debt transactions is {sum(t.amount for t in debt_transactions)}")
     available_amount = payment.amount - sum(t.amount for t in debt_transactions)
 
     fresh_debts = []
     equity_transactions = []
     negative_advances = []
     fresh_advances = []
-    print(f"available amount is {available_amount}")
     if available_amount > ACCOUNTING_ZERO:
         amounts_owed = get_amounts_owed(available_amount, attributions)
-        print(f"total of amounts owed is {sum(v for v in amounts_owed.values())}")
         fresh_debts = create_debts(amounts_owed,
                                    unpayable_contributors,
                                    payment.file)
@@ -629,12 +521,10 @@ def distribute_payment(payment, attributions):
         amounts_payable = {email: amount
                            for email, amount in amounts_owed.items()
                            if email not in unpayable_contributors}
-        print(f"total of amounts payable is {sum(v for v in amounts_payable.values())}")
 
         # use the amount owed to each contributor to draw down any advances
         # they may already have and then decrement their amount payable accordingly
         advance_totals = get_sum_of_advances_by_contributor(attributions)
-        print(f"advance totals is {advance_totals}")
         for email, advance_total in advance_totals.items():
             amount_payable = amounts_payable.get(email, 0)
             drawdown_amount = min(advance_total, amount_payable)
@@ -649,7 +539,6 @@ def distribute_payment(payment, attributions):
         # note that these are drawn down amounts and therefore have negative amounts
         # and that's why we take the absolute value here
         redistribution_pot += sum(abs(a.amount) for a in negative_advances)
-        print(f"redistribution_pot is {redistribution_pot}")
 
         # redistribute the pot over all payable contributors - produce fresh advances and add to amounts payable
         if redistribution_pot > ACCOUNTING_ZERO:
@@ -666,8 +555,6 @@ def distribute_payment(payment, attributions):
                                                  commit_hash=commit_hash)
             equity_transactions.append(new_equity_transaction)
 
-    print(f"sum of equity transactions is {sum(t.amount for t in equity_transactions)}")
-    print(f"sum of debt transactions is {sum(t.amount for t in debt_transactions)}")
     debts = updated_debts + fresh_debts
     transactions = equity_transactions + debt_transactions
     advances = negative_advances + fresh_advances
@@ -724,23 +611,16 @@ def process_payments(instruments, attributions):
     new_transactions = []
     new_itemized_payments = []
     unprocessed_payments = find_unprocessed_payments()
-    print("Unprocessed payments", unprocessed_payments)
     for payment in unprocessed_payments:
         # first, process instruments (i.e. pay fees)
         debts, transactions, advances = distribute_payment(payment, instruments)
-        print(f"payment amount BEFORE deducting fees is {payment.amount}")
         new_transactions += transactions
         new_debts += debts
         new_advances += advances
         fees_paid_out = sum(t.amount for t in transactions)
-        print(f"fees paid out are {fees_paid_out}")
-        print(f"total transaction amounts {sum(t.amount for t in transactions)}")
-        print(f"total debt amounts {sum(d.amount for d in debts)}")
-        print(f"total advance amounts {sum(a.amount for a in advances)}")
         # deduct the amount paid out to instruments before
         # processing it for attributions
         payment.amount -= fees_paid_out
-        print(f"payment amount AFTER deducting fees is {payment.amount}")
         new_itemized_payments.append(
             _create_itemized_payment(payment, fees_paid_out)
         )
@@ -766,7 +646,6 @@ def process_payments_and_record_updates():
     """
     instruments = read_attributions(INSTRUMENTS_FILE, validate=False)
     attributions = read_attributions(ATTRIBUTIONS_FILE)
-    print("Attributions are:", attributions)
 
     (
         debts,
@@ -776,7 +655,6 @@ def process_payments_and_record_updates():
         advances,
     ) = process_payments(instruments, attributions)
 
-    print("Transactions are:", transactions)
     # we only write the changes to disk at the end
     # so that if any errors are encountered, no
     # changes are made.
diff --git a/oldabe/money_out.py b/oldabe/money_out.py
index 41c7406..542ed8a 100755
--- a/oldabe/money_out.py
+++ b/oldabe/money_out.py
@@ -7,12 +7,9 @@
 import re
 from collections import defaultdict
 from decimal import Decimal, getcontext
-
-ABE_ROOT = 'abe'
-PAYOUTS_DIR = os.path.join(ABE_ROOT, 'payouts')
-TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt')
-DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt')
-ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt')
+from .constants import (
+    ABE_ROOT, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE, ADVANCES_FILE
+)
 
 
 def read_transaction_amounts():
diff --git a/oldabe/utils.py b/oldabe/utils.py
new file mode 100644
index 0000000..6f2b7f1
--- /dev/null
+++ b/oldabe/utils.py
@@ -0,0 +1,84 @@
+from decimal import Decimal
+import re
+import subprocess
+
+
+def parse_percentage(value):
+    """
+    Translates values expressed in percentage format (75.234) into
+    their decimal equivalents (0.75234). This effectively divides
+    the value by 100 without losing precision.
+    """
+    value = re.sub("[^0-9.]", "", value)
+    value = "00" + value
+    if "." not in value:
+        value = value + ".0"
+    value = re.sub(
+        r"(?P
\d{2})\.(?P\d+)", r".\g
\g", value
+    )
+    value = Decimal(value)
+    return value
+
+
+def serialize_proportion(value):
+    """
+    Translates values expressed in decimal format (0.75234) into
+    their percentage equivalents (75.234). This effectively multiplies
+    the value by 100 without losing precision.
+    """
+    # otherwise, decimal gets translated '2E-7.0'
+    value = format(value, "f")
+    if "." in value:
+        value = value + "00"
+    else:
+        value = value + ".00"
+    value = re.sub(
+        r"(?P
\d)\.(?P\d{2})(?P\d*)",
+        # match a number followed by a decimal point
+        # followed by at least two digits
+        r"\g
\g.\g",
+        # move the decimal point two places to the right
+        value,
+    )
+    # strip leading insignificant zeroes
+    value = value.lstrip("0")
+    # ensure there's a single leading zero if it is
+    # a decimal value less than 1
+    value = re.sub(r"^\.", r"0.", value)
+    if "." in value:
+        # strip trailing insignificant zeroes
+        value = value.rstrip("0")
+        # remove decimal point if whole number
+        value = re.sub(r"\.$", r"", value)
+    return value
+
+
+def get_rounding_difference(attributions):
+    """
+    Get the difference of the total of the attributions from 1, which is
+    expected to occur due to finite precision. If the difference exceeds the
+    expected error tolerance, an error is signaled.
+    """
+    total = _get_attributions_total(attributions)
+    difference = total - Decimal("1")
+    assert abs(difference) <= ROUNDING_TOLERANCE
+    return difference
+
+
+def correct_rounding_error(attributions, incoming_attribution):
+    """Due to finite precision, the Decimal module will round up or down
+    on the last decimal place. This could result in the aggregate value not
+    quite totaling to 1. This corrects that total by either adding or
+    subtracting the difference from the incoming attribution (by convention).
+    """
+    difference = get_rounding_difference(attributions)
+    attributions[incoming_attribution.email] -= difference
+
+
+def get_git_revision_short_hash() -> str:
+    """From https://stackoverflow.com/a/21901260"""
+    return (
+        subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'])
+        .decode('ascii')
+        .strip()
+    )
diff --git a/tests/unit/money_in_test.py b/tests/unit/money_in_test.py
index 4fb1c90..44120c7 100644
--- a/tests/unit/money_in_test.py
+++ b/tests/unit/money_in_test.py
@@ -1,18 +1,15 @@
 from decimal import Decimal
 from oldabe.money_in import (
     calculate_incoming_investment,
-    parse_percentage,
-    serialize_proportion,
     total_amount_paid_to_project,
     calculate_incoming_attribution,
-    correct_rounding_error,
     generate_transactions,
-    get_rounding_difference,
     ROUNDING_TOLERANCE,
     renormalize,
     inflate_valuation,
     process_payments,
 )
+from oldabe.utils import correct_rounding_error
 from oldabe.models import Attribution, Payment
 import pytest
 from unittest.mock import patch
@@ -29,61 +26,6 @@
 )  # noqa
 
 
-class TestParsePercentage:
-    def test_an_integer(self):
-        assert parse_percentage('75') == Decimal('0.75')
-
-    def test_non_integer_greater_than_1(self):
-        assert parse_percentage('75.334455') == Decimal('0.75334455')
-
-    def test_non_integer_less_than_1(self):
-        assert parse_percentage('0.334455') == Decimal('0.00334455')
-
-    def test_decimal_places_at_precision_context(self):
-        assert parse_percentage('5.1234567891') == Decimal('0.051234567891')
-
-    def test_very_small_number(self):
-        assert parse_percentage('0.000001') == Decimal('0.00000001')
-
-    def test_0(self):
-        assert parse_percentage('0') == Decimal('0')
-
-    def test_0_point_0(self):
-        assert parse_percentage('0.0') == Decimal('0')
-
-    def test_100(self):
-        assert parse_percentage('100') == Decimal('1')
-
-    def test_100_point_0(self):
-        assert parse_percentage('100.0') == Decimal('1')
-
-
-class TestSerializeProportion:
-    def test_0(self):
-        assert serialize_proportion(Decimal('0')) == '0'
-
-    def test_0_point_0(self):
-        assert serialize_proportion(Decimal('0.0')) == '0'
-
-    def test_1(self):
-        assert serialize_proportion(Decimal('1')) == '100'
-
-    def test_1_point_0(self):
-        assert serialize_proportion(Decimal('1.0')) == '100'
-
-    def test_almost_1(self):
-        assert serialize_proportion(Decimal('0.9523452')) == '95.23452'
-
-    def test_low_precision_number(self):
-        assert serialize_proportion(Decimal('0.2')) == '20'
-
-    def test_very_small_number(self):
-        assert serialize_proportion(Decimal('0.0000002')) == '0.00002'
-
-    def test_decimal_places_at_precision_context(self):
-        assert serialize_proportion(Decimal('0.1234567891')) == '12.34567891'
-
-
 class TestGenerateTransactions:
     def test_zero_amount(self, normalized_attributions):
         with pytest.raises(AssertionError):
@@ -239,46 +181,6 @@ def test_large_incoming_investment(self, normalized_attributions):
         )
 
 
-class TestGetRoundingDifference:
-    def test_attributions_already_one(self, normalized_attributions):
-        attributions = normalized_attributions
-        test_diff = Decimal("0")
-        difference = get_rounding_difference(attributions)
-        assert difference == test_diff
-
-    def test_attributions_exceed_below_tolerance(
-        self, normalized_attributions
-    ):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE / Decimal("2")
-        attributions['c@d.com'] = test_diff
-        difference = get_rounding_difference(attributions)
-        assert difference == test_diff
-
-    def test_attributions_exceed_above_tolerance(
-        self, normalized_attributions
-    ):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE * Decimal("2")
-        attributions['c@d.com'] = test_diff
-        with pytest.raises(AssertionError):
-            _ = get_rounding_difference(attributions)
-
-    def test_attributions_exceed_by_tolerance(self, normalized_attributions):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE
-        attributions['c@d.com'] = test_diff
-        difference = get_rounding_difference(attributions)
-        assert difference == test_diff
-
-    def test_attributions_below_one(self, normalized_attributions):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE / Decimal("2")
-        attributions['b@c.com'] -= test_diff
-        difference = get_rounding_difference(attributions)
-        assert difference == -test_diff
-
-
 class TestRenormalize:
     @pytest.mark.parametrize(
         "incoming_attribution, renormalized_attributions",
diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py
new file mode 100644
index 0000000..008ec97
--- /dev/null
+++ b/tests/unit/utils_test.py
@@ -0,0 +1,126 @@
+from decimal import Decimal
+from oldabe.money_in import (
+    calculate_incoming_investment,
+    total_amount_paid_to_project,
+    calculate_incoming_attribution,
+    generate_transactions,
+    ROUNDING_TOLERANCE,
+    renormalize,
+    inflate_valuation,
+    process_payments,
+)
+from oldabe.utils import (
+    parse_percentage,
+    serialize_proportion,
+    get_rounding_difference,
+    correct_rounding_error,
+)
+from oldabe.models import Attribution, Payment
+import pytest
+from unittest.mock import patch
+from .utils import call_sequence
+from .fixtures import (
+    instruments,
+    normalized_attributions,
+    excess_attributions,
+    shortfall_attributions,
+    empty_attributions,
+    single_contributor_attributions,
+    itemized_payments,
+    new_itemized_payments,
+)  # noqa
+
+
+class TestParsePercentage:
+    def test_an_integer(self):
+        assert parse_percentage('75') == Decimal('0.75')
+
+    def test_non_integer_greater_than_1(self):
+        assert parse_percentage('75.334455') == Decimal('0.75334455')
+
+    def test_non_integer_less_than_1(self):
+        assert parse_percentage('0.334455') == Decimal('0.00334455')
+
+    def test_decimal_places_at_precision_context(self):
+        assert parse_percentage('5.1234567891') == Decimal('0.051234567891')
+
+    def test_very_small_number(self):
+        assert parse_percentage('0.000001') == Decimal('0.00000001')
+
+    def test_0(self):
+        assert parse_percentage('0') == Decimal('0')
+
+    def test_0_point_0(self):
+        assert parse_percentage('0.0') == Decimal('0')
+
+    def test_100(self):
+        assert parse_percentage('100') == Decimal('1')
+
+    def test_100_point_0(self):
+        assert parse_percentage('100.0') == Decimal('1')
+
+
+class TestSerializeProportion:
+    def test_0(self):
+        assert serialize_proportion(Decimal('0')) == '0'
+
+    def test_0_point_0(self):
+        assert serialize_proportion(Decimal('0.0')) == '0'
+
+    def test_1(self):
+        assert serialize_proportion(Decimal('1')) == '100'
+
+    def test_1_point_0(self):
+        assert serialize_proportion(Decimal('1.0')) == '100'
+
+    def test_almost_1(self):
+        assert serialize_proportion(Decimal('0.9523452')) == '95.23452'
+
+    def test_low_precision_number(self):
+        assert serialize_proportion(Decimal('0.2')) == '20'
+
+    def test_very_small_number(self):
+        assert serialize_proportion(Decimal('0.0000002')) == '0.00002'
+
+    def test_decimal_places_at_precision_context(self):
+        assert serialize_proportion(Decimal('0.1234567891')) == '12.34567891'
+
+
+class TestGetRoundingDifference:
+    def test_attributions_already_one(self, normalized_attributions):
+        attributions = normalized_attributions
+        test_diff = Decimal("0")
+        difference = get_rounding_difference(attributions)
+        assert difference == test_diff
+
+    def test_attributions_exceed_below_tolerance(
+        self, normalized_attributions
+    ):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE / Decimal("2")
+        attributions['c@d.com'] = test_diff
+        difference = get_rounding_difference(attributions)
+        assert difference == test_diff
+
+    def test_attributions_exceed_above_tolerance(
+        self, normalized_attributions
+    ):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE * Decimal("2")
+        attributions['c@d.com'] = test_diff
+        with pytest.raises(AssertionError):
+            _ = get_rounding_difference(attributions)
+
+    def test_attributions_exceed_by_tolerance(self, normalized_attributions):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE
+        attributions['c@d.com'] = test_diff
+        difference = get_rounding_difference(attributions)
+        assert difference == test_diff
+
+    def test_attributions_below_one(self, normalized_attributions):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE / Decimal("2")
+        attributions['b@c.com'] -= test_diff
+        difference = get_rounding_difference(attributions)
+        assert difference == -test_diff

From 298bac07926b9361102483f88be7b8915b24c4df Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 14 May 2024 21:09:23 -0700
Subject: [PATCH 61/90] fix syntax

---
 oldabe/money_in.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index b4eacf4..3cbf076 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -10,7 +10,7 @@
 from .models import (
     Advance, Attribution, Debt, Payment, ItemizedPayment, Transaction
 )
-from .utils. import (
+from .utils import (
     parse_percentage, serialize_proportion, get_rounding_difference,
     correct_rounding_error, get_git_revision_short_hash
 )

From f5e475ab2210e25e3a75f6b6777863a462435d62 Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 14 May 2024 21:10:56 -0700
Subject: [PATCH 62/90] fix syntax

---
 oldabe/money_in.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index 3cbf076..db9dd67 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -15,7 +15,7 @@
     correct_rounding_error, get_git_revision_short_hash
 )
 from .constants import (
-    ABE_ROOT, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE, ADVANCES_FILE
+    ABE_ROOT, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE, ADVANCES_FILE,
     NONATTRIBUTABLE_PAYMENTS_DIR, UNPAYABLE_CONTRIBUTORS_FILE,
     ITEMIZED_PAYMENTS_FILE, PRICE_FILE, VALUATION_FILE, ATTRIBUTIONS_FILE,
     INSTRUMENTS_FILE

From 72e4d012801f7a154934e42dc1a6bcdbe1665df9 Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 14 May 2024 21:12:25 -0700
Subject: [PATCH 63/90] fix syntax

---
 oldabe/money_in.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index db9dd67..c23c477 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -15,8 +15,8 @@
     correct_rounding_error, get_git_revision_short_hash
 )
 from .constants import (
-    ABE_ROOT, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE, ADVANCES_FILE,
-    NONATTRIBUTABLE_PAYMENTS_DIR, UNPAYABLE_CONTRIBUTORS_FILE,
+    ABE_ROOT, PAYMENTS_DIR, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE,
+    ADVANCES_FILE, NONATTRIBUTABLE_PAYMENTS_DIR, UNPAYABLE_CONTRIBUTORS_FILE,
     ITEMIZED_PAYMENTS_FILE, PRICE_FILE, VALUATION_FILE, ATTRIBUTIONS_FILE,
     INSTRUMENTS_FILE
 )

From b0559a1cb770c0d3766b7256d88b011a844c7046 Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 14 May 2024 21:13:53 -0700
Subject: [PATCH 64/90] fix syntax

---
 oldabe/constants.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/oldabe/constants.py b/oldabe/constants.py
index 5a4040a..4c6d988 100644
--- a/oldabe/constants.py
+++ b/oldabe/constants.py
@@ -2,12 +2,14 @@
 
 ABE_ROOT = 'abe'
 PAYOUTS_DIR = os.path.join(ABE_ROOT, 'payouts')
-TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt')
-DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt')
-ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt')
+PAYMENTS_DIR = os.path.join(ABE_ROOT, 'payments')
 NONATTRIBUTABLE_PAYMENTS_DIR = os.path.join(
     ABE_ROOT, 'payments', 'nonattributable'
 )
+
+TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt')
+DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt')
+ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt')
 UNPAYABLE_CONTRIBUTORS_FILE = 'unpayable_contributors.txt'
 ITEMIZED_PAYMENTS_FILE = 'itemized_payments.txt'
 PRICE_FILE = 'price.txt'

From 48d9937867068ecb020e96c8ef85b53ac29b751b Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 14 May 2024 21:22:46 -0700
Subject: [PATCH 65/90] use money_in value of ABE_ROOT for constants

---
 oldabe/constants.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oldabe/constants.py b/oldabe/constants.py
index 4c6d988..5a7210d 100644
--- a/oldabe/constants.py
+++ b/oldabe/constants.py
@@ -1,6 +1,6 @@
 import os
 
-ABE_ROOT = 'abe'
+ABE_ROOT = './abe'
 PAYOUTS_DIR = os.path.join(ABE_ROOT, 'payouts')
 PAYMENTS_DIR = os.path.join(ABE_ROOT, 'payments')
 NONATTRIBUTABLE_PAYMENTS_DIR = os.path.join(

From 33d2411be897f6a0c237fd633211b326bb86b311 Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 14 May 2024 23:45:08 -0700
Subject: [PATCH 66/90] standardize file construction for constants

---
 oldabe/constants.py |  8 ++++----
 oldabe/money_in.py  | 39 +++++++++++++--------------------------
 2 files changed, 17 insertions(+), 30 deletions(-)

diff --git a/oldabe/constants.py b/oldabe/constants.py
index 5a7210d..a680e8e 100644
--- a/oldabe/constants.py
+++ b/oldabe/constants.py
@@ -10,9 +10,9 @@
 TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt')
 DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt')
 ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt')
-UNPAYABLE_CONTRIBUTORS_FILE = 'unpayable_contributors.txt'
-ITEMIZED_PAYMENTS_FILE = 'itemized_payments.txt'
-PRICE_FILE = 'price.txt'
-VALUATION_FILE = 'valuation.txt'
+UNPAYABLE_CONTRIBUTORS_FILE = os.path.join(ABE_ROOT, 'unpayable_contributors.txt')
+ITEMIZED_PAYMENTS_FILE = os.path.join(ABE_ROOT, 'itemized_payments.txt')
+PRICE_FILE = os.path.join(ABE_ROOT, 'price.txt')
+VALUATION_FILE = os.path.join(ABE_ROOT, 'valuation.txt')
 ATTRIBUTIONS_FILE = 'attributions.txt'
 INSTRUMENTS_FILE = 'instruments.txt'
diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index c23c477..b57ad7d 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -44,8 +44,7 @@ def read_payment(payment_file, attributable=True):
 
 
 def read_price():
-    price_file = os.path.join(ABE_ROOT, PRICE_FILE)
-    with open(price_file) as f:
+    with open(PRICE_FILE) as f:
         price = f.readline()
         price = Decimal(re.sub("[^0-9.]", "", price))
         return price
@@ -54,8 +53,7 @@ def read_price():
 # note that commas are used as a decimal separator in some languages
 # (e.g. Spain Spanish), so that would need to be handled at some point
 def read_valuation():
-    valuation_file = os.path.join(ABE_ROOT, VALUATION_FILE)
-    with open(valuation_file) as f:
+    with open(VALUATION_FILE) as f:
         valuation = f.readline()
         valuation = Decimal(re.sub("[^0-9.]", "", valuation))
         return valuation
@@ -108,9 +106,8 @@ def find_unprocessed_payments():
     Return type: list of Payment objects
     """
     recorded_payments = set()
-    transactions_file = os.path.join(ABE_ROOT, TRANSACTIONS_FILE)
     try:
-        with open(transactions_file) as f:
+        with open(TRANSACTIONS_FILE) as f:
             for (
                 _email,
                 _amount,
@@ -144,9 +141,8 @@ def get_existing_itemized_payments():
     Reads itemized payment files and returns all itemized payment objects.
     """
     itemized_payments = []
-    itemized_payments_file = os.path.join(ABE_ROOT, ITEMIZED_PAYMENTS_FILE)
     try:
-        with open(itemized_payments_file) as f:
+        with open(ITEMIZED_PAYMENTS_FILE) as f:
             for (
                 email,
                 fee_amount,
@@ -231,32 +227,28 @@ def write_attributions(attributions):
         (email, serialize_proportion(share))
         for email, share in attributions.items()
     ]
-    attributions_file = os.path.join(ABE_ROOT, ATTRIBUTIONS_FILE)
-    with open(attributions_file, 'w') as f:
+    with open(ATTRIBUTIONS_FILE, 'w') as f:
         writer = csv.writer(f)
         for row in attributions:
             writer.writerow(row)
 
 
 def write_append_transactions(transactions):
-    transactions_file = os.path.join(ABE_ROOT, TRANSACTIONS_FILE)
-    with open(transactions_file, 'a') as f:
+    with open(TRANSACTIONS_FILE, 'a') as f:
         writer = csv.writer(f)
         for row in transactions:
             writer.writerow(astuple(row))
 
 
 def write_append_itemized_payments(itemized_payments):
-    itemized_payments_file = os.path.join(ABE_ROOT, ITEMIZED_PAYMENTS_FILE)
-    with open(itemized_payments_file, 'a') as f:
+    with open(ITEMIZED_PAYMENTS_FILE, 'a') as f:
         writer = csv.writer(f)
         for row in itemized_payments:
             writer.writerow(astuple(row))
 
 
 def write_append_advances(advances):
-    advances_file = os.path.join(ABE_ROOT, ADVANCES_FILE)
-    with open(advances_file, 'a') as f:
+    with open(ADVANCES_FILE, 'a') as f:
         writer = csv.writer(f)
         for row in advances:
             writer.writerow(astuple(row))
@@ -264,8 +256,7 @@ def write_append_advances(advances):
 
 def write_valuation(valuation):
     rounded_valuation = f"{valuation:.2f}"
-    valuation_file = os.path.join(ABE_ROOT, VALUATION_FILE)
-    with open(valuation_file, 'w') as f:
+    with open(VALUATION_FILE, 'w') as f:
         writer = csv.writer(f)
         writer.writerow((rounded_valuation,))
 
@@ -305,10 +296,9 @@ def inflate_valuation(valuation, amount):
 
 
 def read_debts():
-    debts_file = os.path.join(ABE_ROOT, DEBTS_FILE)
     debts = []
     try:
-        with open(debts_file) as f:
+        with open(DEBTS_FILE) as f:
             for (
                 email,
                 amount,
@@ -325,10 +315,9 @@ def read_debts():
 
 
 def read_advances(attributions):
-    advances_file = os.path.join(ABE_ROOT, ADVANCES_FILE)
     advances = defaultdict(list)
     try:
-        with open(advances_file) as f:
+        with open(ADVANCES_FILE) as f:
             for (
                 email,
                 amount,
@@ -395,10 +384,9 @@ def get_unpayable_contributors():
     Read the unpayable_contributors file to get the list of contributors who
     are unpayable.
     """
-    unpayable_contributors_file = os.path.join(ABE_ROOT, UNPAYABLE_CONTRIBUTORS_FILE)
     contributors = []
     try:
-        with open(unpayable_contributors_file) as f:
+        with open(UNPAYABLE_CONTRIBUTORS_FILE) as f:
             for contributor in f:
                 contributor = contributor.strip()
                 if contributor:
@@ -436,8 +424,7 @@ def write_debts(processed_debts):
     """
     existing_debts = read_debts()
     processed_debts_hash = {debt.key(): debt for debt in processed_debts}
-    debts_file = os.path.join(ABE_ROOT, DEBTS_FILE)
-    with open(debts_file, 'w') as f:
+    with open(DEBTS_FILE, 'w') as f:
         writer = csv.writer(f)
         for existing_debt in existing_debts:
             # if the existing debt has been processed, write the processed version

From 8910c7192c386b0ef6295c067802094a2d42d59d Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Wed, 15 May 2024 00:10:17 -0700
Subject: [PATCH 67/90] fix white space in test files, fix attributions file
 name, move attributions validation to utils

---
 oldabe/money_in.py            | 16 ++++++----------
 oldabe/utils.py               |  9 +++++++++
 tests/integration/fixtures.py | 18 +++++++++---------
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index b57ad7d..c342774 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -12,7 +12,8 @@
 )
 from .utils import (
     parse_percentage, serialize_proportion, get_rounding_difference,
-    correct_rounding_error, get_git_revision_short_hash
+    correct_rounding_error, get_git_revision_short_hash,
+    assert_attributions_normalized
 )
 from .constants import (
     ABE_ROOT, PAYMENTS_DIR, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE,
@@ -22,10 +23,8 @@
 )
 
 
-ROUNDING_TOLERANCE = Decimal("0.000001")
 ACCOUNTING_ZERO = Decimal("0.01")
 
-
 # TODO standardize the parsing from text into python objects
 # e.g. Decimal and DateTime
 
@@ -69,7 +68,7 @@ def read_attributions(attributions_filename, validate=True):
                 email = email.strip()
                 attributions[email] = parse_percentage(percentage)
     if validate:
-        assert _get_attributions_total(attributions) == Decimal("1")
+        assert_attributions_normalized(attributions)
     return attributions
 
 
@@ -208,10 +207,6 @@ def calculate_incoming_attribution(
         return None
 
 
-def _get_attributions_total(attributions):
-    return sum(attributions.values())
-
-
 def normalize(attributions):
     total_share = sum(share for _, share in attributions.items())
     target_proportion = Decimal("1") / total_share
@@ -221,13 +216,14 @@ def normalize(attributions):
 
 def write_attributions(attributions):
     # don't write attributions if they aren't normalized
-    assert _get_attributions_total(attributions) == Decimal("1")
+    assert_attributions_normalized(attributions)
     # format for output as percentages
     attributions = [
         (email, serialize_proportion(share))
         for email, share in attributions.items()
     ]
-    with open(ATTRIBUTIONS_FILE, 'w') as f:
+    attributions_file = os.path.join(ABE_ROOT, ATTRIBUTIONS_FILE)
+    with open(attributions_file, 'w') as f:
         writer = csv.writer(f)
         for row in attributions:
             writer.writerow(row)
diff --git a/oldabe/utils.py b/oldabe/utils.py
index 6f2b7f1..99a62dd 100644
--- a/oldabe/utils.py
+++ b/oldabe/utils.py
@@ -2,6 +2,7 @@
 import re
 import subprocess
 
+ROUNDING_TOLERANCE = Decimal("0.000001")
 
 def parse_percentage(value):
     """
@@ -82,3 +83,11 @@ def get_git_revision_short_hash() -> str:
         .decode('ascii')
         .strip()
     )
+
+
+def assert_attributions_normalized(attributions):
+    assert _get_attributions_total(attributions) == Decimal("1")
+
+
+def _get_attributions_total(attributions):
+    return sum(attributions.values())
diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py
index 2f6c1e8..7cda069 100644
--- a/tests/integration/fixtures.py
+++ b/tests/integration/fixtures.py
@@ -5,13 +5,13 @@
 def abe_fs(fs):
     fs.create_file("./abe/price.txt", contents="10")
     fs.create_file("./abe/valuation.txt", contents="100000")
-    fs.create_file("./abe/instruments.txt", contents="""
-        old abe,1
-        DIA,5
-        """)
-    fs.create_file("./abe/attributions.txt", contents="""
-       sid,50
-       jair,30
-       ariana,20
-       """)
+    fs.create_file("./abe/instruments.txt", contents=(
+        "old abe,1\n"
+        "DIA,5\n"
+        ))
+    fs.create_file("./abe/attributions.txt", contents=(
+       "sid,50\n"
+       "jair,30\n"
+       "ariana,20\n"
+       ))
     return fs

From 1d6bf138f6beb26764c9d4bfa1efed508509ad5d Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Thu, 16 May 2024 20:38:06 -0700
Subject: [PATCH 68/90] put utils into separate modules

---
 oldabe/accounting_utils.py          |  33 ++++++++
 oldabe/git.py                       |   9 ++
 oldabe/money_in.py                  |   7 +-
 oldabe/{utils.py => parsing.py}     |  42 ----------
 tests/unit/accounting_utils_test.py |  48 +++++++++++
 tests/unit/money_in_test.py         |   3 +-
 tests/unit/parsing_test.py          |  61 ++++++++++++++
 tests/unit/utils_test.py            | 126 ----------------------------
 8 files changed, 156 insertions(+), 173 deletions(-)
 create mode 100644 oldabe/accounting_utils.py
 create mode 100644 oldabe/git.py
 rename oldabe/{utils.py => parsing.py} (53%)
 create mode 100644 tests/unit/accounting_utils_test.py
 create mode 100644 tests/unit/parsing_test.py
 delete mode 100644 tests/unit/utils_test.py

diff --git a/oldabe/accounting_utils.py b/oldabe/accounting_utils.py
new file mode 100644
index 0000000..7275b67
--- /dev/null
+++ b/oldabe/accounting_utils.py
@@ -0,0 +1,33 @@
+from decimal import Decimal
+
+ROUNDING_TOLERANCE = Decimal("0.000001")
+
+
+def get_rounding_difference(attributions):
+    """
+    Get the difference of the total of the attributions from 1, which is
+    expected to occur due to finite precision. If the difference exceeds the
+    expected error tolerance, an error is signaled.
+    """
+    total = _get_attributions_total(attributions)
+    difference = total - Decimal("1")
+    assert abs(difference) <= ROUNDING_TOLERANCE
+    return difference
+
+
+def correct_rounding_error(attributions, incoming_attribution):
+    """Due to finite precision, the Decimal module will round up or down
+    on the last decimal place. This could result in the aggregate value not
+    quite totaling to 1. This corrects that total by either adding or
+    subtracting the difference from the incoming attribution (by convention).
+    """
+    difference = get_rounding_difference(attributions)
+    attributions[incoming_attribution.email] -= difference
+
+
+def assert_attributions_normalized(attributions):
+    assert _get_attributions_total(attributions) == Decimal("1")
+
+
+def _get_attributions_total(attributions):
+    return sum(attributions.values())
diff --git a/oldabe/git.py b/oldabe/git.py
new file mode 100644
index 0000000..3c6638e
--- /dev/null
+++ b/oldabe/git.py
@@ -0,0 +1,9 @@
+import subprocess
+
+def get_git_revision_short_hash() -> str:
+    """From https://stackoverflow.com/a/21901260"""
+    return (
+        subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'])
+        .decode('ascii')
+        .strip()
+    )
diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index c342774..01713c6 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -10,9 +10,10 @@
 from .models import (
     Advance, Attribution, Debt, Payment, ItemizedPayment, Transaction
 )
-from .utils import (
-    parse_percentage, serialize_proportion, get_rounding_difference,
-    correct_rounding_error, get_git_revision_short_hash,
+from .parsing import parse_percentage, serialize_proportion
+from .git import get_git_revision_short_hash
+from .accounting_utils import (
+    get_rounding_difference, correct_rounding_error,
     assert_attributions_normalized
 )
 from .constants import (
diff --git a/oldabe/utils.py b/oldabe/parsing.py
similarity index 53%
rename from oldabe/utils.py
rename to oldabe/parsing.py
index 99a62dd..c79ddc8 100644
--- a/oldabe/utils.py
+++ b/oldabe/parsing.py
@@ -1,8 +1,5 @@
 from decimal import Decimal
 import re
-import subprocess
-
-ROUNDING_TOLERANCE = Decimal("0.000001")
 
 def parse_percentage(value):
     """
@@ -52,42 +49,3 @@ def serialize_proportion(value):
         # remove decimal point if whole number
         value = re.sub(r"\.$", r"", value)
     return value
-
-
-def get_rounding_difference(attributions):
-    """
-    Get the difference of the total of the attributions from 1, which is
-    expected to occur due to finite precision. If the difference exceeds the
-    expected error tolerance, an error is signaled.
-    """
-    total = _get_attributions_total(attributions)
-    difference = total - Decimal("1")
-    assert abs(difference) <= ROUNDING_TOLERANCE
-    return difference
-
-
-def correct_rounding_error(attributions, incoming_attribution):
-    """Due to finite precision, the Decimal module will round up or down
-    on the last decimal place. This could result in the aggregate value not
-    quite totaling to 1. This corrects that total by either adding or
-    subtracting the difference from the incoming attribution (by convention).
-    """
-    difference = get_rounding_difference(attributions)
-    attributions[incoming_attribution.email] -= difference
-
-
-def get_git_revision_short_hash() -> str:
-    """From https://stackoverflow.com/a/21901260"""
-    return (
-        subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'])
-        .decode('ascii')
-        .strip()
-    )
-
-
-def assert_attributions_normalized(attributions):
-    assert _get_attributions_total(attributions) == Decimal("1")
-
-
-def _get_attributions_total(attributions):
-    return sum(attributions.values())
diff --git a/tests/unit/accounting_utils_test.py b/tests/unit/accounting_utils_test.py
new file mode 100644
index 0000000..8fde629
--- /dev/null
+++ b/tests/unit/accounting_utils_test.py
@@ -0,0 +1,48 @@
+import pytest
+from decimal import Decimal
+from oldabe.accounting_utils import (
+    get_rounding_difference,
+    correct_rounding_error,
+    ROUNDING_TOLERANCE,
+)
+from .fixtures import normalized_attributions  # noqa
+
+
+class TestGetRoundingDifference:
+    def test_attributions_already_one(self, normalized_attributions):
+        attributions = normalized_attributions
+        test_diff = Decimal("0")
+        difference = get_rounding_difference(attributions)
+        assert difference == test_diff
+
+    def test_attributions_exceed_below_tolerance(
+        self, normalized_attributions
+    ):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE / Decimal("2")
+        attributions['c@d.com'] = test_diff
+        difference = get_rounding_difference(attributions)
+        assert difference == test_diff
+
+    def test_attributions_exceed_above_tolerance(
+        self, normalized_attributions
+    ):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE * Decimal("2")
+        attributions['c@d.com'] = test_diff
+        with pytest.raises(AssertionError):
+            _ = get_rounding_difference(attributions)
+
+    def test_attributions_exceed_by_tolerance(self, normalized_attributions):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE
+        attributions['c@d.com'] = test_diff
+        difference = get_rounding_difference(attributions)
+        assert difference == test_diff
+
+    def test_attributions_below_one(self, normalized_attributions):
+        attributions = normalized_attributions
+        test_diff = ROUNDING_TOLERANCE / Decimal("2")
+        attributions['b@c.com'] -= test_diff
+        difference = get_rounding_difference(attributions)
+        assert difference == -test_diff
diff --git a/tests/unit/money_in_test.py b/tests/unit/money_in_test.py
index 44120c7..0984b6f 100644
--- a/tests/unit/money_in_test.py
+++ b/tests/unit/money_in_test.py
@@ -4,12 +4,11 @@
     total_amount_paid_to_project,
     calculate_incoming_attribution,
     generate_transactions,
-    ROUNDING_TOLERANCE,
     renormalize,
     inflate_valuation,
     process_payments,
 )
-from oldabe.utils import correct_rounding_error
+from oldabe.accounting_utils import correct_rounding_error
 from oldabe.models import Attribution, Payment
 import pytest
 from unittest.mock import patch
diff --git a/tests/unit/parsing_test.py b/tests/unit/parsing_test.py
new file mode 100644
index 0000000..31328e5
--- /dev/null
+++ b/tests/unit/parsing_test.py
@@ -0,0 +1,61 @@
+import pytest
+from decimal import Decimal
+from oldabe.parsing import (
+    parse_percentage,
+    serialize_proportion,
+)
+
+
+class TestParsePercentage:
+    def test_an_integer(self):
+        assert parse_percentage('75') == Decimal('0.75')
+
+    def test_non_integer_greater_than_1(self):
+        assert parse_percentage('75.334455') == Decimal('0.75334455')
+
+    def test_non_integer_less_than_1(self):
+        assert parse_percentage('0.334455') == Decimal('0.00334455')
+
+    def test_decimal_places_at_precision_context(self):
+        assert parse_percentage('5.1234567891') == Decimal('0.051234567891')
+
+    def test_very_small_number(self):
+        assert parse_percentage('0.000001') == Decimal('0.00000001')
+
+    def test_0(self):
+        assert parse_percentage('0') == Decimal('0')
+
+    def test_0_point_0(self):
+        assert parse_percentage('0.0') == Decimal('0')
+
+    def test_100(self):
+        assert parse_percentage('100') == Decimal('1')
+
+    def test_100_point_0(self):
+        assert parse_percentage('100.0') == Decimal('1')
+
+
+class TestSerializeProportion:
+    def test_0(self):
+        assert serialize_proportion(Decimal('0')) == '0'
+
+    def test_0_point_0(self):
+        assert serialize_proportion(Decimal('0.0')) == '0'
+
+    def test_1(self):
+        assert serialize_proportion(Decimal('1')) == '100'
+
+    def test_1_point_0(self):
+        assert serialize_proportion(Decimal('1.0')) == '100'
+
+    def test_almost_1(self):
+        assert serialize_proportion(Decimal('0.9523452')) == '95.23452'
+
+    def test_low_precision_number(self):
+        assert serialize_proportion(Decimal('0.2')) == '20'
+
+    def test_very_small_number(self):
+        assert serialize_proportion(Decimal('0.0000002')) == '0.00002'
+
+    def test_decimal_places_at_precision_context(self):
+        assert serialize_proportion(Decimal('0.1234567891')) == '12.34567891'
diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py
deleted file mode 100644
index 008ec97..0000000
--- a/tests/unit/utils_test.py
+++ /dev/null
@@ -1,126 +0,0 @@
-from decimal import Decimal
-from oldabe.money_in import (
-    calculate_incoming_investment,
-    total_amount_paid_to_project,
-    calculate_incoming_attribution,
-    generate_transactions,
-    ROUNDING_TOLERANCE,
-    renormalize,
-    inflate_valuation,
-    process_payments,
-)
-from oldabe.utils import (
-    parse_percentage,
-    serialize_proportion,
-    get_rounding_difference,
-    correct_rounding_error,
-)
-from oldabe.models import Attribution, Payment
-import pytest
-from unittest.mock import patch
-from .utils import call_sequence
-from .fixtures import (
-    instruments,
-    normalized_attributions,
-    excess_attributions,
-    shortfall_attributions,
-    empty_attributions,
-    single_contributor_attributions,
-    itemized_payments,
-    new_itemized_payments,
-)  # noqa
-
-
-class TestParsePercentage:
-    def test_an_integer(self):
-        assert parse_percentage('75') == Decimal('0.75')
-
-    def test_non_integer_greater_than_1(self):
-        assert parse_percentage('75.334455') == Decimal('0.75334455')
-
-    def test_non_integer_less_than_1(self):
-        assert parse_percentage('0.334455') == Decimal('0.00334455')
-
-    def test_decimal_places_at_precision_context(self):
-        assert parse_percentage('5.1234567891') == Decimal('0.051234567891')
-
-    def test_very_small_number(self):
-        assert parse_percentage('0.000001') == Decimal('0.00000001')
-
-    def test_0(self):
-        assert parse_percentage('0') == Decimal('0')
-
-    def test_0_point_0(self):
-        assert parse_percentage('0.0') == Decimal('0')
-
-    def test_100(self):
-        assert parse_percentage('100') == Decimal('1')
-
-    def test_100_point_0(self):
-        assert parse_percentage('100.0') == Decimal('1')
-
-
-class TestSerializeProportion:
-    def test_0(self):
-        assert serialize_proportion(Decimal('0')) == '0'
-
-    def test_0_point_0(self):
-        assert serialize_proportion(Decimal('0.0')) == '0'
-
-    def test_1(self):
-        assert serialize_proportion(Decimal('1')) == '100'
-
-    def test_1_point_0(self):
-        assert serialize_proportion(Decimal('1.0')) == '100'
-
-    def test_almost_1(self):
-        assert serialize_proportion(Decimal('0.9523452')) == '95.23452'
-
-    def test_low_precision_number(self):
-        assert serialize_proportion(Decimal('0.2')) == '20'
-
-    def test_very_small_number(self):
-        assert serialize_proportion(Decimal('0.0000002')) == '0.00002'
-
-    def test_decimal_places_at_precision_context(self):
-        assert serialize_proportion(Decimal('0.1234567891')) == '12.34567891'
-
-
-class TestGetRoundingDifference:
-    def test_attributions_already_one(self, normalized_attributions):
-        attributions = normalized_attributions
-        test_diff = Decimal("0")
-        difference = get_rounding_difference(attributions)
-        assert difference == test_diff
-
-    def test_attributions_exceed_below_tolerance(
-        self, normalized_attributions
-    ):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE / Decimal("2")
-        attributions['c@d.com'] = test_diff
-        difference = get_rounding_difference(attributions)
-        assert difference == test_diff
-
-    def test_attributions_exceed_above_tolerance(
-        self, normalized_attributions
-    ):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE * Decimal("2")
-        attributions['c@d.com'] = test_diff
-        with pytest.raises(AssertionError):
-            _ = get_rounding_difference(attributions)
-
-    def test_attributions_exceed_by_tolerance(self, normalized_attributions):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE
-        attributions['c@d.com'] = test_diff
-        difference = get_rounding_difference(attributions)
-        assert difference == test_diff
-
-    def test_attributions_below_one(self, normalized_attributions):
-        attributions = normalized_attributions
-        test_diff = ROUNDING_TOLERANCE / Decimal("2")
-        attributions['b@c.com'] -= test_diff
-        difference = get_rounding_difference(attributions)
-        assert difference == -test_diff

From 3f59bbc4b7f9a902cd159cf1313e8d83f77232a4 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Thu, 20 Jun 2024 19:37:16 -0700
Subject: [PATCH 69/90] Update development docs

---
 CONTRIBUTING.md | 34 +++++++++++++++++++++++++++++++---
 DEV.md          | 22 ----------------------
 2 files changed, 31 insertions(+), 25 deletions(-)
 delete mode 100644 DEV.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e65f135..d0a8f45 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,13 +6,37 @@ You could do this in a virtual environment or at the system / user level if you
 $ make build
 ```
 
+You may also need to run:
+
+```
+$ make build-for-test
+```
+
+Together, these should ensure that you have all development dependencies so that the rest of the `make` targets should work.
+
 # Running Tests
 
+## Unit Tests
+
 ```
 $ make test
 ```
 
-## Debugging
+## Integration Tests
+
+```
+$ pytest tests/integration
+```
+
+NOTE: We do have a `make` target for this:
+
+```
+$ make test-integration
+```
+
+But this does not work even though it runs a command identical to the above, due to some weird dependency issue with pytest. For now, just use the earlier command in the shell directly.
+
+# Debugging
 
 To debug a test execution using a step debugger, put this in the body of the test you'd like to debug:
 
@@ -20,6 +44,12 @@ To debug a test execution using a step debugger, put this in the body of the tes
 import pudb; pudb.set_trace()
 ```
 
+or simply,
+
+```
+import pudb; pu.db
+```
+
 Now, when you run `make test` (for example), it will put you in the debugger, allowing you to step through the execution of the code. Here are some things you can do:
 
 * `n` - next
@@ -32,8 +62,6 @@ Now, when you run `make test` (for example), it will put you in the debugger, al
 
 There's more handy stuff that you can do like setting breakpoints and visiting other modules. Press `?` to see all the options.
 
-*Note*: the official docs for `pudb` say that we could also use `pu.db()` instead of `pudb.set_trace()`, but this doesn't seem to work.
-
 # Linting
 
 Linter:
diff --git a/DEV.md b/DEV.md
deleted file mode 100644
index c30c670..0000000
--- a/DEV.md
+++ /dev/null
@@ -1,22 +0,0 @@
-# Install for Development
-
-```
-make build
-```
-
-# Running Tests
-
-Refer to the Makefile for all things related to the development workflow. E.g.
-
-```
-make test
-```
-
-# Drafting a Release
-
-```
-# increment the version
-bump2version [major|minor|patch]
-# push new release tag upstream
-git push --follow-tags
-```

From 575576c4cb940cea336f0b7632e61293e1163851 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Thu, 11 Jul 2024 20:56:01 -0700
Subject: [PATCH 70/90] move logic into a function for integration testing
 purposes

---
 oldabe/money_out.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/oldabe/money_out.py b/oldabe/money_out.py
index 542ed8a..293d222 100755
--- a/oldabe/money_out.py
+++ b/oldabe/money_out.py
@@ -148,13 +148,10 @@ def combined_message(balances_message, debts_message, advances_message):
     return "\r\n".join(line.strip() for line in message.split('\n')).strip()
 
 
-def main():
-    # set decimal precision at 10 to ensure
-    # that it is the same everywhere
-    # and large enough to represent a sufficiently
-    # large number of contributors
-    getcontext().prec = 10
-
+def compile_outstanding_balances():
+    """ Read all accounting records and determine the total outstanding
+    balances, debts, and advances for each contributor.
+    """
     owed = read_transaction_amounts()
     paid = read_payout_amounts()
     balances = compute_balances(owed, paid)
@@ -163,7 +160,16 @@ def main():
     debts_message = prepare_debts_message(outstanding_debts)
     advances = read_advance_amounts()
     advances_message = prepare_advances_message(advances)
-    print(combined_message(balances_message, debts_message, advances_message))
+    return combined_message(balances_message, debts_message, advances_message)
+
+
+def main():
+    # set decimal precision at 10 to ensure
+    # that it is the same everywhere
+    # and large enough to represent a sufficiently
+    # large number of contributors
+    getcontext().prec = 10
+    print(compile_outstanding_balances())
 
 
 if __name__ == "__main__":

From 907cc3270a88e01a0b59201503cf148325a37a2c Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Thu, 11 Jul 2024 20:56:36 -0700
Subject: [PATCH 71/90] Expand integration tests to include "money out"
 functionality

---
 .../{money_in_test.py => old_abe_test.py}     | 101 +++++++++++++++++-
 1 file changed, 98 insertions(+), 3 deletions(-)
 rename tests/integration/{money_in_test.py => old_abe_test.py} (69%)

diff --git a/tests/integration/money_in_test.py b/tests/integration/old_abe_test.py
similarity index 69%
rename from tests/integration/money_in_test.py
rename to tests/integration/old_abe_test.py
index 73b2577..429c6a0 100644
--- a/tests/integration/money_in_test.py
+++ b/tests/integration/old_abe_test.py
@@ -6,19 +6,29 @@
 from oldabe.money_in import (
     process_payments_and_record_updates,
 )
+from oldabe.money_out import (
+    compile_outstanding_balances,
+)
 from .fixtures import abe_fs
 import os
 
 
 class TestNoPayments:
+
     @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
     def test_no_transactions_generated(self, mock_git_rev, abe_fs):
-        with open('abe/price.txt') as f:
-            assert f.read() == "10"
         process_payments_and_record_updates()
         with open('./abe/transactions.txt') as f:
             assert f.read() == ""
 
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
+        process_payments_and_record_updates()
+        message = compile_outstanding_balances()
+        assert "There are no outstanding (payable) balances." in message
+        assert "There are no outstanding (unpayable) debts." in message
+        assert "There are no advances." in message
+
 # TODO: in some cases even though the value is e.g. 1,
 # it's writing out 3 decimal places, like 1.000. We should
 # figure out why this is happening (and whether it's OK)
@@ -26,6 +36,7 @@ def test_no_transactions_generated(self, mock_git_rev, abe_fs):
 
 
 class TestPaymentAbovePrice:
+
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
     @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
     def test_generates_transactions(self, mock_git_rev, abe_fs):
@@ -61,8 +72,25 @@ def test_dilutes_attributions(self, mock_git_rev, abe_fs):
                     "sam,8.5\n"
                 )
 
+    @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
+        with localcontext() as context:
+            context.prec = 2
+            amount = 100
+            abe_fs.create_file("./abe/payments/1.txt",
+                               contents=f"sam,036eaf6,{amount},dummydate")
+            process_payments_and_record_updates()
+
+            message = compile_outstanding_balances()
+
+            assert ("| Name | Balance |\r\n"
+                    "| ---- | --- |\r\n"
+                    "old abe | 1.00\r\n") in message
+
 
 class TestPaymentBelowPrice:
+
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
     @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
     def test_generates_transactions(self, mock_git_rev, abe_fs):
@@ -81,14 +109,30 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
                     "ariana,0.19,1.txt,abcd123,1985-10-26 01:24:00\n"
                 )
 
+    @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
+        with localcontext() as context:
+            context.prec = 2
+            amount = 1
+            abe_fs.create_file("./abe/payments/1.txt",
+                               contents=f"sam,036eaf6,{amount},dummydate")
+            process_payments_and_record_updates()
+            message = compile_outstanding_balances()
+
+            assert ("| Name | Balance |\r\n"
+                    "| ---- | --- |\r\n"
+                    "old abe | 0.01\r\n") in message
+
 
 class TestNonAttributablePayment:
+
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
     @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
     def test_does_not_dilute_attributions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
-            amount = 10000
+            amount = 100
             abe_fs.create_file("./abe/payments/nonattributable/1.txt",
                                contents=f"sam,036eaf6,{amount},dummydate")
             process_payments_and_record_updates()
@@ -99,6 +143,39 @@ def test_does_not_dilute_attributions(self, mock_git_rev, abe_fs):
                     "ariana,20\n"
                 )
 
+    @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_generates_transactions(self, mock_git_rev, abe_fs):
+        with localcontext() as context:
+            context.prec = 2
+            amount = 100
+            abe_fs.create_file("./abe/payments/1.txt",
+                               contents=f"sam,036eaf6,{amount},dummydate")
+            process_payments_and_record_updates()
+            with open('./abe/transactions.txt') as f:
+                assert f.read() == (
+                    "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "sid,47,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "jair,28,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "ariana,19,1.txt,abcd123,1985-10-26 01:24:00\n"
+                )
+
+    @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
+        with localcontext() as context:
+            context.prec = 2
+            amount = 100
+            abe_fs.create_file("./abe/payments/nonattributable/1.txt",
+                               contents=f"sam,036eaf6,{amount},dummydate")
+            process_payments_and_record_updates()
+            message = compile_outstanding_balances()
+
+            assert ("| Name | Balance |\r\n"
+                    "| ---- | --- |\r\n"
+                    "old abe | 1.00\r\n") in message
+
 
 class TestUnpayableContributor:
 
@@ -145,6 +222,15 @@ def test_records_advances(self, mock_git_rev, abe_fs):
                 "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n"
             )
 
+    @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
+        self._call(abe_fs)
+        message = compile_outstanding_balances()
+        assert ("| Name | Debt |\r\n"
+                "| ---- | --- |\r\n"
+                "ariana | 19.00\r\n") in message
+
 
 class TestUnpayableContributorBecomesPayable:
 
@@ -218,3 +304,12 @@ def test_advances(self, mock_git_rev, abe_fs):
                 "jair,5.4,2.txt,abcd123,1985-10-26 01:24:00\n"
                 "ariana,3.6,2.txt,abcd123,1985-10-26 01:24:00\n"
             )
+
+    @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
+    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
+        self._call(abe_fs)
+        message = compile_outstanding_balances()
+        assert ("| Name | Debt |\r\n"
+                "| ---- | --- |\r\n"
+                "ariana | 0.00\r\n") in message

From d2447be93cd3b15305641baa7f5121efa161dea8 Mon Sep 17 00:00:00 2001
From: Jair Trejo 
Date: Sat, 24 Aug 2024 13:41:47 -0700
Subject: [PATCH 72/90] Jair's refactor

- Move I/O into repositories
- Create Tally concept to sum lists of stuff
- Functional re-organizing
---
 oldabe/accounting_utils.py        |   1 +
 oldabe/constants.py               |   7 +-
 oldabe/distribution.py            |  48 ++
 oldabe/git.py                     |   3 +
 oldabe/models.py                  |  80 ++-
 oldabe/money_in.py                | 784 +++++++++++-------------------
 oldabe/money_out.py               |  80 +--
 oldabe/repos.py                   | 163 +++++++
 oldabe/tally.py                   |  30 ++
 tests/integration/old_abe_test.py |  78 ++-
 10 files changed, 650 insertions(+), 624 deletions(-)
 create mode 100644 oldabe/distribution.py
 create mode 100644 oldabe/repos.py
 create mode 100644 oldabe/tally.py

diff --git a/oldabe/accounting_utils.py b/oldabe/accounting_utils.py
index 7275b67..6d6672d 100644
--- a/oldabe/accounting_utils.py
+++ b/oldabe/accounting_utils.py
@@ -26,6 +26,7 @@ def correct_rounding_error(attributions, incoming_attribution):
 
 
 def assert_attributions_normalized(attributions):
+    print(_get_attributions_total(attributions))
     assert _get_attributions_total(attributions) == Decimal("1")
 
 
diff --git a/oldabe/constants.py b/oldabe/constants.py
index a680e8e..d943070 100644
--- a/oldabe/constants.py
+++ b/oldabe/constants.py
@@ -1,4 +1,7 @@
 import os
+from decimal import Decimal
+
+ACCOUNTING_ZERO = Decimal("0.01")
 
 ABE_ROOT = './abe'
 PAYOUTS_DIR = os.path.join(ABE_ROOT, 'payouts')
@@ -14,5 +17,5 @@
 ITEMIZED_PAYMENTS_FILE = os.path.join(ABE_ROOT, 'itemized_payments.txt')
 PRICE_FILE = os.path.join(ABE_ROOT, 'price.txt')
 VALUATION_FILE = os.path.join(ABE_ROOT, 'valuation.txt')
-ATTRIBUTIONS_FILE = 'attributions.txt'
-INSTRUMENTS_FILE = 'instruments.txt'
+ATTRIBUTIONS_FILE = os.path.join(ABE_ROOT, 'attributions.txt')
+INSTRUMENTS_FILE = os.path.join(ABE_ROOT, 'instruments.txt')
diff --git a/oldabe/distribution.py b/oldabe/distribution.py
new file mode 100644
index 0000000..6f1bb12
--- /dev/null
+++ b/oldabe/distribution.py
@@ -0,0 +1,48 @@
+from decimal import Decimal
+from typing import Set
+
+
+class Distribution(dict[str | None, Decimal]):
+    """
+    A dictionary of shareholders to proportions
+
+    None can be used as a shareholder to omit a percentage from distribution
+    and enumeration.
+    """
+
+    def _normalized(self) -> "Distribution":
+        """
+        Return a distribution with the same proportions that adds up to 1
+        """
+        total = sum(self.values())
+
+        return Distribution(
+            {
+                shareholder: share * (Decimal("1") / total)
+                for shareholder, share in self.items()
+            }
+        )
+
+    def without(self, exclude: Set[str]) -> "Distribution":
+        """
+        Return a distribution without the shareholders in exclude
+
+        Other shareholders retain their relative proportions.
+        """
+        return Distribution(
+            {
+                shareholder: share
+                for shareholder, share in self.items()
+                if shareholder not in exclude
+            }
+        )
+
+    def distribute(self, amount: Decimal) -> dict[str, Decimal]:
+        """
+        Distribute an amount amongst shareholders
+        """
+        return {
+            shareholder: amount * share
+            for shareholder, share in self._normalized().items()
+            if shareholder is not None
+        }
diff --git a/oldabe/git.py b/oldabe/git.py
index 3c6638e..1bae937 100644
--- a/oldabe/git.py
+++ b/oldabe/git.py
@@ -1,5 +1,8 @@
 import subprocess
+from functools import cache
 
+
+@cache
 def get_git_revision_short_hash() -> str:
     """From https://stackoverflow.com/a/21901260"""
     return (
diff --git a/oldabe/models.py b/oldabe/models.py
index 59c51df..d8b0904 100644
--- a/oldabe/models.py
+++ b/oldabe/models.py
@@ -1,26 +1,42 @@
-from datetime import datetime
 from dataclasses import dataclass, field
+from datetime import datetime
 from decimal import Decimal
 
+from oldabe.git import get_git_revision_short_hash
+from oldabe.parsing import parse_percentage
+
+
+# Wrapping so that tests can mock it
+def default_commit_hash():
+    return get_git_revision_short_hash()
+
 
 @dataclass
 class Transaction:
-    email: str = None
-    amount: Decimal = 0
-    payment_file: str = None
-    commit_hash: str = None
+    email: str
+    amount: Decimal
+    payment_file: str
+    commit_hash: str = field(default_factory=lambda: default_commit_hash())
+    created_at: datetime = field(default_factory=datetime.utcnow)
+
+
+@dataclass
+class Payout:
+    name: str
+    email: str
+    amount: Decimal
     created_at: datetime = field(default_factory=datetime.utcnow)
 
 
 @dataclass
 class Debt:
-    email: str = None
-    amount: Decimal = None
+    email: str
+    amount: Decimal
     # amount_paid is a running tally of how much of this debt has been paid
     # in future will link to Transaction objects instead
-    amount_paid: Decimal = 0
-    payment_file: str = None
-    commit_hash: str = None
+    amount_paid: Decimal
+    payment_file: str
+    commit_hash: str = field(default_factory=lambda: default_commit_hash())
     created_at: datetime = field(default_factory=datetime.utcnow)
 
     def key(self):
@@ -32,6 +48,12 @@ def is_fulfilled(self):
     def amount_remaining(self):
         return self.amount - self.amount_paid
 
+# These are not recorded (yet?), they just represent an intention to record a payment
+@dataclass
+class DebtPayment:
+    debt: Debt
+    amount: Decimal
+
 
 # Individual advances can have a positive or negative amount (to
 # indicate an actual advance payment, or a drawn down advance).
@@ -39,19 +61,22 @@ def amount_remaining(self):
 # sum all of their existing Advance objects.
 @dataclass
 class Advance:
-    email: str = None
-    amount: Decimal = 0
-    payment_file: str = None
-    commit_hash: str = None
+    email: str
+    amount: Decimal
+    payment_file: str
+    commit_hash: str = field(default_factory=lambda: default_commit_hash())
     created_at: datetime = field(default_factory=datetime.utcnow)
 
 
 @dataclass
 class Payment:
-    email: str = None
-    amount: Decimal = 0
+    email: str
+    name: str
+    amount: Decimal
+    # Should move this, but it will invalidate existing files
+    created_at: datetime = field(default_factory=datetime.utcnow)
     attributable: bool = True
-    file: str = None
+    file: str = ''
 
 
 # ItemizedPayment acts as a proxy for a Payment object that keeps track
@@ -60,17 +85,18 @@ class Payment:
 # avoid mutating Payment records.
 @dataclass
 class ItemizedPayment:
-    email: str = None
-    fee_amount: Decimal = 0  # instruments
-    project_amount: Decimal = 0  # attributions
-    attributable: bool = True
-    payment_file: str = (
-        None  # acts like a foreign key to original payment object
-    )
+    email: str
+    fee_amount: Decimal # instruments
+    project_amount: Decimal # attributions
+    attributable: bool
+    payment_file: str # acts like a foreign key to original payment object
 
 
 @dataclass
 class Attribution:
-    email: str = None
-    share: Decimal = 0
-    dilutable: bool = True
+    email: str
+    share: str
+
+    @property
+    def decimal_share(self):
+        return parse_percentage(self.share)
diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index 01713c6..ede7f4e 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -1,49 +1,28 @@
 #!/usr/bin/env python
 
 import csv
-from collections import defaultdict
-from decimal import Decimal, getcontext
-from dataclasses import astuple
+import dataclasses
 import re
-import os
-import subprocess
-from .models import (
-    Advance, Attribution, Debt, Payment, ItemizedPayment, Transaction
-)
-from .parsing import parse_percentage, serialize_proportion
-from .git import get_git_revision_short_hash
-from .accounting_utils import (
-    get_rounding_difference, correct_rounding_error,
-    assert_attributions_normalized
-)
-from .constants import (
-    ABE_ROOT, PAYMENTS_DIR, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE,
-    ADVANCES_FILE, NONATTRIBUTABLE_PAYMENTS_DIR, UNPAYABLE_CONTRIBUTORS_FILE,
-    ITEMIZED_PAYMENTS_FILE, PRICE_FILE, VALUATION_FILE, ATTRIBUTIONS_FILE,
-    INSTRUMENTS_FILE
-)
-
-
-ACCOUNTING_ZERO = Decimal("0.01")
-
-# TODO standardize the parsing from text into python objects
-# e.g. Decimal and DateTime
-
-def read_payment(payment_file, attributable=True):
-    """
-    Reads a payment file and uses the contents to create a Payment object.
-    """
-    payments_dir = (
-        PAYMENTS_DIR if attributable else NONATTRIBUTABLE_PAYMENTS_DIR
-    )
-    with open(os.path.join(payments_dir, payment_file)) as f:
-        for row in csv.reader(f, skipinitialspace=True):
-            name, _email, amount, _date = row
-            amount = re.sub("[^0-9.]", "", amount)
-            return Payment(name, Decimal(amount), attributable, payment_file)
-
-
-def read_price():
+from dataclasses import astuple, replace
+from decimal import Decimal, getcontext
+from itertools import accumulate
+from typing import Iterable, List, Set, Tuple
+
+from .accounting_utils import (assert_attributions_normalized,
+                               correct_rounding_error)
+from .constants import (ACCOUNTING_ZERO, ATTRIBUTIONS_FILE, DEBTS_FILE,
+                        PRICE_FILE, VALUATION_FILE)
+from .distribution import Distribution
+from .models import (Advance, Attribution, Debt, DebtPayment, ItemizedPayment,
+                     Payment, Transaction)
+from .parsing import serialize_proportion
+from .repos import (AdvancesRepo, AllPaymentsRepo, AttributionsRepo, DebtsRepo,
+                    InstrumentsRepo, ItemizedPaymentsRepo, TransactionsRepo,
+                    UnpayableContributorsRepo)
+from .tally import Tally
+
+
+def read_price() -> Decimal:
     with open(PRICE_FILE) as f:
         price = f.readline()
         price = Decimal(re.sub("[^0-9.]", "", price))
@@ -52,145 +31,261 @@ def read_price():
 
 # note that commas are used as a decimal separator in some languages
 # (e.g. Spain Spanish), so that would need to be handled at some point
-def read_valuation():
+def read_valuation() -> Decimal:
     with open(VALUATION_FILE) as f:
         valuation = f.readline()
         valuation = Decimal(re.sub("[^0-9.]", "", valuation))
         return valuation
 
 
-def read_attributions(attributions_filename, validate=True):
-    attributions = {}
-    attributions_file = os.path.join(ABE_ROOT, attributions_filename)
-    with open(attributions_file) as f:
-        for row in csv.reader(f):
-            if row and row[0].strip():
-                email, percentage = row
-                email = email.strip()
-                attributions[email] = parse_percentage(percentage)
-    if validate:
-        assert_attributions_normalized(attributions)
-    return attributions
+def write_valuation(valuation):
+    rounded_valuation = f"{valuation:.2f}"
+    with open(VALUATION_FILE, "w") as f:
+        writer = csv.writer(f)
+        writer.writerow((rounded_valuation,))
+
+
+def write_attributions(attributions):
+    # don't write attributions if they aren't normalized
+    assert_attributions_normalized(attributions)
+    # format for output as percentages
+    attributions = [
+        (email, serialize_proportion(share))
+        for email, share in attributions.items()
+    ]
+    with open(ATTRIBUTIONS_FILE, "w") as f:
+        writer = csv.writer(f)
+        for row in attributions:
+            writer.writerow(row)
 
 
-def get_all_payments():
+def write_debts(new_debts, debt_payments):
     """
-    Reads payment files and returns all existing payment objects.
+    1. Build a hash of all the processed debts, generating an id for each
+       (based on email and payment file).
+    2. read the existing debts file, row by row.
+    3. if the debt in the row is in the "processed" hash, then write the
+       processed version instead of the input version and remove it from the
+       hash, otherwise write the input version.
+    4. write the debts that remain in the processed hash.
     """
-    try:
-        payments = [
-            read_payment(f, attributable=True)
-        for f in os.listdir(PAYMENTS_DIR)
-        if not os.path.isdir(os.path.join(PAYMENTS_DIR, f))
-        ]
-    except FileNotFoundError:
-        payments = []
-    try:
-        payments += [
-            read_payment(f, attributable=False)
-            for f in os.listdir(NONATTRIBUTABLE_PAYMENTS_DIR)
-            if not os.path.isdir(os.path.join(NONATTRIBUTABLE_PAYMENTS_DIR, f))
-        ]
-    except FileNotFoundError:
-        pass
-    return payments
+    print(new_debts, debt_payments)
+    total_debt_payments = Tally(
+        (dp.debt.key(), dp.amount) for dp in debt_payments
+    )
+    replacement = [
+        (
+            dataclasses.replace(
+                debt,
+                amount_paid=debt.amount_paid + total_debt_payments[debt.key()],
+            )
+            if debt.key() in total_debt_payments
+            else debt
+        )
+        for debt in [*DebtsRepo(), *new_debts]
+    ]
+    print(total_debt_payments, list(DebtsRepo()), replacement)
 
+    with open(DEBTS_FILE, "w") as f:
+        writer = csv.writer(f)
+        for debt in replacement:
+            writer.writerow(astuple(debt))
 
-def find_unprocessed_payments():
-    """
-    1. Read the transactions file to find out which payments are already
-       recorded as transactions
-    2. Read the payments folder to get all payments, as Payment objects
-    3. Return those which haven't been recorded in a transaction
 
-    Return type: list of Payment objects
-    """
-    recorded_payments = set()
-    try:
-        with open(TRANSACTIONS_FILE) as f:
-            for (
-                _email,
-                _amount,
-                payment_file,
-                _commit_hash,
-                _created_at,
-            ) in csv.reader(f):
-                recorded_payments.add(payment_file)
-    except FileNotFoundError:
-        pass
-    all_payments = get_all_payments()
-    return [p for p in all_payments if p.file not in recorded_payments]
-
-
-def generate_transactions(amount, attributions, payment_file, commit_hash):
+def pay_outstanding_debts(
+    available_amount: Decimal,
+    all_debts: Iterable[Debt],
+    payable_contributors: Set[str],
+) -> List[DebtPayment]:
     """
-    Generate transactions reflecting the amount owed to each contributor from
-    a fresh payment amount -- one transaction per attributable contributor.
+    Given an available amount return debt payments for as many debts as can
+    be covered
     """
-    assert amount > 0
-    assert attributions
-    transactions = []
-    for email, amount_owed in get_amounts_owed(amount, attributions):
-        t = Transaction(email, amount_owed, payment_file, commit_hash)
-        transactions.append(t)
-    return transactions
+    payable_debts = [
+        d
+        for d in all_debts
+        if not d.is_fulfilled() and d.email in payable_contributors
+    ]
 
+    cummulative_debt = [
+        amount
+        for amount in accumulate(
+            (d.amount_remaining() for d in payable_debts), initial=0
+        )
+        if amount <= available_amount
+    ]
 
-def get_existing_itemized_payments():
-    """
-    Reads itemized payment files and returns all itemized payment objects.
-    """
-    itemized_payments = []
-    try:
-        with open(ITEMIZED_PAYMENTS_FILE) as f:
-            for (
-                email,
-                fee_amount,
-                project_amount,
-                attributable,
-                payment_file,
-            ) in csv.reader(f):
-                itemized_payment = ItemizedPayment(
-                    email,
-                    Decimal(fee_amount),
-                    Decimal(project_amount),
-                    attributable,
-                    payment_file,
-                )
-                itemized_payments.append(itemized_payment)
-    except FileNotFoundError:
-        itemized_payments = []
-    return itemized_payments
-
-
-def total_amount_paid_to_project(for_email, new_itemized_payments):
+    return [
+        DebtPayment(
+            debt=d,
+            amount=min(d.amount_remaining(), available_amount - already_paid),
+        )
+        for d, already_paid in zip(payable_debts, cummulative_debt)
+    ]
+
+
+def create_debts(
+    available_amount: Decimal,
+    distribution: Distribution,
+    payable_contributors: Set[str],
+    payment: Payment,
+):
+    return [
+        Debt(
+            email=email,
+            amount=amount,
+            amount_paid=Decimal(0),
+            payment_file=payment.file,
+        )
+        for email, amount in distribution.distribute(available_amount).items()
+        if email not in payable_contributors
+    ]
+
+
+def distribute_payment(
+    payment: Payment, distribution: Distribution
+) -> Tuple[List[Debt], List[DebtPayment], List[Transaction], List[Advance]]:
     """
-    Calculates the sum of a single user's attributable payments (minus
-    fees paid towards instruments) for determining how much the user
-    has invested in the project so far. Non-attributable payments do
-    not count towards investment.
+    Generate transactions to contributors from a (new) payment.
+
+    We consult the attribution file and determine how much is owed
+    to each contributor based on the current percentages, generating a
+    fresh entry in the transactions file for each contributor.
     """
-    all_itemized_payments = (
-        get_existing_itemized_payments() + new_itemized_payments
+
+    # 1. check payable outstanding debts
+    # 2. pay them off in chronological order (maybe partially)
+    # 3. (if leftover) identify unpayable people in the relevant distribution file
+    # 4. record debt for each of them according to their attribution
+
+    unpayable_contributors = set(UnpayableContributorsRepo())
+    payable_contributors = {
+        email
+        for email in distribution
+        if email and email not in unpayable_contributors
+    }
+
+    #
+    # Pay as many outstanding debts as possible
+    #
+
+    debt_payments = pay_outstanding_debts(
+        payment.amount, DebtsRepo(), payable_contributors
     )
-    return sum(
-        p.project_amount
-        for p in all_itemized_payments
-        if p.attributable and p.email == for_email
+
+    # The "available" amount is what is left over after paying off debts
+    available_amount = payment.amount - sum(dp.amount for dp in debt_payments)
+
+    #
+    # Create fresh debts for anyone we can't pay
+    #
+    # TODO: Make it clearer that some people get debts and the others get N advances (maybe zero)
+
+    fresh_debts = create_debts(
+        available_amount, distribution, payable_contributors, payment
     )
 
+    #
+    # Draw dawn contributor's existing advances first, before paying them
+    #
+
+    advance_totals = Tally((a.email, a.amount) for a in AdvancesRepo())
+
+    negative_advances = [
+        Advance(
+            email=email,
+            amount=-min(
+                payable_amount, advance_totals[email]
+            ),  # Note the negative sign
+            payment_file=payment.file,
+        )
+        for email, payable_amount in distribution.without(
+            unpayable_contributors
+        )
+        .distribute(available_amount)
+        .items()
+        if advance_totals[email] > ACCOUNTING_ZERO
+    ]
+
+    #
+    # Advance payable contributors any extra money
+    #
+
+    redistribution_pot = Decimal(
+        # amount we will not pay because we created debts instead
+        sum(d.amount for d in fresh_debts)
+        # amount we will not pay because we drew down advances instead
+        # these are negative amounts, hence the abs
+        + sum(abs(a.amount) for a in negative_advances)
+    )
+
+    fresh_advances = (
+        [
+            Advance(
+                email=email,
+                amount=amount,
+                payment_file=payment.file,
+            )
+            for email, amount in distribution.without(unpayable_contributors)
+            .distribute(redistribution_pot)
+            .items()
+        ]
+        if redistribution_pot > ACCOUNTING_ZERO
+        else []
+    )
+
+    #
+    # Create equity transactions for the total amounts of outgoing money
+    #
+
+    negative_advance_totals = Tally(
+        (a.email, a.amount) for a in negative_advances
+    )
+    fresh_advance_totals = Tally((a.email, a.amount) for a in fresh_advances)
+    debt_payments_totals = Tally(
+        (dp.debt.email, dp.amount) for dp in debt_payments
+    )
+
+    transactions = [
+        Transaction(
+            email=email,
+            payment_file=payment.file,
+            amount=(
+                # what you would normally get
+                equity
+                # minus amount drawn from your advances
+                - abs(negative_advance_totals[email])
+                # plus new advances from the pot
+                + fresh_advance_totals[email]
+                # plus any payments for old debts
+                + debt_payments_totals[email]
+            ),
+        )
+        for email, equity in distribution.distribute(available_amount).items()
+        if email in payable_contributors
+    ]
+
+    processed_debts = fresh_debts
+    advances = negative_advances + fresh_advances
+
+    return processed_debts, debt_payments, transactions, advances
+
 
 def calculate_incoming_investment(payment, price, new_itemized_payments):
     """
     If the payment brings the aggregate amount paid by the payee
     above the price, then that excess is treated as investment.
     """
-    total_payments = total_amount_paid_to_project(
-        payment.email, new_itemized_payments
+    total_attributable_payments = sum(
+        p.project_amount
+        for p in [*ItemizedPaymentsRepo(), *new_itemized_payments]
+        if p.attributable and p.email == payment.email
     )
-    previous_total = total_payments - payment.amount  # fees already deducted
-    # how much of the incoming amount goes towards investment?
-    incoming_investment = total_payments - max(price, previous_total)
+
+    incoming_investment = min(
+        total_attributable_payments - price, payment.amount
+    )
+
     return max(0, incoming_investment)
 
 
@@ -208,56 +303,6 @@ def calculate_incoming_attribution(
         return None
 
 
-def normalize(attributions):
-    total_share = sum(share for _, share in attributions.items())
-    target_proportion = Decimal("1") / total_share
-    for email in attributions:
-        attributions[email] *= target_proportion
-
-
-def write_attributions(attributions):
-    # don't write attributions if they aren't normalized
-    assert_attributions_normalized(attributions)
-    # format for output as percentages
-    attributions = [
-        (email, serialize_proportion(share))
-        for email, share in attributions.items()
-    ]
-    attributions_file = os.path.join(ABE_ROOT, ATTRIBUTIONS_FILE)
-    with open(attributions_file, 'w') as f:
-        writer = csv.writer(f)
-        for row in attributions:
-            writer.writerow(row)
-
-
-def write_append_transactions(transactions):
-    with open(TRANSACTIONS_FILE, 'a') as f:
-        writer = csv.writer(f)
-        for row in transactions:
-            writer.writerow(astuple(row))
-
-
-def write_append_itemized_payments(itemized_payments):
-    with open(ITEMIZED_PAYMENTS_FILE, 'a') as f:
-        writer = csv.writer(f)
-        for row in itemized_payments:
-            writer.writerow(astuple(row))
-
-
-def write_append_advances(advances):
-    with open(ADVANCES_FILE, 'a') as f:
-        writer = csv.writer(f)
-        for row in advances:
-            writer.writerow(astuple(row))
-
-
-def write_valuation(valuation):
-    rounded_valuation = f"{valuation:.2f}"
-    with open(VALUATION_FILE, 'w') as f:
-        writer = csv.writer(f)
-        writer.writerow((rounded_valuation,))
-
-
 def dilute_attributions(incoming_attribution, attributions):
     """
     Incorporate a fresh attributive share by diluting existing attributions,
@@ -284,268 +329,6 @@ def dilute_attributions(incoming_attribution, attributions):
     correct_rounding_error(attributions, incoming_attribution)
 
 
-def inflate_valuation(valuation, amount):
-    """
-    Determine the posterior valuation as the fresh investment amount
-    added to the prior valuation.
-    """
-    return valuation + amount
-
-
-def read_debts():
-    debts = []
-    try:
-        with open(DEBTS_FILE) as f:
-            for (
-                email,
-                amount,
-                amount_paid,
-                payment_file,
-                commit_hash,
-                created_at,
-            ) in csv.reader(f):
-                debts.append(Debt(email, Decimal(amount), Decimal(amount_paid), payment_file, commit_hash, created_at))
-    except FileNotFoundError:
-        pass
-
-    return debts
-
-
-def read_advances(attributions):
-    advances = defaultdict(list)
-    try:
-        with open(ADVANCES_FILE) as f:
-            for (
-                email,
-                amount,
-                payment_file,
-                commit_hash,
-                created_at,
-            ) in csv.reader(f):
-                if email in attributions:
-                    advances[email].append(Advance(email, Decimal(amount), payment_file, commit_hash, created_at))
-    except FileNotFoundError:
-        pass
-
-    return advances
-
-
-def get_sum_of_advances_by_contributor(attributions):
-    """
-    Sum all Advance objects for each contributor to get the total amount
-    that they currently have in advances and have not yet drawn down.
-    Return a dictionary with the contributor's email as the key and the
-    their advance amount as the value.
-    """
-    all_advances = read_advances(attributions)
-    advance_totals = {email: sum(a.amount for a in advances)
-                      for email, advances
-                      in all_advances.items()}
-    return advance_totals
-
-
-def get_payable_debts(unpayable_contributors, attributions):
-    debts = read_debts()
-    debts = [d for d in debts
-             if not d.is_fulfilled()
-             and d.email in attributions
-             and d.email not in unpayable_contributors]
-    return debts
-
-
-def pay_debts(payable_debts, payment):
-    """
-    Go through debts in chronological order, and pay each as much as possible,
-    stopping when either the money runs out, or there are no further debts.
-    Returns the updated debts reflecting fresh payments to be made this time,
-    and transactions representing those fresh payments.
-    """
-    updated_debts = []
-    transactions = []
-    available_amount = payment.amount
-    for debt in sorted(payable_debts, key=lambda x: x.created_at):
-        payable_amount = min(available_amount, debt.amount_remaining())
-        if payable_amount < ACCOUNTING_ZERO:
-            break
-        debt.amount_paid += payable_amount
-        available_amount -= payable_amount
-        transaction = Transaction(debt.email, payable_amount, payment.file, debt.commit_hash)
-        transactions.append(transaction)
-        updated_debts.append(debt)
-
-    return updated_debts, transactions
-
-
-def get_unpayable_contributors():
-    """
-    Read the unpayable_contributors file to get the list of contributors who
-    are unpayable.
-    """
-    contributors = []
-    try:
-        with open(UNPAYABLE_CONTRIBUTORS_FILE) as f:
-            for contributor in f:
-                contributor = contributor.strip()
-                if contributor:
-                    contributors.append(contributor)
-    except FileNotFoundError:
-        pass
-    return contributors
-
-
-def create_debts(amounts_owed, unpayable_contributors, payment_file):
-    """
-    Create fresh debts (to unpayable contributors).
-    """
-    amounts_unpayable = {email: amount
-                         for email, amount in amounts_owed.items()
-                         if email in unpayable_contributors}
-    debts = []
-    commit_hash = get_git_revision_short_hash()
-    for email, amount in amounts_unpayable.items():
-        debt = Debt(email, amount, payment_file=payment_file, commit_hash=commit_hash)
-        debts.append(debt)
-
-    return debts
-
-
-def write_debts(processed_debts):
-    """
-    1. Build a hash of all the processed debts, generating an id for each
-       (based on email and payment file).
-    2. read the existing debts file, row by row.
-    3. if the debt in the row is in the "processed" hash, then write the
-       processed version instead of the input version and remove it from the
-       hash, otherwise write the input version.
-    4. write the debts that remain in the processed hash.
-    """
-    existing_debts = read_debts()
-    processed_debts_hash = {debt.key(): debt for debt in processed_debts}
-    with open(DEBTS_FILE, 'w') as f:
-        writer = csv.writer(f)
-        for existing_debt in existing_debts:
-            # if the existing debt has been processed, write the processed version
-            # otherwise re-write the existing version
-            if processed_debt := processed_debts_hash.get(existing_debt.key()):
-                writer.writerow(astuple(processed_debt))
-                del processed_debts_hash[processed_debt.key()]
-            else:
-                writer.writerow(astuple(existing_debt))
-        for debt in processed_debts_hash.values():
-            writer.writerow(astuple(debt))
-
-
-def renormalize(attributions, excluded_contributors):
-    target_proportion = 1 / (1 - sum(attributions[email] for email in excluded_contributors))
-    remainder_attributions = {}
-    for email in attributions:
-        # renormalize to reflect dilution
-        remainder_attributions[email] = attributions[email] * target_proportion
-    return remainder_attributions
-
-
-def get_amounts_owed(total_amount, attributions):
-    return {email: share * total_amount
-            for email, share in attributions.items()}
-
-
-def redistribute_pot(redistribution_pot, attributions, unpayable_contributors, payment_file, amounts_payable):
-    """
-    Redistribute the pot of remaining money over all payable contributors, according to attributions
-    share (normalized to 100%). Create advances for those amounts (because they are in excess
-    of the amount owed to each contributor from the original payment) and add the amounts to the
-    amounts_payable dictionary to keep track of the full amount we are about to pay everyone.
-    """
-    fresh_advances = []
-    payable_attributions = {email: share for email, share in attributions.items() if email not in unpayable_contributors}
-    normalize(payable_attributions)
-    for email, share in payable_attributions.items():
-        advance_amount = redistribution_pot * share
-        fresh_advances.append(Advance(email=email,
-                                      amount=advance_amount,
-                                      payment_file=payment_file,
-                                      commit_hash=get_git_revision_short_hash()))
-        amounts_payable[email] += advance_amount
-
-    return fresh_advances
-
-
-def distribute_payment(payment, attributions):
-    """
-    Generate transactions to contributors from a (new) payment.
-
-    We consult the attribution file and determine how much is owed
-    to each contributor based on the current percentages, generating a
-    fresh entry in the transactions file for each contributor.
-    """
-
-    # 1. check payable outstanding debts
-    # 2. pay them off in chronological order (maybe partially)
-    # 3. (if leftover) identify unpayable people in the relevant attributions file
-    # 4. record debt for each of them according to their attribution
-    commit_hash = get_git_revision_short_hash()
-    unpayable_contributors = get_unpayable_contributors()
-    payable_debts = get_payable_debts(unpayable_contributors, attributions)
-    updated_debts, debt_transactions = pay_debts(payable_debts, payment)
-    # The "available" amount is what is left over after paying off debts
-    available_amount = payment.amount - sum(t.amount for t in debt_transactions)
-
-    fresh_debts = []
-    equity_transactions = []
-    negative_advances = []
-    fresh_advances = []
-    if available_amount > ACCOUNTING_ZERO:
-        amounts_owed = get_amounts_owed(available_amount, attributions)
-        fresh_debts = create_debts(amounts_owed,
-                                   unpayable_contributors,
-                                   payment.file)
-        redistribution_pot = sum(d.amount for d in fresh_debts)
-
-        # just retain payable people and their amounts owed
-        amounts_payable = {email: amount
-                           for email, amount in amounts_owed.items()
-                           if email not in unpayable_contributors}
-
-        # use the amount owed to each contributor to draw down any advances
-        # they may already have and then decrement their amount payable accordingly
-        advance_totals = get_sum_of_advances_by_contributor(attributions)
-        for email, advance_total in advance_totals.items():
-            amount_payable = amounts_payable.get(email, 0)
-            drawdown_amount = min(advance_total, amount_payable)
-            if drawdown_amount > ACCOUNTING_ZERO:
-                negative_advance = Advance(email=email,
-                                           amount=-drawdown_amount, # note minus sign
-                                           payment_file=payment.file,
-                                           commit_hash=commit_hash)
-                negative_advances.append(negative_advance)
-                amounts_payable[email] -= drawdown_amount
-
-        # note that these are drawn down amounts and therefore have negative amounts
-        # and that's why we take the absolute value here
-        redistribution_pot += sum(abs(a.amount) for a in negative_advances)
-
-        # redistribute the pot over all payable contributors - produce fresh advances and add to amounts payable
-        if redistribution_pot > ACCOUNTING_ZERO:
-            fresh_advances = redistribute_pot(redistribution_pot,
-                                              attributions,
-                                              unpayable_contributors,
-                                              payment.file,
-                                              amounts_payable)
-
-        for email, amount in amounts_payable.items():
-            new_equity_transaction = Transaction(email=email,
-                                                 amount=amount,
-                                                 payment_file=payment.file,
-                                                 commit_hash=commit_hash)
-            equity_transactions.append(new_equity_transaction)
-
-    debts = updated_debts + fresh_debts
-    transactions = equity_transactions + debt_transactions
-    advances = negative_advances + fresh_advances
-
-    return debts, transactions, advances
-
-
 def handle_investment(
     payment, new_itemized_payments, attributions, price, prior_valuation
 ):
@@ -560,9 +343,8 @@ def handle_investment(
         payment, price, new_itemized_payments
     )
     # inflate valuation by the amount of the fresh investment
-    posterior_valuation = inflate_valuation(
-        prior_valuation, incoming_investment
-    )
+    posterior_valuation = prior_valuation + incoming_investment
+
     incoming_attribution = calculate_incoming_attribution(
         payment.email, incoming_investment, posterior_valuation
     )
@@ -571,16 +353,6 @@ def handle_investment(
     return posterior_valuation
 
 
-def _create_itemized_payment(payment, fee_amount):
-    return ItemizedPayment(
-        payment.email,
-        fee_amount,
-        payment.amount,  # already has fees deducted
-        payment.attributable,
-        payment.file,
-    )
-
-
 def process_payments(instruments, attributions):
     """
     Process new payments by paying out instruments and then, from the amount
@@ -591,35 +363,66 @@ def process_payments(instruments, attributions):
     price = read_price()
     valuation = read_valuation()
     new_debts = []
+    new_debt_payments = []
     new_advances = []
     new_transactions = []
     new_itemized_payments = []
-    unprocessed_payments = find_unprocessed_payments()
+
+    processed_payment_files = {t.payment_file for t in TransactionsRepo()}
+    unprocessed_payments = [
+        p for p in AllPaymentsRepo() if p.file not in processed_payment_files
+    ]
+
     for payment in unprocessed_payments:
         # first, process instruments (i.e. pay fees)
-        debts, transactions, advances = distribute_payment(payment, instruments)
+        debts, debt_payments, transactions, advances = distribute_payment(
+            payment,
+            Distribution(
+                # The missing percentage in the instruments file
+                # should not be distributed to anyone (shareholder: None)
+                # TODO: Move to process_payments_and_record_updates
+                {**instruments, None: Decimal(1) - sum(instruments.values())}
+            ),
+        )
         new_transactions += transactions
         new_debts += debts
+        new_debt_payments += debt_payments
         new_advances += advances
         fees_paid_out = sum(t.amount for t in transactions)
         # deduct the amount paid out to instruments before
         # processing it for attributions
         payment.amount -= fees_paid_out
         new_itemized_payments.append(
-            _create_itemized_payment(payment, fees_paid_out)
+            ItemizedPayment(
+                payment.email,
+                fees_paid_out,
+                payment.amount,  # already has fees deducted
+                payment.attributable,
+                payment.file,
+            )
         )
         # next, process attributions - using the amount owed to the project
         # (which is the amount leftover after paying instruments/fees)
         if payment.amount > ACCOUNTING_ZERO:
-            debts, transactions, advances = distribute_payment(payment, attributions)
+            debts, debt_payments, transactions, advances = distribute_payment(
+                payment, Distribution(attributions)
+            )
             new_transactions += transactions
             new_debts += debts
+            new_debt_payments += debt_payments
             new_advances += advances
         if payment.attributable:
             valuation = handle_investment(
                 payment, new_itemized_payments, attributions, price, valuation
             )
-    return new_debts, new_transactions, valuation, new_itemized_payments, new_advances
+    return (
+        new_debts,
+        new_debt_payments,
+        new_transactions,
+        valuation,
+        new_itemized_payments,
+        new_advances,
+    )
 
 
 def process_payments_and_record_updates():
@@ -628,11 +431,14 @@ def process_payments_and_record_updates():
     and attributions files. Record updated transactions, valuation, and
     renormalized attributions only after all payments have been processed.
     """
-    instruments = read_attributions(INSTRUMENTS_FILE, validate=False)
-    attributions = read_attributions(ATTRIBUTIONS_FILE)
+    instruments = {a.email: a.decimal_share for a in InstrumentsRepo()}
+    attributions = {a.email: a.decimal_share for a in AttributionsRepo()}
+
+    assert_attributions_normalized(attributions)
 
     (
-        debts,
+        new_debts,
+        debt_payments,
         transactions,
         posterior_valuation,
         new_itemized_payments,
@@ -642,12 +448,12 @@ def process_payments_and_record_updates():
     # we only write the changes to disk at the end
     # so that if any errors are encountered, no
     # changes are made.
-    write_debts(debts)
-    write_append_transactions(transactions)
+    write_debts(new_debts, debt_payments)
+    TransactionsRepo().extend(transactions)
     write_attributions(attributions)
     write_valuation(posterior_valuation)
-    write_append_itemized_payments(new_itemized_payments)
-    write_append_advances(advances)
+    ItemizedPaymentsRepo().extend(new_itemized_payments)
+    AdvancesRepo().extend(advances)
 
 
 def main():
diff --git a/oldabe/money_out.py b/oldabe/money_out.py
index 293d222..53e501a 100755
--- a/oldabe/money_out.py
+++ b/oldabe/money_out.py
@@ -1,46 +1,10 @@
 #!/usr/bin/env python
 
-from .models import Transaction, Debt, Advance
-from datetime import datetime
-import csv
-import os
-import re
 from collections import defaultdict
 from decimal import Decimal, getcontext
-from .constants import (
-    ABE_ROOT, PAYOUTS_DIR, TRANSACTIONS_FILE, DEBTS_FILE, ADVANCES_FILE
-)
 
-
-def read_transaction_amounts():
-    balances = defaultdict(int)
-    with open(TRANSACTIONS_FILE) as f:
-        for row in csv.reader(f):
-            t = Transaction(*row)
-            t.amount = Decimal(t.amount)
-            t.created_at = datetime.fromisoformat(t.created_at)
-            balances[t.email] += t.amount
-    return balances
-
-
-def read_payout(payout_file):
-    with open(os.path.join(PAYOUTS_DIR, payout_file)) as f:
-        for row in csv.reader(f):
-            name, _email, amount, _date = row
-            amount = Decimal(re.sub("[^0-9.]", "", amount))
-            return name, amount
-
-
-def read_payout_amounts():
-    balances = defaultdict(int)
-    try:
-        payout_files = os.listdir(PAYOUTS_DIR)
-    except FileNotFoundError:
-        payout_files = []
-    for payout_file in payout_files:
-        name, amount = read_payout(payout_file)
-        balances[name] += amount
-    return balances
+from .repos import AdvancesRepo, DebtsRepo, PayoutsRepo, TransactionsRepo
+from .tally import Tally
 
 
 def compute_balances(owed: dict, paid: dict):
@@ -49,6 +13,7 @@ def compute_balances(owed: dict, paid: dict):
     """
     balances = defaultdict(int)
     for email in owed.keys():
+        # TODO: We are not testing alreayd paid
         balance = owed[email] - paid[email]
         if balance > Decimal("0"):
             balances[email] = balance
@@ -60,7 +25,8 @@ def prepare_balances_message(balances: dict):
         return "There are no outstanding (payable) balances."
     balances_table = ""
     for name, balance in balances.items():
-        balances_table += f"{name} | {balance:.2f}\n"
+        if balance > 0:
+            balances_table += f"{name} | {balance:.2f}\n"
     message = f"""
     The current outstanding (payable) balances are:
 
@@ -73,18 +39,6 @@ def prepare_balances_message(balances: dict):
     return "\r\n".join(line.strip() for line in message.split('\n')).strip()
 
 
-def read_outstanding_debt_amounts():
-    outstanding_debts = defaultdict(int)
-    with open(DEBTS_FILE) as f:
-        for row in csv.reader(f):
-            d = Debt(*row)
-            d.amount = Decimal(d.amount)
-            d.amount_paid = Decimal(d.amount_paid)
-            outstanding_amount = d.amount - d.amount_paid
-            outstanding_debts[d.email] += outstanding_amount
-    return outstanding_debts
-
-
 def prepare_debts_message(outstanding_debts: dict):
     if not outstanding_debts:
         return "There are no outstanding (unpayable) debts."
@@ -103,16 +57,6 @@ def prepare_debts_message(outstanding_debts: dict):
     return "\r\n".join(line.strip() for line in message.split('\n')).strip()
 
 
-def read_advance_amounts():
-    advances = defaultdict(int)
-    with open(ADVANCES_FILE) as f:
-        for row in csv.reader(f):
-            a = Advance(*row)
-            a.amount = Decimal(a.amount)
-            advances[a.email] += a.amount
-    return advances
-
-
 def prepare_advances_message(advances: dict):
     """ A temporary message reporting aggregate advances, for testing purposes.
     """
@@ -152,14 +96,18 @@ def compile_outstanding_balances():
     """ Read all accounting records and determine the total outstanding
     balances, debts, and advances for each contributor.
     """
-    owed = read_transaction_amounts()
-    paid = read_payout_amounts()
-    balances = compute_balances(owed, paid)
+    # owed = read_owed_amounts()
+    owed = Tally((t.email, t.amount) for t in TransactionsRepo())
+    paid = Tally((p.email, p.amount) for p in PayoutsRepo())
+    balances = owed - paid
     balances_message = prepare_balances_message(balances)
-    outstanding_debts = read_outstanding_debt_amounts()
+
+    outstanding_debts = Tally((d.email, d.amount_remaining()) for d in DebtsRepo())
     debts_message = prepare_debts_message(outstanding_debts)
-    advances = read_advance_amounts()
+
+    advances = Tally((a.email, a.amount) for a in AdvancesRepo())
     advances_message = prepare_advances_message(advances)
+
     return combined_message(balances_message, debts_message, advances_message)
 
 
diff --git a/oldabe/repos.py b/oldabe/repos.py
new file mode 100644
index 0000000..b3506aa
--- /dev/null
+++ b/oldabe/repos.py
@@ -0,0 +1,163 @@
+import csv
+import dataclasses
+import os
+import re
+from datetime import datetime
+from decimal import Decimal
+from typing import Any, Generic, Iterable, Iterator, List, Type, TypeVar
+
+from oldabe.constants import (ADVANCES_FILE, ATTRIBUTIONS_FILE, DEBTS_FILE,
+                              INSTRUMENTS_FILE, ITEMIZED_PAYMENTS_FILE,
+                              NONATTRIBUTABLE_PAYMENTS_DIR, PAYMENTS_DIR,
+                              PAYOUTS_DIR, TRANSACTIONS_FILE,
+                              UNPAYABLE_CONTRIBUTORS_FILE)
+from oldabe.models import (Advance, Attribution, Debt, ItemizedPayment,
+                           Payment, Payout, Transaction)
+
+
+def fix_types(row: List[str], Model: type) -> List[Any]:
+    """
+    Cast string field values from the CSV into the proper types
+    """
+    def _cast(field, value):
+        if field.type is Decimal:
+            return Decimal(re.sub("[^0-9.]", "", value))
+        elif field.type is datetime:
+            return datetime.fromisoformat(value)
+        else:
+            return value
+
+    return [
+        _cast(field, value) for field, value in zip(dataclasses.fields(Model), row)
+    ]
+
+
+T = TypeVar('T')
+
+
+class FileRepo(Generic[T]):
+    """
+    A sequence of dataclass instances stored as rows in a CSV
+    """
+
+    filename: str
+    Model: Type[T]
+
+    def __iter__(self) -> Iterator[T]:
+        objs = []
+        try:
+            with open(self.filename) as f:
+                for row in csv.reader(f, skipinitialspace=True):
+                    if dataclasses.is_dataclass(self.Model):
+                        row = fix_types(row, self.Model)
+                    obj = self.Model(*row)
+                    objs.append(obj)
+        except FileNotFoundError:
+            pass
+
+        yield from objs
+
+    def extend(self, objs: Iterable[T]):
+        with open(self.filename, "a") as f:
+            writer = csv.writer(f)
+            for obj in objs:
+                writer.writerow(dataclasses.astuple(obj))
+
+
+class DirRepo(Generic[T]):
+    """
+    A sequence of dataclass instances stored as single row CSV files in a dir
+    """
+
+    dirname: str
+    Model: Type[T]
+
+    def __iter__(self) -> Iterator[T]:
+        objs = []
+        try:
+            filenames = [
+                f
+                for f in os.listdir(self.dirname)
+                if not os.path.isdir(os.path.join(self.dirname, f))
+            ]
+        except FileNotFoundError:
+            filenames = []
+
+        for filename in filenames:
+            with open(os.path.join(self.dirname, filename)) as f:
+                row = next(csv.reader(f, skipinitialspace=True))
+                if dataclasses.is_dataclass(self.Model):
+                    row = fix_types(row, self.Model)
+                obj = self.Model(*row)
+                setattr(obj, "file", filename)
+                objs.append(obj)
+
+        yield from objs
+
+
+class TransactionsRepo(FileRepo[Transaction]):
+    filename = TRANSACTIONS_FILE
+    Model = Transaction
+
+
+class PayoutsRepo(DirRepo):
+    dirname = PAYOUTS_DIR
+    Model = Payout
+
+
+class DebtsRepo(FileRepo[Debt]):
+    filename = DEBTS_FILE
+    Model = Debt
+
+
+class AdvancesRepo(FileRepo[Advance]):
+    filename = ADVANCES_FILE
+    Model = Advance
+
+
+class AttributionsRepo(FileRepo[Attribution]):
+    filename = ATTRIBUTIONS_FILE
+    Model = Attribution
+
+
+class InstrumentsRepo(FileRepo[Attribution]):
+    filename = INSTRUMENTS_FILE
+    Model = Attribution
+
+
+class AttributablePaymentsRepo(DirRepo[Payment]):
+    dirname = PAYMENTS_DIR
+    Model = Payment
+
+    def __iter__(self):
+        yield from (
+            dataclasses.replace(obj, attributable=True)
+            for obj in super().__iter__()
+        )
+
+
+class NonAttributablePaymentsRepo(DirRepo[Payment]):
+    dirname = NONATTRIBUTABLE_PAYMENTS_DIR
+    Model = Payment
+
+    def __iter__(self):
+        yield from (
+            dataclasses.replace(obj, attributable=False)
+            for obj in super().__iter__()
+        )
+
+
+class AllPaymentsRepo:
+    def __iter__(self):
+        yield from AttributablePaymentsRepo()
+        yield from NonAttributablePaymentsRepo()
+
+
+class ItemizedPaymentsRepo(FileRepo[ItemizedPayment]):
+    filename = ITEMIZED_PAYMENTS_FILE
+    Model = ItemizedPayment
+
+
+class UnpayableContributorsRepo(FileRepo[str]):
+    filename = UNPAYABLE_CONTRIBUTORS_FILE
+    Model = str
diff --git a/oldabe/tally.py b/oldabe/tally.py
new file mode 100644
index 0000000..41a49c9
--- /dev/null
+++ b/oldabe/tally.py
@@ -0,0 +1,30 @@
+from collections import defaultdict
+from decimal import Decimal
+from operator import sub
+
+
+class Tally(defaultdict[str, Decimal]):
+    """
+    A dictionary for keeping the tally of an amount
+
+    Inspired by collections.Counter, but instead of a count of ocurrences it
+    keeps the sum of an amount.
+    """
+
+    def __init__(self, source=[]):
+        if type(source) is dict:
+            super().__init__(Decimal, source)
+            return
+
+        super().__init__(Decimal)
+        for key, amount in source:
+            self[key] += amount
+
+    def combine(self, combinator, other):
+        result = Tally()
+        for key in dict(**self, **other).keys():
+            result[key] = combinator(self[key], other[key])
+        return result
+
+    def __sub__(self, other):
+        return self.combine(sub, other)
diff --git a/tests/integration/old_abe_test.py b/tests/integration/old_abe_test.py
index 429c6a0..c30e8a9 100644
--- a/tests/integration/old_abe_test.py
+++ b/tests/integration/old_abe_test.py
@@ -1,27 +1,26 @@
-import pytest
+import os
 from datetime import datetime
 from decimal import localcontext
-import time_machine
 from unittest.mock import patch
-from oldabe.money_in import (
-    process_payments_and_record_updates,
-)
-from oldabe.money_out import (
-    compile_outstanding_balances,
-)
+
+import pytest
+import time_machine
+
+from oldabe.money_in import process_payments_and_record_updates
+from oldabe.money_out import compile_outstanding_balances
+
 from .fixtures import abe_fs
-import os
 
 
 class TestNoPayments:
 
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_no_transactions_generated(self, mock_git_rev, abe_fs):
         process_payments_and_record_updates()
         with open('./abe/transactions.txt') as f:
             assert f.read() == ""
 
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         process_payments_and_record_updates()
         message = compile_outstanding_balances()
@@ -38,13 +37,13 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
 class TestPaymentAbovePrice:
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_generates_transactions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             with open('./abe/transactions.txt') as f:
                 assert f.read() == (
@@ -56,13 +55,13 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
                 )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_dilutes_attributions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 10000
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             with open('./abe/attributions.txt') as f:
                 assert f.read() == (
@@ -73,13 +72,13 @@ def test_dilutes_attributions(self, mock_git_rev, abe_fs):
                 )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
 
             message = compile_outstanding_balances()
@@ -92,13 +91,13 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
 class TestPaymentBelowPrice:
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_generates_transactions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 1
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             with open('./abe/transactions.txt') as f:
                 assert f.read() == (
@@ -110,13 +109,13 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
                 )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 1
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             message = compile_outstanding_balances()
 
@@ -128,13 +127,13 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
 class TestNonAttributablePayment:
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_does_not_dilute_attributions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
             abe_fs.create_file("./abe/payments/nonattributable/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             with open('./abe/attributions.txt') as f:
                 assert f.read() == (
@@ -144,13 +143,13 @@ def test_does_not_dilute_attributions(self, mock_git_rev, abe_fs):
                 )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_generates_transactions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             with open('./abe/transactions.txt') as f:
                 assert f.read() == (
@@ -162,13 +161,13 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
                 )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
             abe_fs.create_file("./abe/payments/nonattributable/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
             message = compile_outstanding_balances()
 
@@ -184,13 +183,13 @@ def _call(self, abe_fs):
             context.prec = 2
             amount = 100
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             abe_fs.create_file("./abe/unpayable_contributors.txt",
                                contents=f"ariana")
             process_payments_and_record_updates()
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_generates_transactions(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         with open('./abe/transactions.txt') as f:
@@ -202,7 +201,7 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
             )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_records_debt(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         with open('./abe/debts.txt') as f:
@@ -211,7 +210,7 @@ def test_records_debt(self, mock_git_rev, abe_fs):
             )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_records_advances(self, mock_git_rev, abe_fs):
         # advances for payable people
         # and none for unpayable
@@ -223,7 +222,7 @@ def test_records_advances(self, mock_git_rev, abe_fs):
             )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         message = compile_outstanding_balances()
@@ -254,13 +253,13 @@ def _call(self, abe_fs):
                                    "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n"
                                ))
             abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             abe_fs.create_file("./abe/payments/2.txt",
-                               contents=f"sam,036eaf6,{amount},dummydate")
+                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
             process_payments_and_record_updates()
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_debt_paid(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         with open('./abe/debts.txt') as f:
@@ -269,7 +268,7 @@ def test_debt_paid(self, mock_git_rev, abe_fs):
             )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_transactions(self, mock_git_rev, abe_fs):
         # here, because the two payment amounts are the same,
         # it ends up correcting immediately. We might consider
@@ -286,12 +285,11 @@ def test_transactions(self, mock_git_rev, abe_fs):
                 "DIA,5.0,2.txt,abcd123,1985-10-26 01:24:00\n"
                 "sid,36,2.txt,abcd123,1985-10-26 01:24:00\n"
                 "jair,20,2.txt,abcd123,1985-10-26 01:24:00\n"
-                "ariana,19,2.txt,abcd123,1985-10-26 01:24:00\n"
-                "ariana,19,2.txt,abcd123,1985-10-26 01:24:00\n"
+                "ariana,38,2.txt,abcd123,1985-10-26 01:24:00\n"
             )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_advances(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         with open('./abe/advances.txt') as f:
@@ -306,7 +304,7 @@ def test_advances(self, mock_git_rev, abe_fs):
             )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
-    @patch('oldabe.money_in.get_git_revision_short_hash', return_value='abcd123')
+    @patch('oldabe.models.default_commit_hash', return_value='abcd123')
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         message = compile_outstanding_balances()

From 59075eb850c058610d3126135dec5c268a30fa0f Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Thu, 26 Sep 2024 20:13:03 -0700
Subject: [PATCH 73/90] remove outdated money_in unit tests

rely on integration tests for now
---
 tests/unit/money_in_test.py | 379 +-----------------------------------
 1 file changed, 4 insertions(+), 375 deletions(-)

diff --git a/tests/unit/money_in_test.py b/tests/unit/money_in_test.py
index 0984b6f..800fba5 100644
--- a/tests/unit/money_in_test.py
+++ b/tests/unit/money_in_test.py
@@ -1,380 +1,9 @@
 from decimal import Decimal
-from oldabe.money_in import (
-    calculate_incoming_investment,
-    total_amount_paid_to_project,
-    calculate_incoming_attribution,
-    generate_transactions,
-    renormalize,
-    inflate_valuation,
-    process_payments,
-)
-from oldabe.accounting_utils import correct_rounding_error
-from oldabe.models import Attribution, Payment
-import pytest
-from unittest.mock import patch
-from .utils import call_sequence
-from .fixtures import (
-    instruments,
-    normalized_attributions,
-    excess_attributions,
-    shortfall_attributions,
-    empty_attributions,
-    single_contributor_attributions,
-    itemized_payments,
-    new_itemized_payments,
-)  # noqa
 
+# See integration tests for now.
 
-class TestGenerateTransactions:
-    def test_zero_amount(self, normalized_attributions):
-        with pytest.raises(AssertionError):
-            generate_transactions(
-                0, normalized_attributions, 'payment-1.txt', 'abc123'
-            )
-
-    def test_empty_attributions(self, empty_attributions):
-        with pytest.raises(AssertionError):
-            generate_transactions(
-                100, empty_attributions, 'payment-1.txt', 'abc123'
-            )
-
-    def test_single_contributor_attributions(
-        self, single_contributor_attributions
-    ):
-        amount = 100
-        payment_file = 'payment-1.txt'
-        commit_hash = 'abc123'
-        result = generate_transactions(
-            amount, single_contributor_attributions, payment_file, commit_hash
-        )
-        t = result[0]
-        assert t.amount == amount
-
-    def test_as_many_transactions_as_attributions(
-        self, normalized_attributions
-    ):
-        amount = 100
-        payment_file = 'payment-1.txt'
-        commit_hash = 'abc123'
-        result = generate_transactions(
-            amount, normalized_attributions, payment_file, commit_hash
-        )
-        assert len(result) == len(normalized_attributions)
-
-    def test_transactions_reflect_attributions(self, normalized_attributions):
-        amount = 100
-        payment_file = 'payment-1.txt'
-        commit_hash = 'abc123'
-        result = generate_transactions(
-            amount, normalized_attributions, payment_file, commit_hash
-        )
-        for t in result:
-            assert t.amount == normalized_attributions[t.email] * amount
-
-    def test_everyone_in_attributions_are_represented(
-        self, normalized_attributions
-    ):
-        amount = 100
-        payment_file = 'payment-1.txt'
-        commit_hash = 'abc123'
-        result = generate_transactions(
-            amount, normalized_attributions, payment_file, commit_hash
-        )
-        emails = [t.email for t in result]
-        for contributor in normalized_attributions:
-            assert contributor in emails
-
-    def test_transaction_refers_to_payment(self, normalized_attributions):
-        amount = 100
-        payment_file = 'payment-1.txt'
-        commit_hash = 'abc123'
-        result = generate_transactions(
-            amount, normalized_attributions, payment_file, commit_hash
-        )
-        t = result[0]
-        assert t.payment_file == payment_file
-
-    def test_transaction_refers_to_commit(self, normalized_attributions):
-        amount = 100
-        payment_file = 'payment-1.txt'
-        commit_hash = 'abc123'
-        result = generate_transactions(
-            amount, normalized_attributions, payment_file, commit_hash
-        )
-        t = result[0]
-        assert t.commit_hash == commit_hash
-
-
-class TestProcessPayments:
-    @patch('oldabe.money_in._get_unprocessed_payments')
-    @patch('oldabe.money_in.get_all_payments')
-    @patch('oldabe.money_in.read_valuation')
-    @patch('oldabe.money_in.read_price')
-    def test_collects_transactions_for_all_payments(
-        self,
-        mock_read_price,
-        mock_read_valuation,
-        mock_get_all_payments,
-        mock_unprocessed_files,
-        normalized_attributions,
-        instruments,
-    ):
-        price = 100
-        valuation = 1000
-        payments = [
-            Payment('a@b.com', 100, True),
-            Payment('a@b.com', 200, False),
-        ]
-        mock_read_price.return_value = price
-        mock_read_valuation.return_value = valuation
-        mock_unprocessed_files.return_value = payments
-        mock_get_all_payments.return_value = payments
-
-        transactions, _, _ = process_payments(
-            instruments, normalized_attributions
-        )
-        # generates a transaction for each payment and each contributor in the
-        # attributions and instruments files
-        expected_total = len(payments) * (
-            len(normalized_attributions) + len(instruments)
-        )
-        assert len(transactions) == expected_total
-
-    # def test_attributable_payments_update_valuation
-    # def test_non_attributable_payments_do_not_update_valuation
-    # def test_attributions_are_renormalized_by_attributable_payments
-    # def test_instruments_are_not_renormalized_by_attributable_payments
-
-
-class TestCalculateIncomingAttribution:
-    def test_incoming_investment_less_than_zero(self, normalized_attributions):
-        assert (
-            calculate_incoming_attribution(
-                'a@b.co', Decimal('-50'), Decimal('10000')
-            )
-            == None
-        )
-
-    def test_incoming_investment_is_zero(self, normalized_attributions):
-        assert (
-            calculate_incoming_attribution(
-                'a@b.co', Decimal('0'), Decimal('10000')
-            )
-            == None
-        )
-
-    def test_normal_incoming_investment(self, normalized_attributions):
-        assert calculate_incoming_attribution(
-            'a@b.co', Decimal('50'), Decimal('10000')
-        ) == Attribution(
-            'a@b.co',
-            Decimal('0.005'),
-        )
-
-    def test_large_incoming_investment(self, normalized_attributions):
-        assert calculate_incoming_attribution(
-            'a@b.co', Decimal('5000'), Decimal('10000')
-        ) == Attribution(
-            'a@b.co',
-            Decimal('0.5'),
-        )
-
-
-class TestRenormalize:
-    @pytest.mark.parametrize(
-        "incoming_attribution, renormalized_attributions",
-        [
-            # new investor
-            (
-                Attribution('c@d.com', Decimal('0.05')),
-                {
-                    'a@b.com': Decimal('0.19'),
-                    'b@c.com': Decimal('0.76'),
-                    'c@d.com': Decimal('0.05'),
-                },
-            ),
-            (
-                Attribution('c@d.com', Decimal('0.5')),
-                {
-                    'a@b.com': Decimal('0.1'),
-                    'b@c.com': Decimal('0.4'),
-                    'c@d.com': Decimal('0.5'),
-                },
-            ),
-            (
-                Attribution('c@d.com', Decimal('0.7')),
-                {
-                    'a@b.com': Decimal('0.06'),
-                    'b@c.com': Decimal('0.24'),
-                    'c@d.com': Decimal('0.7'),
-                },
-            ),
-            # existing investor
-            (
-                Attribution('b@c.com', Decimal('0.05')),
-                {
-                    'a@b.com': Decimal('0.19'),
-                    'b@c.com': Decimal('0.81'),
-                },
-            ),
-            (
-                Attribution('b@c.com', Decimal('0.5')),
-                {
-                    'a@b.com': Decimal('0.1'),
-                    'b@c.com': Decimal('0.9'),
-                },
-            ),
-            (
-                Attribution('b@c.com', Decimal('0.7')),
-                {
-                    'a@b.com': Decimal('0.06'),
-                    'b@c.com': Decimal('0.94'),
-                },
-            ),
-        ],
-    )
-    def test_matrix(
-        self,
-        incoming_attribution,
-        renormalized_attributions,
-        normalized_attributions,
-    ):
-        attributions = normalized_attributions
-        renormalize(attributions, incoming_attribution)
-        assert attributions == renormalized_attributions
-
-
-class TestCorrectRoundingError:
-    @pytest.mark.parametrize(
-        "attributions, test_diff",
-        [
-            ("normalized_attributions", Decimal("0")),
-            ("excess_attributions", Decimal("0")),
-            ("shortfall_attributions", Decimal("0")),
-            ("normalized_attributions", Decimal("0.1")),
-            ("excess_attributions", Decimal("0.1")),
-            ("shortfall_attributions", Decimal("0.1")),
-            ("normalized_attributions", Decimal("-0.1")),
-            ("excess_attributions", Decimal("-0.1")),
-            ("shortfall_attributions", Decimal("-0.1")),
-        ],
-    )
-    @patch('oldabe.money_in.get_rounding_difference')
-    def test_incoming_is_adjusted_by_rounding_error(
-        self, mock_rounding_difference, attributions, test_diff, request
-    ):
-        # Re: request, see: https://stackoverflow.com/q/42014484
-        attributions = request.getfixturevalue(attributions)
-        mock_rounding_difference.return_value = test_diff
-        incoming_email = 'a@b.com'
-        incoming_attribution = Attribution(
-            incoming_email, attributions[incoming_email]
-        )
-        other_attributions = attributions.copy()
-        other_attributions.pop(incoming_attribution.email)
-        original_share = incoming_attribution.share
-        correct_rounding_error(attributions, incoming_attribution)
-        assert (
-            attributions[incoming_attribution.email]
-            == original_share - test_diff
-        )
-
-    @pytest.mark.parametrize(
-        "attributions, test_diff",
-        [
-            ("normalized_attributions", Decimal("0")),
-            ("excess_attributions", Decimal("0")),
-            ("shortfall_attributions", Decimal("0")),
-            ("normalized_attributions", Decimal("0.1")),
-            ("excess_attributions", Decimal("0.1")),
-            ("shortfall_attributions", Decimal("0.1")),
-            ("normalized_attributions", Decimal("-0.1")),
-            ("excess_attributions", Decimal("-0.1")),
-            ("shortfall_attributions", Decimal("-0.1")),
-        ],
-    )
-    @patch('oldabe.money_in.get_rounding_difference')
-    def test_existing_attributions_are_unchanged(
-        self, mock_rounding_difference, attributions, test_diff, request
-    ):
-        attributions = request.getfixturevalue(attributions)
-        mock_rounding_difference.return_value = test_diff
-        incoming_email = 'a@b.com'
-        incoming_attribution = Attribution(
-            incoming_email, attributions[incoming_email]
-        )
-        other_attributions = attributions.copy()
-        other_attributions.pop(incoming_attribution.email)
-        correct_rounding_error(attributions, incoming_attribution)
-        attributions.pop(incoming_attribution.email)
-        assert attributions == other_attributions
-
-
-class TestInflateValuation:
-    @patch('oldabe.money_in.open')
-    def test_valuation_inflates_by_fresh_value(self, mock_open):
-        amount = 100
-        valuation = 1000
-        new_valuation = inflate_valuation(valuation, amount)
-        assert new_valuation == amount + valuation
-
-
-class TotalAmountPaidToProject:
-    @patch('oldabe.money_in.get_existing_itemized_payments')
-    def test_total_amount_for_a_all_attributable(
-        self,
-        get_existing_itemized_payments,
-        itemized_payments,
-        new_itemized_payments,
-    ):
-        get_existing_itemized_payments.return_value = itemized_payments
-        total = total_amount_paid_to_project('a@b.com', new_itemized_payments)
-        assert total == 194
-
-    @patch('oldabe.money_in.get_existing_itemized_payments')
-    def test_total_amount_for_c_one_non_attributable(
-        self,
-        get_existing_itemized_payments,
-        itemized_payments,
-        new_itemized_payments,
-    ):
-        get_existing_itemized_payments.return_value = itemized_payments
-        total = total_amount_paid_to_project('c@d.com', new_itemized_payments)
-        assert total == 30
-
-
-class TestCalculateIncomingInvestment:
-    @pytest.mark.parametrize(
-        "prior_contribution, incoming_amount, price, expected_investment",
-        [
-            (Decimal("0"), Decimal("0"), Decimal("100"), Decimal("0")),
-            (Decimal("0"), Decimal("20"), Decimal("100"), Decimal("0")),
-            (Decimal("0"), Decimal("100"), Decimal("100"), Decimal("0")),
-            (Decimal("0"), Decimal("120"), Decimal("100"), Decimal("20")),
-            (Decimal("20"), Decimal("0"), Decimal("100"), Decimal("0")),
-            (Decimal("20"), Decimal("20"), Decimal("100"), Decimal("0")),
-            (Decimal("20"), Decimal("80"), Decimal("100"), Decimal("0")),
-            (Decimal("20"), Decimal("100"), Decimal("100"), Decimal("20")),
-            (Decimal("100"), Decimal("0"), Decimal("100"), Decimal("0")),
-            (Decimal("100"), Decimal("20"), Decimal("100"), Decimal("20")),
-            (Decimal("120"), Decimal("0"), Decimal("100"), Decimal("0")),
-            (Decimal("120"), Decimal("20"), Decimal("100"), Decimal("20")),
-        ],
-    )
-    @patch('oldabe.money_in.total_amount_paid_to_project')
-    def test_matrix(
+class TestPlaceholder:
+    def test_something(
         self,
-        total_amount_paid_to_project,
-        prior_contribution,
-        incoming_amount,
-        price,
-        expected_investment,
     ):
-        email = 'dummy@abe.org'
-        payment = Payment(email, incoming_amount)
-        # this is the total amount paid _including_ the incoming amount
-        total_amount_paid_to_project.return_value = (
-            prior_contribution + incoming_amount
-        )
-        result = calculate_incoming_investment(payment, price, [])
-        assert result == expected_investment
+        assert True

From acaea38e7447bc6007ed3484b5ba25fc209fe8a5 Mon Sep 17 00:00:00 2001
From: apromessi 
Date: Tue, 1 Oct 2024 19:37:23 -0700
Subject: [PATCH 74/90] update docs to include debts and advances, add logo

---
 docs/assets/img/logo.png | Bin 0 -> 8140 bytes
 docs/oldabe.scrbl        |  80 ++++++++-------------------------------
 2 files changed, 16 insertions(+), 64 deletions(-)
 create mode 100644 docs/assets/img/logo.png

diff --git a/docs/assets/img/logo.png b/docs/assets/img/logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..a830defc5fbc96280c3aa2ba0ee5d2eae3063c91
GIT binary patch
literal 8140
zcmX|G2{hE-`^I3#K4Zyb$r7@gWC>Y@v2SUSC0R-&vhNy=eV4M73R%lugsfwWjF?I(
zWXVn#vakO;`hEYM<8;pb%;(Zh1qCG({L#W_
z!HDzLMJn(g)JNY~8#+eEL5~#1O~OpT5S@pfr8fnI^l9=RWk7+t9~h)|LK~wgDBdJ9
z?Ay~&Q1DsnW6)OuhL&DHuW$@De^B(8qoPuPBRZ+Ma|B)3CB*$42y7imA};B#0%Xtv
zaXR-AHwDZSepEhq;Bgk|%CFEU`bHw|6%HE1AM<5(>QiaS%E_BbzDHZVJE38@NveNtf>Cb0y%7uRQBg}w?wsxe@nKMQ6KdYQnIy*WTf7bghPE8Xs_WGYGy1&cF
z7~HgPkJ8?n|9a;A&8{TQrKKfzxjSEky4u=uA5`V1=7r@&)y_&SoEKN>>HYMHq3-Aq
z7arbzm6J1Lpfw!YW%B&I=Wwoee0pSLlNhBTp7y9Kna}z|M@IlPEiLWF%9wg^z<;Yva;@iAg(~m8Ks@?-1){zOLa!5(D+#m
z-H`6TH6;hWeM@2Bls+$QUDMm*)SuNdzp&6O#7J$_O-(~1h1=vH;5^aX2Lkl;^jym$
z@4ZL1S10{5pFNvJFui}Q5f5S(tQDMa{H@q$gbW!Kk`vUX-bcT6g=xUh#iiIT+gHPk
zHUH!O>FzTo&$k^m|NK-R`tf5#IKWSXVZJ?@g&Q5s1J?MgP*YrNFx#DS`e0k8V>{Xt
z7IIOJSyi=ZRue16psg?0TUu83P1)~{MgRQQqUfhjo{-#6g+Vco-@JR*1mX3Ngi`9g
z5V_G5^zKs_4TEN;{M~e8B#dT_@4Ai6<(H78X?}{=Lv?T8%C>)crf9k_@cOd&)^Lj;
zV+7@nL^za1jq2YNSEFni{K_^;EvvpOI1_W8px_i!?FlE(-RASbzu}?gcQgWN$Dwax
zvaIAWjz6C}AtOsiSI&dQF`rKlr^t^{+L!3pP2!pFw^7cAHy((d7cqJQXZ|<$jhmM)
zUAlh$g^&e(Sfq;g56XYT=2~9f-rSyA?;ej&hmTK$Bs@5tONbzh2?*E@U}JkM*oL<|
zEiM*)K&0#fNvL*PGcfS|C!_5l+k0X8rV)>7o|xg}}9r)KJPe6skAI#nanc
zh3%wqX?D)i((=>)58oXuKgdL(8e@X=4Gq0<%t|0fgSPOqv$NLVf~^kY!&y+d
zYPq?&I-1AH(6lmswEoq!nVK(zRsMO`n8tsje=8i4`P9{(qflT|
zc%*iJSBRCC_N1BW?`1gnz{*?lxB9bGZCM+`
z1;op5G`XidU^at>RB-rmEpKh
z#tal{D#js!SH+Ld=Lh(tw6{7Ni-Rn-iziS3Uz@0S{Uo6}1*K*!`)h=25jX)|b}hk_
z##iO#-*L>TA;W}h$Tmsqe}l3K4b|r{
zNTd%2YgOYK$e>KJU}iM98WkBGZ3+@oIT!w}prC-3_!3NER4iLsSxGdqwrlVxj`e}u$!L9()ZXsx+cvDy)knp#enOtsjQBtRhZ8m%4do-?1OOC)fC`WgX3D86iI*)WJb4=b@QT}Yef`)JjNohV
z4Jv|sRZjiPk<4o85UoqmfvUlqMk2?#`MoRdD3
zc1kFI-;>>;yW@ESZ4n#MFE6I%o3H`kulqLvIh{EF_p{8p;H~A=2%Np{RC{l4Dae*f
zR5nx=%=k-1u`GOQuDu5_Ag^$`Qgw1^Es!!j%#(1u|5a88<(dy8gg>*
za>Bx$zJzqC$NL8d8OdDo;%@5E2tYViJ#lw1nfTH4H*xp%R
z@|m!!$DByN^_q29(IdS9oBr)T2i;L=c
zr&x_S%DjHmxIJUBx2uEOPfktwWCiKTU9Ik!_>jn5MpwS_Ks2_cB^0*t(7D(-4N@)<
zD;l4kZho-Ra&&l7!$|IObq~&on1l8N$e|VSku=esM-mW?%e7=iqtTSdgVE*X%ZAHe
zsyWgj+Mgw(J32ZJ!F|!9j$V)E!#??YJLzI7riup?qF$I9V`TTo7{uw#rlwIy+#~07
z$dUPrckedtx8d;uZ8_H}Zgkm#upMpkycCVStwdrxDJv9(Lixb<>fXN>{@YN~qM{{g
zkZZ#ne?rR20ExU1aVI}3D=WB%K-d^n{PzffK-eO-C$x~-hK40UKi@k3Rj(-&0*hs|
zlZi7MZ4}&|MMb4ySWHaJMqU2jOBET3?Om@@jN%c8gvx>*T$)C9Hnz#~?xa!jHW~L>
zXLomZ_YY}^kT?o_T3hGb+}uQu>yz+Yp`mF~7hmZ3BhSn(2}Q-)yw%m!gR5`iPmnjN
zrKi_@fl9p;M#IR+Xi#MIl+|y#hvR+oWxSUirrH2!+iwq`<+87j4~d}Xd3p)ZXXV*<
z?w+2W5Z_+bxJLsv^!_WOZmXa`%D8;a?c}9WuZzh=F^yxxf(AiP4-
z(dN!*Q?R+jweshKuP+x>VIqUi2)eSu;gzrI>#d@{TZ+a#y0WdEH$C+3ZVQ=M(zj|7
zMMr-9x_Jld4wBo^sQA?@;;&lw>{&bxk_2)>(&Kk4FQZ!R?MLfusLh&
z-coHzZ*MO>z=c@+2q1`)Opd-@%ifPer)B@ZS)`KiZjAMY^GF1m=2Tto_`u5Q;~3ueJDd(2p4
z<3v9X4-yq1rko&dS`j#mV4(hFx=7q3K!22!ls*BzX$lGf6XJxjvNFfjt5+?U9X=03
z5B~hD>Pt^ge^1Wgi0JruL;4%$F?#OI+knnC02Y)6r;TDn2z`l(i8%`vV-2lb8EY9E
zX=|g(w$xycI+A@`Ue28D4S&3S6$|QQ-&i3v9|}|1sEpkY@aO7$Fs)$*{+*)6*1;*i
z*1C@cSRId@I&~@<)c7bE=Q89rGCdsT=-?1GCeLrMI`=tG#De)ZyV^mFXy>m6f44V0
z7*SPK)$4RzW^t93l^?+_u4G8tTsnR|gPB+P`tFa>bU+Wk0fONv?~J+A*4}=^0FF9n
zZ7P$MVaoD4T5Ui;Pmm4Fm7*a
zWvkI|Df8gj)$ptffetz(FCACXnEJMZ63pT)*V@{&q0O?@7bQVKhkbA$oVziV2P+_@bgm==*3cNV;c^$uMOTj;efaPpF?oD>B-e2qj}sq`<<{0G7LXLYXL>Es
zWECLQXkcPu{!#g{s6m)?MD0XN9r;STQK@&waCvxp?_K5SSIZHltA~$HH>@;UYm(nl
zbz+&HpXaUQp%$Ez4aj))>eU1_HU=So_Rg1lDUcWlpOcLQz!I(58q3725)u-!(lRo*
z?UY~|L`RbJe!2e4%uJnaQ4G*d56%F+L~TR65+3Rdpcv43((0<&!WSyAO>L5zxP*kS
zXS~|#mkQgKJTNod2N4qU<;$gZN(gNSz}k~$Qiq3!1h5nSIo53?p#RE1OO63ng)W{X1iR
zp7h5*Z)_A27JjcU#QN?YDZaZ8RENpq=_@7Ll6X~`2flo1vwnt0a&xPz+~4(4l!T_x
zE6>Hf1tr65-^3sS_t04oh->3deSI}iq*LHj1+GQq=_c2EX5SEwz*RbPf#Rd`$^B-`
zG`|BHdfif1R(44K?)N_e_WY#$FODu$gtCvikCfX`36=cE&D6T^G{O28tzO3d?
zfPpiSi*kul`QQtmi5{SGUNSXLH(-j_G-hKpaSRPvM*RNlbz=TR51Nz*>
zk~kvqqAeBQiNyT-dz%^b3=G^3N-;*nm*zBSX=(L|T=GE}BrLA*3s4*RySqRZouyO$
zQOqwX=_+o|&(D=4r=Z}6N)L~r=5=*^4NRDts;UO<@%ERiy-&~0o&hnTQy%GIMIg>*
zqTq}X`}5P&?;r#;CXBgOmNZxgATK&TeiTc@M^jDA5nr~`jItfCFctSGHFe6t4}w_1Q54LBYX!O_Wa09lRTZo!|qR?+%R85$@kO-
z@ga$q8;yiT73Nh`*k)yB4yDx`k>{&#++XeDOZ2QT3SeYnB7i%(hwi0_dz8~IErY}1
z05kP)mfi39N|um}@(}VX4Hrmvp>pn{7vfQHxN*c-&CA_N!20qTpPm3n
zq6!;r&QFSr)E))ilBWAeojp_)0{A-c{Vw!osj8%+;3rb>73o)ZGC5ge^GiV$rKzc@
zscrCG^z-`N5j%XX%t~HSN~-MKkr#LMmIX5fa01P@i&p+DEma`4FVZCr0caG8h>F7A
z-(847pH3a1}-rxnp;W&SjbtLNEqLu>(95H&>;zs)vzIFOFFkWQOJ|pmJ*wWI{dz}#XQ`o38UF2=f*w$xyMaj&saD+u9AiSP9jz-AmIEFwNSG~fzI{o*ZVZ~5~lh2f$jVl)Mr~=-6C*l_){)tJbwI@ddMBr
zZrIRVJg1DG910~WOltiMGcU8+kgA>~SJ(WHBSlZHg4)D~pw-M1BdhV>oB{u=M`}k@
zC<}0RHZwrzKFihI`?bJy?ES&ou4Bt60sj(h)3_*yz_$L_-w!$W&+LME&-P~@soL`k
zW%dI3#5^!K7!C#Iko{)`KcKGj=+p>!sxJkG8N29z!*cScPv@yN>vgjG0`bmEcWeS_
z6Nn(&&jCi(K&=Mvww{}xZ!$76@(DlofpYCd{M}0A?eGsyN8nKWp`Tc?wWwkMO{EV?M-lq!F!u-
zDga~+%)+2TL-NI&{>Mn
zi_4N@x{dZWmt@laanFluT;R$)Git*V
z1=?D_e)UckU$M&()xU8Q7|v0wA=`8*Ec0(230G?F$izKjKkEdbGYg0TM{4XG9EW#z
zU9Si|DB(*Fe~|wQJy>B|>+S2Ss!y^P#psYVCr}=)WJ2s{^jop}V7@53kei!31Bf{m
zdp;l>8Dwmm2ik#af5kI%3)mcm?NLlB8YD@o6x_$hXWYlf`dhv1t`|3UvU74O;z1n6
z$sb#oLYBllz$U-ss6-kV7?=Z1F?*M)+n-Lsw*@fBPVyxpJ1u0dxtK7)v_JGHM8DeQ
z+k&#a9TKtW4BX@vj-5kFZT$uDXhqQExzpLz#e!6D{UJc^q_yEM3jRR>E-4QTOoLz3
zBn7~Un06YtfghxA!QOrM2DuT~2JBZk!h~B~T%6G+Xmh!Mwuya(?aY^{BI%um3;TCT
zGTTm+nR0j5a)8nsP$WKhr-BGFA6mxaD02(912c
zfd^wE-Yp7=lU^V$E9731cgqoBP(A*2g8@*;fzKr?d5~;Zlo}1O*k;S?*PjOi#jOtb
z>1p+&{l7-Bm`f%|Oa)zxB9!Y
zwd7hw7Uzi*3e&(v$YWz;`?aIW2PFTD$JVmV;u+JdCmw^NLvl&pz<0gj=jT@iWM|Sy
zcHR+Sv?7wW%^|How`+lpP9X8;&z%d`gaSv#N|I(gd1P;O3s{b|kPEz6NW#K`%Tj5n
zWXWV#a_Ae-?Wiy>wKy9(+89{!-t;3c(BNmAf;N;}BUc5F#0d9?MkfP$&4BwAF1TD_
zzDB#uTKHsdYyZsJk;jGKbz^|PKO3p|G*yR>++p9+4^G`v&D$|`;N#2wgvDa-0*h+>
z0CZmV+YUafep|m&&1Ta{8YoeKX
zx%zk^Ex>Sa-~AmgljNPfmK+n*x0VmYB_G*;&VeEjr#d@3bpY6|ba2j)J85;7R9$c0
z^tJ}BSCpt39H=AHQ|j3LkHX7<)2ceh1c*vW$r~9OF7E&BS2ukDI+{A5ne4saoQX*a
zLxL{mvZ<;nFr?>jnjOYK&;kun@OR-`Nz1g%aGhcEnJG4ZnMaQv1=$1Zcy0nM7e6#)
zwFFqXzfGNoy&j<4?kQU%N}hgx&7?OnXUJ`x@wKc7)qUQu>BDM18y|}99Z;U@c
zcQgwuj6~{hDz$p&at}bnjw5Vwa&q!l=N0B7hG8t$d}b)B3bfrR^tDYe<(hV3{|6E3
BK@I=_

literal 0
HcmV?d00001

diff --git a/docs/oldabe.scrbl b/docs/oldabe.scrbl
index 14cc7c0..0615306 100644
--- a/docs/oldabe.scrbl
+++ b/docs/oldabe.scrbl
@@ -1,7 +1,14 @@
 #lang scribble/manual
 
+@require[racket/runtime-path]
+
 @title{Old Abe: The Accountant for All of your ABE Needs}
 
+@(define-runtime-path logo-path "assets/img/logo.png")
+@(if (file-exists? logo-path)
+     (image logo-path #:scale 1.0)
+     (printf "[WARNING] No ~a file found!~%" logo-path))
+
 @table-of-contents[]
 
 @section{Intro}
@@ -25,6 +32,8 @@ Old Abe considers precisely three inputs in doing all of its accounting, and it
 
 @item{Attributions -- an association of contributor to percentage of value allocated from the value represented by the project as a whole.}
 
+@item{Instruments -- an association of an instrument to percentage of value allocated from the value represented by the project as a whole.}
+
 @item{Price -- a generic "fair market value" provided by the project to its users (Like many concepts, this concept named price has a distinct role in ABE from its traditional role in capitalism).}
 
 @item{Valuation -- the assessed present value of the project as a whole.}
@@ -34,81 +43,24 @@ All of these are determined through the process of Dialectical Inheritance Attri
 
 @section{Accounting Flows}
 
-There are several actions ("accountable actions") pertaining to a project that trigger accounting by Old Abe. These are:
+Current accounting flows are a mix of manual and automated actions. Old Abe is not directly connected to any financial systems, so its primary role is to run the accounting logic, keep track of any project investors, and tell the maintainer how much to pay project contributors. It is up to the maintainer to record incoming payments (@code{abe/payments/}) and outgoing payouts (@code{abe/payouts/}).
 
 @itemlist[
 #:style 'ordered
 
-@item{Work is done for the project.}
+@item{Recording a Payment - triggers a GitHub Action that runs the accounting logic (details below) and produces a report of all Outstanding Balances as a GitHub Issue. The maintainer can refer to the Issue to find out how much to pay.}
 
-@item{A financial contribution is made to the project.}
-
-@item{An appointed project representative fulfills a payout to a contributor.}
-
-@item{A "fiat" change is made to one of the inputs, i.e. either attributions, price or valuation, which is typically a resolution by DIA.}
+@item{Recording a Payout - triggers a GitHub Action that simply updates the Outstanding Balances issue to reflect the updated amounts owed.}
 
 ]
 
-We will learn more about each of these, in turn.
-
-@subsection{Work}
-
-Work done could be either labor, capital, or ideas, as defined in the @hyperlink["https://github.com/drym-org/finance/blob/main/finance.md"]{ABE financial model}. Regardless of what kind of work it is, its appraisal takes the form of an "incoming attribution," which is an association of a set of contributors to percentage of value contributed, as judged in related to existing attribution allocations in the project.
-
-Old Abe will account this by "renormalizing" the attributions to total to 100% after incorporating the fresh values.
-
-TODO: flesh out
-
 @subsection{Payment}
 
-When a payment comes in, we first pay out any @tech{instruments}. Then, with the remaining amount, we pay project contributors.
-
-TODO: flesh out
-
-@subsection{Payout}
-
-TODO: flesh out
-
-@subsection{Fiat Change in Inputs}
-
-TODO: flesh out, including backpropagation
-
-@section{Modules}
-
-The accounting flows mentioned earlier correspond to distinct modules that handle them.
-
-@subsection{Money In}
-
-This module handles incoming payments.
-
-First it finds all payments that have not already been processed, that
-is, which do not appear in the transactions file.
-
-For each of these payments, it consults the current attributions for
-the project, and does three things.
-
-First, it figures out how much each person in the attributions file is
-owed from this fresh payment, generating a transaction for each
-stakeholder.
-
-Second, it determines how much of the incoming payment can be
-considered an "investment" by comparing the project price with the
-total amount paid by this payer up to this point -- the excess, if
-any, is investment.
+When someone makes a payment to a project, Old Abe allocates portions of that payment to project contributors and creates a report that tells maintainers how much money the project owes to each individual contributor. We'll get deep into the weeds of how it does that in a moment. If the incoming payment represents an investment (that is, it brings the payer's total amount paid above the project price), the payer is considered a project contributor. The system adds them to the attributions file with a share equal to their investment (or increases their pre-existing attributive share). The project valuation is increased by the investment amount and all existing attributive shares are diluted so that percentage shares still add up 100%.
 
-Third, it increases the current valuation by the investment amount
-determined, and, at the same time, "dilutes" the attributions by
-making the payer an attributive stakeholder with a share proportionate
-to their incoming investment amount (or if the payer is already a
-stakeholder, increases their existing share) in relation to the
-valuation.
+Now, let's talk about how Old Abe allocates an incoming payment.
 
-@subsection{Money Out}
+First, we pay off any processing fees (found in @code{instruments.txt}). These fees have fixed percentages that apply to every incoming payment and do not get diluted by investments. They are somewhat analogous to credit card fees. They go towards the Old Abe system itself and to those who have contributed to the DIA process for this project.
 
-This module determines outstanding balances owed.
+Next, we divide the remainder among the contributors in the attributions file. Ideally, this is as simple as dividing the amount according to each contributor's attributive share. However, sometimes certain contributors are temporarily unpayable (e.g. they might not have provided their payment information yet, etc.). In that case, we record the amount owed to that contributor as a "debt" so that the project can pay them later. To avoid having money sitting around in maintainers' accounts, we divide any amount left over among payable contributors, according to attributive share. Any amount we pay someone in excess of what we owed them originally, we record as an "advance." The idea here is that when someone eventually becomes payable, we can prioritize paying off the debt we owe them by allocating money to them first whenever a new payment comes in. Anyone who has been accumulating advances will receive a little less than their attributive share until their total advance amount has been "drawn down" and the scales have been balanced between debts and advances.
 
-First, it reads all generated transactions from payments that have come in to
-determine the amount owed to each contributor.  Then, it looks at all recorded
-payouts to see the total amounts that have already been paid out.  It then
-reports the difference of these values, by contributor, as the balance still
-owed.

From cdbd15af8a8e71bea66bff0a9b6f219dd9478914 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 19:56:30 -0700
Subject: [PATCH 75/90] whitelist .github path for grep search

---
 .ignore | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .ignore

diff --git a/.ignore b/.ignore
new file mode 100644
index 0000000..1902214
--- /dev/null
+++ b/.ignore
@@ -0,0 +1 @@
+!/.github/

From 2e3af3a5146a2163e9499ab624286ab612deed26 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 19:57:09 -0700
Subject: [PATCH 76/90] try bumping python in workflows to fix GA build failure

---
 .github/workflows/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d4b7ccc..a0cfc91 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,8 +11,8 @@ jobs:
         strategy:
             matrix:
                 py:
-                    - "3.9"
-                    - "pypy3.9"
+                    - "3.12"
+                    - "pypy3.12"
                 os:
                     - "macos-latest"
                 architecture:
@@ -42,7 +42,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.9
+                  python-version: 3.12
                   architecture: x64
             - name: Install dependencies
               run: make build-for-test
@@ -56,7 +56,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.9
+                  python-version: 3.12
                   architecture: x64
             - name: Install dependencies
               run: make build

From d5cbcf3a12f92e2ecffe9cf15edac17c8ed48958 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:02:05 -0700
Subject: [PATCH 77/90] use python 3.11 to avoid setuptools issue

---
 .github/workflows/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a0cfc91..96fe492 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,8 +11,8 @@ jobs:
         strategy:
             matrix:
                 py:
-                    - "3.12"
-                    - "pypy3.12"
+                    - "3.11"
+                    - "pypy3.11"
                 os:
                     - "macos-latest"
                 architecture:
@@ -42,7 +42,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.12
+                  python-version: 3.11
                   architecture: x64
             - name: Install dependencies
               run: make build-for-test
@@ -56,7 +56,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.12
+                  python-version: 3.11
                   architecture: x64
             - name: Install dependencies
               run: make build

From c34cff5b94c337f0f6f252cfbef8eb224440791a Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:09:33 -0700
Subject: [PATCH 78/90] blacken..

---
 oldabe/constants.py               |   4 +-
 oldabe/models.py                  |   7 +-
 oldabe/money_in.py                |  39 ++++++--
 oldabe/money_out.py               |   9 +-
 oldabe/parsing.py                 |   1 +
 oldabe/repos.py                   |  32 ++++--
 tests/integration/fixtures.py     |  14 +--
 tests/integration/old_abe_test.py | 155 ++++++++++++++++++------------
 tests/unit/money_in_test.py       |   1 +
 9 files changed, 165 insertions(+), 97 deletions(-)

diff --git a/oldabe/constants.py b/oldabe/constants.py
index d943070..2a71c61 100644
--- a/oldabe/constants.py
+++ b/oldabe/constants.py
@@ -13,7 +13,9 @@
 TRANSACTIONS_FILE = os.path.join(ABE_ROOT, 'transactions.txt')
 DEBTS_FILE = os.path.join(ABE_ROOT, 'debts.txt')
 ADVANCES_FILE = os.path.join(ABE_ROOT, 'advances.txt')
-UNPAYABLE_CONTRIBUTORS_FILE = os.path.join(ABE_ROOT, 'unpayable_contributors.txt')
+UNPAYABLE_CONTRIBUTORS_FILE = os.path.join(
+    ABE_ROOT, 'unpayable_contributors.txt'
+)
 ITEMIZED_PAYMENTS_FILE = os.path.join(ABE_ROOT, 'itemized_payments.txt')
 PRICE_FILE = os.path.join(ABE_ROOT, 'price.txt')
 VALUATION_FILE = os.path.join(ABE_ROOT, 'valuation.txt')
diff --git a/oldabe/models.py b/oldabe/models.py
index d8b0904..8c08bd0 100644
--- a/oldabe/models.py
+++ b/oldabe/models.py
@@ -48,6 +48,7 @@ def is_fulfilled(self):
     def amount_remaining(self):
         return self.amount - self.amount_paid
 
+
 # These are not recorded (yet?), they just represent an intention to record a payment
 @dataclass
 class DebtPayment:
@@ -86,10 +87,10 @@ class Payment:
 @dataclass
 class ItemizedPayment:
     email: str
-    fee_amount: Decimal # instruments
-    project_amount: Decimal # attributions
+    fee_amount: Decimal  # instruments
+    project_amount: Decimal  # attributions
     attributable: bool
-    payment_file: str # acts like a foreign key to original payment object
+    payment_file: str  # acts like a foreign key to original payment object
 
 
 @dataclass
diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index ede7f4e..2219391 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -8,17 +8,38 @@
 from itertools import accumulate
 from typing import Iterable, List, Set, Tuple
 
-from .accounting_utils import (assert_attributions_normalized,
-                               correct_rounding_error)
-from .constants import (ACCOUNTING_ZERO, ATTRIBUTIONS_FILE, DEBTS_FILE,
-                        PRICE_FILE, VALUATION_FILE)
+from .accounting_utils import (
+    assert_attributions_normalized,
+    correct_rounding_error,
+)
+from .constants import (
+    ACCOUNTING_ZERO,
+    ATTRIBUTIONS_FILE,
+    DEBTS_FILE,
+    PRICE_FILE,
+    VALUATION_FILE,
+)
 from .distribution import Distribution
-from .models import (Advance, Attribution, Debt, DebtPayment, ItemizedPayment,
-                     Payment, Transaction)
+from .models import (
+    Advance,
+    Attribution,
+    Debt,
+    DebtPayment,
+    ItemizedPayment,
+    Payment,
+    Transaction,
+)
 from .parsing import serialize_proportion
-from .repos import (AdvancesRepo, AllPaymentsRepo, AttributionsRepo, DebtsRepo,
-                    InstrumentsRepo, ItemizedPaymentsRepo, TransactionsRepo,
-                    UnpayableContributorsRepo)
+from .repos import (
+    AdvancesRepo,
+    AllPaymentsRepo,
+    AttributionsRepo,
+    DebtsRepo,
+    InstrumentsRepo,
+    ItemizedPaymentsRepo,
+    TransactionsRepo,
+    UnpayableContributorsRepo,
+)
 from .tally import Tally
 
 
diff --git a/oldabe/money_out.py b/oldabe/money_out.py
index 53e501a..01c36a3 100755
--- a/oldabe/money_out.py
+++ b/oldabe/money_out.py
@@ -58,8 +58,7 @@ def prepare_debts_message(outstanding_debts: dict):
 
 
 def prepare_advances_message(advances: dict):
-    """ A temporary message reporting aggregate advances, for testing purposes.
-    """
+    """A temporary message reporting aggregate advances, for testing purposes."""
     if not advances:
         return "There are no advances."
     advances_table = ""
@@ -93,7 +92,7 @@ def combined_message(balances_message, debts_message, advances_message):
 
 
 def compile_outstanding_balances():
-    """ Read all accounting records and determine the total outstanding
+    """Read all accounting records and determine the total outstanding
     balances, debts, and advances for each contributor.
     """
     # owed = read_owed_amounts()
@@ -102,7 +101,9 @@ def compile_outstanding_balances():
     balances = owed - paid
     balances_message = prepare_balances_message(balances)
 
-    outstanding_debts = Tally((d.email, d.amount_remaining()) for d in DebtsRepo())
+    outstanding_debts = Tally(
+        (d.email, d.amount_remaining()) for d in DebtsRepo()
+    )
     debts_message = prepare_debts_message(outstanding_debts)
 
     advances = Tally((a.email, a.amount) for a in AdvancesRepo())
diff --git a/oldabe/parsing.py b/oldabe/parsing.py
index c79ddc8..4f3e6ae 100644
--- a/oldabe/parsing.py
+++ b/oldabe/parsing.py
@@ -1,6 +1,7 @@
 from decimal import Decimal
 import re
 
+
 def parse_percentage(value):
     """
     Translates values expressed in percentage format (75.234) into
diff --git a/oldabe/repos.py b/oldabe/repos.py
index b3506aa..1ab97e4 100644
--- a/oldabe/repos.py
+++ b/oldabe/repos.py
@@ -6,19 +6,34 @@
 from decimal import Decimal
 from typing import Any, Generic, Iterable, Iterator, List, Type, TypeVar
 
-from oldabe.constants import (ADVANCES_FILE, ATTRIBUTIONS_FILE, DEBTS_FILE,
-                              INSTRUMENTS_FILE, ITEMIZED_PAYMENTS_FILE,
-                              NONATTRIBUTABLE_PAYMENTS_DIR, PAYMENTS_DIR,
-                              PAYOUTS_DIR, TRANSACTIONS_FILE,
-                              UNPAYABLE_CONTRIBUTORS_FILE)
-from oldabe.models import (Advance, Attribution, Debt, ItemizedPayment,
-                           Payment, Payout, Transaction)
+from oldabe.constants import (
+    ADVANCES_FILE,
+    ATTRIBUTIONS_FILE,
+    DEBTS_FILE,
+    INSTRUMENTS_FILE,
+    ITEMIZED_PAYMENTS_FILE,
+    NONATTRIBUTABLE_PAYMENTS_DIR,
+    PAYMENTS_DIR,
+    PAYOUTS_DIR,
+    TRANSACTIONS_FILE,
+    UNPAYABLE_CONTRIBUTORS_FILE,
+)
+from oldabe.models import (
+    Advance,
+    Attribution,
+    Debt,
+    ItemizedPayment,
+    Payment,
+    Payout,
+    Transaction,
+)
 
 
 def fix_types(row: List[str], Model: type) -> List[Any]:
     """
     Cast string field values from the CSV into the proper types
     """
+
     def _cast(field, value):
         if field.type is Decimal:
             return Decimal(re.sub("[^0-9.]", "", value))
@@ -28,7 +43,8 @@ def _cast(field, value):
             return value
 
     return [
-        _cast(field, value) for field, value in zip(dataclasses.fields(Model), row)
+        _cast(field, value)
+        for field, value in zip(dataclasses.fields(Model), row)
     ]
 
 
diff --git a/tests/integration/fixtures.py b/tests/integration/fixtures.py
index 7cda069..549ea7b 100644
--- a/tests/integration/fixtures.py
+++ b/tests/integration/fixtures.py
@@ -5,13 +5,9 @@
 def abe_fs(fs):
     fs.create_file("./abe/price.txt", contents="10")
     fs.create_file("./abe/valuation.txt", contents="100000")
-    fs.create_file("./abe/instruments.txt", contents=(
-        "old abe,1\n"
-        "DIA,5\n"
-        ))
-    fs.create_file("./abe/attributions.txt", contents=(
-       "sid,50\n"
-       "jair,30\n"
-       "ariana,20\n"
-       ))
+    fs.create_file("./abe/instruments.txt", contents=("old abe,1\n" "DIA,5\n"))
+    fs.create_file(
+        "./abe/attributions.txt",
+        contents=("sid,50\n" "jair,30\n" "ariana,20\n"),
+    )
     return fs
diff --git a/tests/integration/old_abe_test.py b/tests/integration/old_abe_test.py
index c30e8a9..8626ec0 100644
--- a/tests/integration/old_abe_test.py
+++ b/tests/integration/old_abe_test.py
@@ -28,6 +28,7 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         assert "There are no outstanding (unpayable) debts." in message
         assert "There are no advances." in message
 
+
 # TODO: in some cases even though the value is e.g. 1,
 # it's writing out 3 decimal places, like 1.000. We should
 # figure out why this is happening (and whether it's OK)
@@ -42,8 +43,10 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             with open('./abe/transactions.txt') as f:
                 assert f.read() == (
@@ -60,15 +63,14 @@ def test_dilutes_attributions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 10000
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             with open('./abe/attributions.txt') as f:
                 assert f.read() == (
-                    "sid,46\n"
-                    "jair,28\n"
-                    "ariana,18\n"
-                    "sam,8.5\n"
+                    "sid,46\n" "jair,28\n" "ariana,18\n" "sam,8.5\n"
                 )
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
@@ -77,15 +79,19 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
 
             message = compile_outstanding_balances()
 
-            assert ("| Name | Balance |\r\n"
-                    "| ---- | --- |\r\n"
-                    "old abe | 1.00\r\n") in message
+            assert (
+                "| Name | Balance |\r\n"
+                "| ---- | --- |\r\n"
+                "old abe | 1.00\r\n"
+            ) in message
 
 
 class TestPaymentBelowPrice:
@@ -96,8 +102,10 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 1
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             with open('./abe/transactions.txt') as f:
                 assert f.read() == (
@@ -114,14 +122,18 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 1
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             message = compile_outstanding_balances()
 
-            assert ("| Name | Balance |\r\n"
-                    "| ---- | --- |\r\n"
-                    "old abe | 0.01\r\n") in message
+            assert (
+                "| Name | Balance |\r\n"
+                "| ---- | --- |\r\n"
+                "old abe | 0.01\r\n"
+            ) in message
 
 
 class TestNonAttributablePayment:
@@ -132,15 +144,13 @@ def test_does_not_dilute_attributions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file("./abe/payments/nonattributable/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/nonattributable/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             with open('./abe/attributions.txt') as f:
-                assert f.read() == (
-                    "sid,50\n"
-                    "jair,30\n"
-                    "ariana,20\n"
-                )
+                assert f.read() == ("sid,50\n" "jair,30\n" "ariana,20\n")
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
     @patch('oldabe.models.default_commit_hash', return_value='abcd123')
@@ -148,8 +158,10 @@ def test_generates_transactions(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             with open('./abe/transactions.txt') as f:
                 assert f.read() == (
@@ -166,14 +178,18 @@ def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file("./abe/payments/nonattributable/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                "./abe/payments/nonattributable/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
             message = compile_outstanding_balances()
 
-            assert ("| Name | Balance |\r\n"
-                    "| ---- | --- |\r\n"
-                    "old abe | 1.00\r\n") in message
+            assert (
+                "| Name | Balance |\r\n"
+                "| ---- | --- |\r\n"
+                "old abe | 1.00\r\n"
+            ) in message
 
 
 class TestUnpayableContributor:
@@ -182,10 +198,13 @@ def _call(self, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
-            abe_fs.create_file("./abe/unpayable_contributors.txt",
-                               contents=f"ariana")
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
+            abe_fs.create_file(
+                "./abe/unpayable_contributors.txt", contents=f"ariana"
+            )
             process_payments_and_record_updates()
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
@@ -226,9 +245,9 @@ def test_records_advances(self, mock_git_rev, abe_fs):
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         message = compile_outstanding_balances()
-        assert ("| Name | Debt |\r\n"
-                "| ---- | --- |\r\n"
-                "ariana | 19.00\r\n") in message
+        assert (
+            "| Name | Debt |\r\n" "| ---- | --- |\r\n" "ariana | 19.00\r\n"
+        ) in message
 
 
 class TestUnpayableContributorBecomesPayable:
@@ -237,25 +256,35 @@ def _call(self, abe_fs):
         with localcontext() as context:
             context.prec = 2
             amount = 100
-            abe_fs.create_file('./abe/transactions.txt',
-                               contents=(
-                                   "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n"
-                                   "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n"
-                                   "sid,58,1.txt,abcd123,1985-10-26 01:24:00\n"
-                                   "jair,35,1.txt,abcd123,1985-10-26 01:24:00\n"
-                               ))
-
-            abe_fs.create_file("./abe/debts.txt",
-                               contents="ariana,19,0,1.txt,abcd123,1985-10-26 01:24:00\n")
-            abe_fs.create_file("./abe/advances.txt",
-                               contents=(
-                                   "sid,11,1.txt,abcd123,1985-10-26 01:24:00\n"
-                                   "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n"
-                               ))
-            abe_fs.create_file("./abe/payments/1.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
-            abe_fs.create_file("./abe/payments/2.txt",
-                               contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00")
+            abe_fs.create_file(
+                './abe/transactions.txt',
+                contents=(
+                    "old abe,1.0,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "DIA,5.0,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "sid,58,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "jair,35,1.txt,abcd123,1985-10-26 01:24:00\n"
+                ),
+            )
+
+            abe_fs.create_file(
+                "./abe/debts.txt",
+                contents="ariana,19,0,1.txt,abcd123,1985-10-26 01:24:00\n",
+            )
+            abe_fs.create_file(
+                "./abe/advances.txt",
+                contents=(
+                    "sid,11,1.txt,abcd123,1985-10-26 01:24:00\n"
+                    "jair,6.8,1.txt,abcd123,1985-10-26 01:24:00\n"
+                ),
+            )
+            abe_fs.create_file(
+                "./abe/payments/1.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
+            abe_fs.create_file(
+                "./abe/payments/2.txt",
+                contents=f"sam,036eaf6,{amount},1987-06-30 06:25:00",
+            )
             process_payments_and_record_updates()
 
     @time_machine.travel(datetime(1985, 10, 26, 1, 24), tick=False)
@@ -308,6 +337,6 @@ def test_advances(self, mock_git_rev, abe_fs):
     def test_compiled_outstanding_balances(self, mock_git_rev, abe_fs):
         self._call(abe_fs)
         message = compile_outstanding_balances()
-        assert ("| Name | Debt |\r\n"
-                "| ---- | --- |\r\n"
-                "ariana | 0.00\r\n") in message
+        assert (
+            "| Name | Debt |\r\n" "| ---- | --- |\r\n" "ariana | 0.00\r\n"
+        ) in message
diff --git a/tests/unit/money_in_test.py b/tests/unit/money_in_test.py
index 800fba5..cec8fb7 100644
--- a/tests/unit/money_in_test.py
+++ b/tests/unit/money_in_test.py
@@ -2,6 +2,7 @@
 
 # See integration tests for now.
 
+
 class TestPlaceholder:
     def test_something(
         self,

From 98ec6e8590052fdc2829955bea2820604a0cc885 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:15:48 -0700
Subject: [PATCH 79/90] fix remaining lint errors

---
 oldabe/money_in.py  | 8 +++++---
 oldabe/money_out.py | 3 ++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/oldabe/money_in.py b/oldabe/money_in.py
index 2219391..8376615 100755
--- a/oldabe/money_in.py
+++ b/oldabe/money_in.py
@@ -3,7 +3,7 @@
 import csv
 import dataclasses
 import re
-from dataclasses import astuple, replace
+from dataclasses import astuple
 from decimal import Decimal, getcontext
 from itertools import accumulate
 from typing import Iterable, List, Set, Tuple
@@ -176,7 +176,8 @@ def distribute_payment(
 
     # 1. check payable outstanding debts
     # 2. pay them off in chronological order (maybe partially)
-    # 3. (if leftover) identify unpayable people in the relevant distribution file
+    # 3. (if leftover) identify unpayable people in the relevant
+    #    distribution file
     # 4. record debt for each of them according to their attribution
 
     unpayable_contributors = set(UnpayableContributorsRepo())
@@ -200,7 +201,8 @@ def distribute_payment(
     #
     # Create fresh debts for anyone we can't pay
     #
-    # TODO: Make it clearer that some people get debts and the others get N advances (maybe zero)
+    # TODO: Make it clearer that some people get debts and the others
+    # get N advances (maybe zero)
 
     fresh_debts = create_debts(
         available_amount, distribution, payable_contributors, payment
diff --git a/oldabe/money_out.py b/oldabe/money_out.py
index 01c36a3..ad7e9a5 100755
--- a/oldabe/money_out.py
+++ b/oldabe/money_out.py
@@ -58,7 +58,8 @@ def prepare_debts_message(outstanding_debts: dict):
 
 
 def prepare_advances_message(advances: dict):
-    """A temporary message reporting aggregate advances, for testing purposes."""
+    """A temporary message reporting aggregate advances, for testing
+    purposes."""
     if not advances:
         return "There are no advances."
     advances_table = ""

From 6cc1d9545dee793922781cadbc308e6ee43ccad3 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:18:14 -0700
Subject: [PATCH 80/90] lint..

---
 oldabe/models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/oldabe/models.py b/oldabe/models.py
index 8c08bd0..7426b52 100644
--- a/oldabe/models.py
+++ b/oldabe/models.py
@@ -49,7 +49,8 @@ def amount_remaining(self):
         return self.amount - self.amount_paid
 
 
-# These are not recorded (yet?), they just represent an intention to record a payment
+# These are not recorded (yet?), they just represent an intention
+# to record a payment
 @dataclass
 class DebtPayment:
     debt: Debt

From 1cda37517604ac7f7d60c410e5d9c474ab1cfd9b Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:19:10 -0700
Subject: [PATCH 81/90] remove pypy from the matrix for now

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 96fe492..78c0d4d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,6 @@ jobs:
             matrix:
                 py:
                     - "3.11"
-                    - "pypy3.11"
                 os:
                     - "macos-latest"
                 architecture:

From 60fda207936d302745ec896ff217a3723c1ec4d8 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:26:56 -0700
Subject: [PATCH 82/90] remove deprecated `tests_require`

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 386ad35..93f81cd 100644
--- a/setup.py
+++ b/setup.py
@@ -30,6 +30,5 @@
     test_suite='tests',
     install_requires=requirements,
     setup_requires=setup_requirements,
-    tests_require=test_requirements,
     extras_require={'dev': dev_requirements, 'test': test_requirements},
 )

From edb19281cfae1ac87d1adca3b006dbc52bfafff3 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:45:40 -0700
Subject: [PATCH 83/90] try python 3.9 with ubuntu

---
 .github/workflows/test.yml | 12 ++++++------
 setup.py                   |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 78c0d4d..302630c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,9 +11,9 @@ jobs:
         strategy:
             matrix:
                 py:
-                    - "3.11"
+                    - "3.9"
                 os:
-                    - "macos-latest"
+                    - "ubuntu-latest"
                 architecture:
                     - x64
         name: "Python: ${{ matrix.py }}-${{ matrix.architecture }} on ${{ matrix.os }}"
@@ -30,7 +30,7 @@ jobs:
             - name: Run tests
               run: make test-matrix
     coverage:
-        runs-on: macos-latest
+        runs-on: ubuntu-latest
         name: Report coverage
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
@@ -41,21 +41,21 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.11
+                  python-version: 3.9
                   architecture: x64
             - name: Install dependencies
               run: make build-for-test
             - name: Report coverage
               run: make cover-coveralls
     lint:
-        runs-on: macos-latest
+        runs-on: ubuntu-latest
         name: Lint the package
         steps:
             - uses: actions/checkout@v3
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.11
+                  python-version: 3.9
                   architecture: x64
             - name: Install dependencies
               run: make build
diff --git a/setup.py b/setup.py
index 93f81cd..386ad35 100644
--- a/setup.py
+++ b/setup.py
@@ -30,5 +30,6 @@
     test_suite='tests',
     install_requires=requirements,
     setup_requires=setup_requirements,
+    tests_require=test_requirements,
     extras_require={'dev': dev_requirements, 'test': test_requirements},
 )

From 47e3ed16b35a995ef2d962ab281101e3716dcf89 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:47:30 -0700
Subject: [PATCH 84/90] remove tests_require ... again

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 386ad35..93f81cd 100644
--- a/setup.py
+++ b/setup.py
@@ -30,6 +30,5 @@
     test_suite='tests',
     install_requires=requirements,
     setup_requires=setup_requirements,
-    tests_require=test_requirements,
     extras_require={'dev': dev_requirements, 'test': test_requirements},
 )

From 4e61abaf0a32e31ee56ea15768a60cab80de0100 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:52:51 -0700
Subject: [PATCH 85/90] add pypy back

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 302630c..d84a5f9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,6 +12,7 @@ jobs:
             matrix:
                 py:
                     - "3.9"
+                    - "pypy3.9"
                 os:
                     - "ubuntu-latest"
                 architecture:

From 484b7259622086ef2a627d6f0204040b57285cbe Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:54:30 -0700
Subject: [PATCH 86/90] revert to macos to find original error

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d84a5f9..7d01b65 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ jobs:
                     - "3.9"
                     - "pypy3.9"
                 os:
-                    - "ubuntu-latest"
+                    - "macos-latest"
                 architecture:
                     - x64
         name: "Python: ${{ matrix.py }}-${{ matrix.architecture }} on ${{ matrix.os }}"

From a20dba690319d16afe722294333791ecdb21300e Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 20:59:16 -0700
Subject: [PATCH 87/90] try ARM64

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7d01b65..c4b13d9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,7 +16,7 @@ jobs:
                 os:
                     - "macos-latest"
                 architecture:
-                    - x64
+                    - ARM64
         name: "Python: ${{ matrix.py }}-${{ matrix.architecture }} on ${{ matrix.os }}"
         runs-on: ${{ matrix.os }}
         steps:

From ab116a354dcf7ceb801b45abbf89ea7692ed235a Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 21:00:10 -0700
Subject: [PATCH 88/90] remove pypy

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c4b13d9..0f4f9dc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,6 @@ jobs:
             matrix:
                 py:
                     - "3.9"
-                    - "pypy3.9"
                 os:
                     - "macos-latest"
                 architecture:

From 8b6ec570f0045640133debd1065022c713a2c741 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 21:09:41 -0700
Subject: [PATCH 89/90] use `pytest` instead of `tox`

---
 .github/workflows/test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0f4f9dc..de0cf56 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,11 +11,11 @@ jobs:
         strategy:
             matrix:
                 py:
-                    - "3.9"
+                    - "3.12"
                 os:
-                    - "macos-latest"
+                    - "ubuntu-latest"
                 architecture:
-                    - ARM64
+                    - x64
         name: "Python: ${{ matrix.py }}-${{ matrix.architecture }} on ${{ matrix.os }}"
         runs-on: ${{ matrix.os }}
         steps:
@@ -28,7 +28,7 @@ jobs:
             - name: Install dependencies
               run: make build-for-test
             - name: Run tests
-              run: make test-matrix
+              run: pytest
     coverage:
         runs-on: ubuntu-latest
         name: Report coverage
@@ -41,7 +41,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.9
+                  python-version: 3.12
                   architecture: x64
             - name: Install dependencies
               run: make build-for-test
@@ -55,7 +55,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.9
+                  python-version: 3.12
                   architecture: x64
             - name: Install dependencies
               run: make build

From bc50f2c3edf9cf7190f8a44ec991988fd80c1b58 Mon Sep 17 00:00:00 2001
From: Siddhartha 
Date: Tue, 1 Oct 2024 21:11:09 -0700
Subject: [PATCH 90/90] use python 3.11

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index de0cf56..cf794a9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,7 +11,7 @@ jobs:
         strategy:
             matrix:
                 py:
-                    - "3.12"
+                    - "3.11"
                 os:
                     - "ubuntu-latest"
                 architecture:
@@ -41,7 +41,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.12
+                  python-version: 3.11
                   architecture: x64
             - name: Install dependencies
               run: make build-for-test
@@ -55,7 +55,7 @@ jobs:
             - name: Setup python
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.12
+                  python-version: 3.11
                   architecture: x64
             - name: Install dependencies
               run: make build