From 5b516802e4ba832157a4809c82369fc06327a576 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Wed, 30 Oct 2024 02:12:01 +0100 Subject: [PATCH 1/8] add one-time-script --- ..._production_duplicates_after_enrollment.py | 50 +++++++++ ..._production_duplicates_after_enrollment.py | 103 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py create mode 100644 tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py diff --git a/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py b/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py new file mode 100644 index 0000000000..872e80aab9 --- /dev/null +++ b/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py @@ -0,0 +1,50 @@ +import logging + +from django.db.models import Count + +from hct_mis_api.apps.household.models import Household + +logger = logging.getLogger(__name__) + + +def remove_production_duplicates_after_enrollment(): + # Exceptions from the further rules + for household in Household.objects.filter( + id__in=[ + "8b9bf768-4837-49aa-a598-5ad3c5822ca8", + "33a7bdf0-650d-49b4-b333-c49a7eb05356", + ] + ): + household.delete(soft=False) + + households_with_duplicates = ( + Household.objects.values("unicef_id", "program") + .annotate(household_count=Count("id")) + .filter(household_count__gt=1) + .order_by("copied_from__registration_data_import") + ) + logger.info(f"Found {households_with_duplicates.count()} households with duplicates") + + for i, entry in enumerate(households_with_duplicates, 1): + unicef_id = entry["unicef_id"] + program = entry["program"] + + households = Household.objects.filter(unicef_id=unicef_id, program=program).order_by("created_at") + + # Keep the first household and delete the duplicates + first = True + households_to_remove = [] + for household in households: + if first: + first = False + continue + if household.payment_set.exists(): + logger.info(f"Skipping {household.id} because it has payments") + continue + else: + households_to_remove.append(household) + for duplicate in households_to_remove: + duplicate.delete(soft=False) + + if i % 100 == 0: + logger.info(f"Processed {i}/{households_with_duplicates.count()} households") diff --git a/tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py b/tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py new file mode 100644 index 0000000000..0edcc9d0a0 --- /dev/null +++ b/tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py @@ -0,0 +1,103 @@ +from django.test import TestCase + +from hct_mis_api.apps.core.fixtures import create_afghanistan +from hct_mis_api.apps.household.fixtures import create_household_and_individuals +from hct_mis_api.apps.household.models import Household, Individual +from hct_mis_api.apps.payment.fixtures import PaymentFactory +from hct_mis_api.apps.program.fixtures import ProgramFactory +from hct_mis_api.one_time_scripts.remove_production_duplicates_after_enrollment import ( + remove_production_duplicates_after_enrollment, +) + + +class TestRemoveProductionDuplicatesAfterEnrollment(TestCase): + def test_remove_production_duplicates_after_enrollment(self) -> None: + business_area = create_afghanistan() + program = ProgramFactory(business_area=business_area) + program2 = ProgramFactory(business_area=business_area) + hh_unicef_id = "HH-20-0000.0001" + household_special_case1, individuals_special_case1 = create_household_and_individuals( + household_data={ + "id": "8b9bf768-4837-49aa-a598-5ad3c5822ca8", + "unicef_id": hh_unicef_id, + "business_area": program.business_area, + "program": program, + }, + individuals_data=[{}], + ) + household_special_case2, individuals_special_case2 = create_household_and_individuals( + household_data={ + "id": "33a7bdf0-650d-49b4-b333-c49a7eb05356", + "unicef_id": hh_unicef_id, + "business_area": program.business_area, + "program": program, + }, + individuals_data=[{}], + ) + household1, individuals1 = create_household_and_individuals( + household_data={ + "unicef_id": hh_unicef_id, + "business_area": program.business_area, + "program": program, + }, + individuals_data=[{}, {}], + ) + household2, individuals2 = create_household_and_individuals( + household_data={ + "unicef_id": hh_unicef_id, + "business_area": program.business_area, + "program": program, + }, + individuals_data=[{}, {}], + ) + household3, individuals3 = create_household_and_individuals( + household_data={ + "unicef_id": hh_unicef_id, + "business_area": program.business_area, + "program": program, + }, + individuals_data=[{}], + ) + PaymentFactory(household=household3) + + household4, individuals4 = create_household_and_individuals( + household_data={ + "unicef_id": hh_unicef_id, + "business_area": program.business_area, + "program": program, + }, + individuals_data=[{}], + ) + household_from_another_program, individuals_from_another_program = create_household_and_individuals( + household_data={ + "unicef_id": hh_unicef_id, + "business_area": program2.business_area, + "program": program2, + }, + individuals_data=[{}], + ) + + remove_production_duplicates_after_enrollment() + + self.assertIsNotNone(Household.all_objects.filter(id=household1.id).first()) + self.assertIsNotNone(Individual.all_objects.filter(id=individuals1[0].id).first()) + self.assertIsNotNone(Individual.all_objects.filter(id=individuals1[1].id).first()) + + self.assertIsNone(Household.all_objects.filter(id=household2.id).first()) + self.assertIsNone(Individual.all_objects.filter(id=individuals2[0].id).first()) + self.assertIsNone(Individual.all_objects.filter(id=individuals2[1].id).first()) + + self.assertIsNotNone(Individual.all_objects.filter(id=individuals3[0].id).first()) + self.assertIsNotNone(Household.all_objects.filter(id=household3.id).first()) + + self.assertIsNone(Household.all_objects.filter(id=household4.id).first()) + self.assertIsNone(Individual.all_objects.filter(id=individuals4[0].id).first()) + + self.assertIsNotNone(Household.all_objects.filter(id=household_from_another_program.id).first()) + self.assertIsNotNone(Individual.all_objects.filter(id=individuals_from_another_program[0].id).first()) + + self.assertIsNone(Household.all_objects.filter(id=household_special_case1.id).first()) + self.assertIsNone(Individual.all_objects.filter(id=individuals_special_case1[0].id).first()) + + self.assertIsNone(Household.all_objects.filter(id=household_special_case2.id).first()) + self.assertIsNone(Individual.all_objects.filter(id=individuals_special_case2[0].id).first()) From f7a9cbc3927df6d9cdc314570fcd1461ec45e6f3 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Wed, 30 Oct 2024 10:05:06 +0100 Subject: [PATCH 2/8] typing --- .../remove_production_duplicates_after_enrollment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py b/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py index 872e80aab9..7b4f5a4a5d 100644 --- a/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py +++ b/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -def remove_production_duplicates_after_enrollment(): +def remove_production_duplicates_after_enrollment() -> None: # Exceptions from the further rules for household in Household.objects.filter( id__in=[ From fd4410906ac9e3b8dded03fad2e52b65cec7998d Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Wed, 30 Oct 2024 11:55:46 +0100 Subject: [PATCH 3/8] ndividual page not accessible by Sudan CO --- pyproject.toml | 2 +- src/frontend/package.json | 2 +- src/hct_mis_api/apps/periodic_data_update/api/views.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 99dca58af0..0644a75878 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ distribution = true [project] name = "hope" -version = "2.12.0" +version = "2.12.1" description = "HCT MIS is UNICEF's humanitarian cash transfer platform." authors = [ { name = "Tivix" }, diff --git a/src/frontend/package.json b/src/frontend/package.json index 66a7aa2a09..c5e62b840d 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "2.12.0", + "version": "2.12.1", "private": true, "type": "module", "scripts": { diff --git a/src/hct_mis_api/apps/periodic_data_update/api/views.py b/src/hct_mis_api/apps/periodic_data_update/api/views.py index 68f8757903..561c840a3b 100644 --- a/src/hct_mis_api/apps/periodic_data_update/api/views.py +++ b/src/hct_mis_api/apps/periodic_data_update/api/views.py @@ -9,6 +9,7 @@ from rest_framework.decorators import action from rest_framework.exceptions import ValidationError from rest_framework.filters import OrderingFilter +from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response from rest_framework.serializers import BaseSerializer @@ -182,7 +183,7 @@ class PeriodicFieldViewSet( GenericViewSet, ): serializer_class = PeriodicFieldSerializer - permission_classes = [PDUViewListAndDetailsPermission] + permission_classes = [IsAuthenticated] filter_backends = (OrderingFilter,) def get_queryset(self) -> QuerySet: From 5b12950a63857d27bf7097124589a6b8b4e08966 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Wed, 30 Oct 2024 14:21:00 +0100 Subject: [PATCH 4/8] remove unnecessary test --- .../test_periodic_data_field_views.py | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py b/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py index 241090716d..496b787455 100644 --- a/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py +++ b/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py @@ -57,55 +57,6 @@ def set_up(self, api_client: Callable, afghanistan: BusinessAreaFactory, id_to_b }, ) - @pytest.mark.parametrize( - "permissions, partner_permissions, access_to_program, expected_status", - [ - ([], [], True, status.HTTP_403_FORBIDDEN), - ([Permissions.PDU_VIEW_LIST_AND_DETAILS], [], True, status.HTTP_200_OK), - ([], [Permissions.PDU_VIEW_LIST_AND_DETAILS], True, status.HTTP_200_OK), - ( - [Permissions.PDU_VIEW_LIST_AND_DETAILS], - [Permissions.PDU_VIEW_LIST_AND_DETAILS], - True, - status.HTTP_200_OK, - ), - ([], [], False, status.HTTP_403_FORBIDDEN), - ([Permissions.PDU_VIEW_LIST_AND_DETAILS], [], False, status.HTTP_403_FORBIDDEN), - ([], [Permissions.PDU_VIEW_LIST_AND_DETAILS], False, status.HTTP_403_FORBIDDEN), - ( - [Permissions.PDU_VIEW_LIST_AND_DETAILS], - [Permissions.PDU_VIEW_LIST_AND_DETAILS], - False, - status.HTTP_403_FORBIDDEN, - ), - ], - ) - def test_list_periodic_fields_permission( - self, - permissions: list, - partner_permissions: list, - access_to_program: bool, - expected_status: str, - api_client: Callable, - afghanistan: BusinessAreaFactory, - create_user_role_with_permissions: Callable, - create_partner_role_with_permissions: Callable, - update_partner_access_to_program: Callable, - id_to_base64: Callable, - ) -> None: - self.set_up(api_client, afghanistan, id_to_base64) - create_user_role_with_permissions( - self.user, - permissions, - self.afghanistan, - ) - create_partner_role_with_permissions(self.partner, partner_permissions, self.afghanistan) - if access_to_program: - update_partner_access_to_program(self.partner, self.program1) - - response = self.client.get(self.url_list) - assert response.status_code == expected_status - def test_list_periodic_fields( self, api_client: Callable, From c9f4eccb665418cdb609f89553cb4d41f17acac2 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Wed, 30 Oct 2024 15:10:29 +0100 Subject: [PATCH 5/8] fixing test --- .../test_periodic_data_field_views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py b/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py index 496b787455..144aaf5a04 100644 --- a/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py +++ b/tests/unit/apps/periodic_data_update/test_periodic_data_field_views.py @@ -137,7 +137,7 @@ def test_list_periodic_fields_caching( etag = response.headers["etag"] assert json.loads(cache.get(etag)[0].decode("utf8")) == response.json() - assert len(ctx.captured_queries) == 11 + assert len(ctx.captured_queries) == 6 # Test that reoccurring requests use cached data with CaptureQueriesContext(connection) as ctx: @@ -146,7 +146,7 @@ def test_list_periodic_fields_caching( etag_second_call = response.headers["etag"] assert json.loads(cache.get(response.headers["etag"])[0].decode("utf8")) == response.json() - assert len(ctx.captured_queries) == 5 + assert len(ctx.captured_queries) == 0 assert etag_second_call == etag @@ -159,7 +159,7 @@ def test_list_periodic_fields_caching( etag_call_after_update = response.headers["etag"] assert json.loads(cache.get(response.headers["etag"])[0].decode("utf8")) == response.json() - assert len(ctx.captured_queries) == 11 + assert len(ctx.captured_queries) == 6 assert etag_call_after_update != etag @@ -170,7 +170,7 @@ def test_list_periodic_fields_caching( etag_call_after_update_second_call = response.headers["etag"] assert json.loads(cache.get(response.headers["etag"])[0].decode("utf8")) == response.json() - assert len(ctx.captured_queries) == 5 + assert len(ctx.captured_queries) == 0 assert etag_call_after_update_second_call == etag_call_after_update @@ -183,6 +183,6 @@ def test_list_periodic_fields_caching( etag_call_after_update_2 = response.headers["etag"] assert json.loads(cache.get(response.headers["etag"])[0].decode("utf8")) == response.json() - assert len(ctx.captured_queries) == 11 + assert len(ctx.captured_queries) == 6 assert etag_call_after_update_2 != etag_call_after_update_second_call From 98269e454fff4400787687d7669a6ab32b4a9bc1 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 31 Oct 2024 11:34:01 +0100 Subject: [PATCH 6/8] Fix detecting duplicated individual without names --- src/hct_mis_api/apps/household/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hct_mis_api/apps/household/models.py b/src/hct_mis_api/apps/household/models.py index 7ab8ae8220..4fb4d04b8e 100644 --- a/src/hct_mis_api/apps/household/models.py +++ b/src/hct_mis_api/apps/household/models.py @@ -1093,6 +1093,7 @@ def get_hash_key(self) -> str: "given_name", "middle_name", "family_name", + "full_name", "sex", "birth_date", "phone_no", From 1111cb08191b31e01c6c2264c9b6a80bff196c7e Mon Sep 17 00:00:00 2001 From: Domenico Date: Tue, 29 Oct 2024 16:54:41 +0100 Subject: [PATCH 7/8] Merge pull request #4378 from unicef/add-household-checks-for-individuals [219858] Prevent errors when individual has no household --- .../needs_adjudication_ticket_services.py | 8 ++-- .../apps/grievance/test_services_utils.py | 47 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/hct_mis_api/apps/grievance/services/needs_adjudication_ticket_services.py b/src/hct_mis_api/apps/grievance/services/needs_adjudication_ticket_services.py index de0065c09d..3350faf786 100644 --- a/src/hct_mis_api/apps/grievance/services/needs_adjudication_ticket_services.py +++ b/src/hct_mis_api/apps/grievance/services/needs_adjudication_ticket_services.py @@ -368,7 +368,7 @@ def mark_as_distinct_individual( individual_to_distinct, ) individual_marked_as_distinct.send(sender=Individual, instance=individual_to_distinct) - household = individual_to_distinct.household - household.refresh_from_db() - if household.active_individuals.count() > 0: - household.unwithdraw() + if household := individual_to_distinct.household: + household.refresh_from_db() + if household.active_individuals.count() > 0: + household.unwithdraw() diff --git a/tests/unit/apps/grievance/test_services_utils.py b/tests/unit/apps/grievance/test_services_utils.py index 08198b31f6..6c6a9ad171 100644 --- a/tests/unit/apps/grievance/test_services_utils.py +++ b/tests/unit/apps/grievance/test_services_utils.py @@ -408,6 +408,53 @@ def test_close_needs_adjudication_ticket_service(self) -> None: assert ind_1.duplicate is False assert ind_2.duplicate is True + def test_close_needs_adjudication_ticket_service_individual_without_household(self) -> None: + user = UserFactory() + ba = BusinessAreaFactory(slug="afghanistan") + program = ProgramFactory(business_area=ba) + + grievance = GrievanceTicketFactory( + category=GrievanceTicket.CATEGORY_NEEDS_ADJUDICATION, + business_area=ba, + status=GrievanceTicket.STATUS_FOR_APPROVAL, + description="GrievanceTicket", + issue_type=GrievanceTicket.ISSUE_TYPE_UNIQUE_IDENTIFIERS_SIMILARITY, + ) + grievance.programs.add(program) + ind_data = { + "given_name": "John", + "family_name": "Doe", + "middle_name": "", + "full_name": "John Doe", + } + ind_1 = IndividualFactory(household=None, program=program, **ind_data) + document = DocumentFactory(individual=ind_1, status=Document.STATUS_INVALID) + _, individuals_2 = create_household( + {"size": 1, "business_area": ba, "program": program}, + ind_data, + ) + ind_2 = individuals_2[0] + + ticket_details = TicketNeedsAdjudicationDetailsFactory( + ticket=grievance, + golden_records_individual=ind_1, + is_multiple_duplicates_version=True, + selected_individual=None, + ) + ticket_details.selected_distinct.set([ind_1, ind_2]) + ticket_details.ticket = grievance + ticket_details.save() + + close_needs_adjudication_ticket_service(grievance, user) + + ind_1.refresh_from_db() + ind_2.refresh_from_db() + document.refresh_from_db() + + self.assertEqual(ind_1.duplicate, False) + self.assertEqual(ind_2.duplicate, False) + self.assertEqual(document.status, Document.STATUS_VALID) + def test_close_needs_adjudication_ticket_service_when_just_duplicates(self) -> None: user = UserFactory() ba = BusinessAreaFactory(slug="afghanistan") From 92b1920aca39f63ca1c6b56327451cc06b40e012 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 31 Oct 2024 17:01:31 +0100 Subject: [PATCH 8/8] upgr version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0644a75878..d84337f4c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ distribution = true [project] name = "hope" -version = "2.12.1" +version = "2.12.2" description = "HCT MIS is UNICEF's humanitarian cash transfer platform." authors = [ { name = "Tivix" },