Merge pull request #4395 from unicef/develop

dev2stg

.github/helpers/.env-selenium

@@ -1,11 +1,11 @@
-REDIS_INSTANCE=redis:6379
 PYTHONUNBUFFERED=1
 SECRET_KEY=secretkey
 ENV=dev
 DEBUG=true
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CACHE_LOCATION=redis://redis:6379/1
+CONSTANCE_REDIS_CONNECTION=redis://redis:6379/0
 DATABASE_URL=postgis://postgres:postgres@db:5432/postgres
 USE_DUMMY_EXCHANGE_RATES=yes
 CELERY_TASK_ALWAYS_EAGER=true

.github/helpers/.env-unit

@@ -12,5 +12,6 @@ KOBO_KF_URL=https://kobo.humanitarianresponse.info
 KOBO_KC_URL=https://kobo.humanitarianresponse.info
 KOBO_MASTER_API_TOKEN=token
 EXCHANGE_RATES_API_KEY=token
-REDIS_INSTANCE=redis:6379
+CACHE_LOCATION=redis://redis:6379/1
+CONSTANCE_REDIS_CONNECTION=redis://redis:6379/0
 PYTHONUNBUFFERED=1

.github/helpers/docker-compose.selenium.yml

@@ -31,7 +31,7 @@ services:
       - backend-web-app:/tmp/
     command: |
       sh -c "
-        cp -r /packages/__pypackages__/3.11/lib/hct_mis_api/apps/web/* /tmp/
+        cp -r /packages/__pypackages__/3.12/lib/hct_mis_api/apps/web/* /tmp/
       "
     restart: "no"
 

development_tools/.env.example

@@ -29,7 +29,11 @@ DATAMART_URL=https://datamart.unicef.io
 EXCHANGE_RATES_API_KEY=
 ADMIN_PANEL_URL=unicorn
 ROOT_ACCESS_TOKEN=test
-CELERY_BROKER_URL="redis://redis:6379/0"
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CONSTANCE_REDIS_CONNECTION=redis://redis:6379/0
+CACHE_LOCATION=redis://redis:6379/1
+
 PROFILING=off
 USE_DUMMY_EXCHANGE_RATES=no
 OPENAPI_URL=127.0.0.1:8080/api/rest/

development_tools/compose.yml

@@ -16,7 +16,6 @@ services:
     env_file:
       - .env
     environment:
-      - REDIS_INSTANCE=redis:6379
       - PYTHONUNBUFFERED=1
     build:
       context: ..
@@ -61,8 +60,6 @@ services:
     image: unicef/hct-mis-backend
     env_file:
       - .env
-    environment:
-      - REDIS_INSTANCE=redis:6379
     volumes:
       - ../src:/code/
       - backend-data:/data
@@ -90,8 +87,6 @@ services:
     image: unicef/hct-mis-backend
     env_file:
       - .env
-    environment:
-      - REDIS_INSTANCE=redis:6379
     volumes:
       - ../src:/code/
       - backend-data:/data
@@ -186,9 +181,6 @@ services:
     ports:
       - "5555:5555"
     environment:
-      - REDIS_INSTANCE=redis:6379
-      - CELERY_BROKER_URL=redis://redis:6379/0
-      - CELERY_RESULT_BACKEND=redis://redis:6379/0
       - PYTHONUNBUFFERED=1
     depends_on:
       - celery-worker

development_tools/local_selenium_init.sh

@@ -27,13 +27,12 @@ export KOBO_KF_URL=https://kobo.humanitarianresponse.info
 export KOBO_KC_URL=https://kobo.humanitarianresponse.info
 export ADMIN_PANEL_URL=unicorn
 export PROFILING=off
-export CYPRESS_TESTING=yes
 export DJANGO_ALLOWED_HOST=localhost
 export HCT_MIS_FRONTEND_HOST=localhost:8080
-export REDIS_INSTANCE=localhost:6379
 export PYTHONUNBUFFERED=1
 export CELERY_BROKER_URL=redis://localhost:6379/0
 export CELERY_RESULT_BACKEND=redis://localhost:6379/0
+export CONSTANCE_REDIS_CONNECTION=redis://localhost:6379/0
 export CACHE_LOCATION=redis://localhost:6379/1
 export USE_DUMMY_EXCHANGE_RATES=yes
 export ELASTICSEARCH_HOST=http://localhost:9200

docker/Dockerfile

@@ -9,7 +9,7 @@ FROM curl as certs
 RUN curl -o /data/psql-cert.crt -L https://dl.cacerts.digicert.com/DigiCertGlobalRootCA.crt.pem
 
 # Base image
-FROM python:3.11.7-slim-bookworm as base
+FROM python:3.12.7-slim-bookworm as base
 
 ARG UID=82
 
@@ -42,7 +42,7 @@ RUN apt-get update \
 ENV PDM_PACKAGES=/packages
 ENV CODE=/code
 ENV PDM_PROJECT=$PDM_PACKAGES
-ENV PYPACKAGES=$PDM_PACKAGES/__pypackages__/3.11
+ENV PYPACKAGES=$PDM_PACKAGES/__pypackages__/3.12
 ENV PYTHONPYCACHEPREFIX=/tmp/pycache \
     PYTHONPATH=$PYPACKAGES/lib:$PYTHONPATH \
     PATH=$PYPACKAGES/bin:$PATH \
@@ -55,7 +55,8 @@ COPY --from=waitforit /data/waitforit /usr/local/bin/waitforit
 # Dist builder image
 FROM base as pdm
 RUN pip install --upgrade pip &&\
-    pip install pdm==2.15.2 &&\
+    pip install pdm==2.19.2 &&\
+    pip install pdm-backend &&\
     pip install setuptools==71.1.0 &&\
     pdm config cache_dir /var/cache/pdm &&\
     pdm config python.use_venv false &&\

pyproject.toml

@@ -21,7 +21,7 @@ exclude = '''
 '''
 # TODO: remove migrations exclude rule once it won't create much conflicts between feature branches and develop
 [tool.isort]
-py_version = 39
+py_version = 312
 profile = "black"
 default_section = "THIRDPARTY"
 known_first_party = [
@@ -48,7 +48,7 @@ include_trailing_comma = true
 skip = ["migrations", "snapshots", "venv", ".venv", "__pypackages__", "frontend"]
 
 [tool.mypy]
-python_version = 3.11
+python_version = 3.12
 show_error_codes = true
 exclude = [
     "migrations",
@@ -137,7 +137,7 @@ distribution = true
 
 [project]
 name = "hope"
-version = "2.12.0"
+version = "2.13.0"
 description = "HCT MIS is UNICEF's humanitarian cash transfer platform."
 authors = [
     { name = "Tivix" },
@@ -190,6 +190,7 @@ dependencies = [
     "django-reversion<6.0.0,>=5.0.2",
     "django-silk<6.0.0,>=5.0.1",
     "django-smart-admin<3,>=2",
+    "django-smart-env",
     "django-sql-explorer[xls]<4,>=3",
     "django-storages[azure]<2.0.0,>=1.12.3",
     "django-strategy-field<4.0,>=3.0",
@@ -239,7 +240,7 @@ dependencies = [
     "flower>=2.0.1",
     "factory-boy<4,>=3",
 ]
-requires-python = "==3.11.*"
+requires-python = "==3.12.*"
 readme = "README.md"
 license = { text = "None" }
 

src/hct_mis_api/apps/grievance/fixtures.py

@@ -61,9 +61,11 @@ class Meta:
     business_area = factory.LazyAttribute(lambda o: BusinessArea.objects.first())
     created_at = factory.Faker("date_time_this_decade", before_now=False, after_now=True, tzinfo=utc)
     issue_type = factory.LazyAttribute(
-        lambda o: factory.fuzzy.FuzzyChoice(list(GrievanceTicket.ISSUE_TYPES_CHOICES.get(o.category, {}).keys())).fuzz()
-        if GrievanceTicket.ISSUE_TYPES_CHOICES.get(o.category)
-        else None
+        lambda o: (
+            factory.fuzzy.FuzzyChoice(list(GrievanceTicket.ISSUE_TYPES_CHOICES.get(o.category, {}).keys())).fuzz()
+            if GrievanceTicket.ISSUE_TYPES_CHOICES.get(o.category)
+            else None
+        )
     )
 
 

src/hct_mis_api/apps/program/mutations.py

@@ -89,9 +89,11 @@ def processed_mutate(cls, root: Any, info: Any, program_data: Dict) -> "CreatePr
 
         cls.validate(
             start_date=datetime.combine(program_data["start_date"], datetime.min.time()),
-            end_date=datetime.combine(program_data["end_date"], datetime.min.time())
-            if program_data.get("end_date")
-            else None,
+            end_date=(
+                datetime.combine(program_data["end_date"], datetime.min.time())
+                if program_data.get("end_date")
+                else None
+            ),
             data_collecting_type=data_collecting_type,
             business_area=business_area,
             programme_code=programme_code,

src/hct_mis_api/apps/program/utils.py

@@ -323,9 +323,9 @@ def copy_program_related_data(copy_from_program_id: str, new_program: Program, u
     rdi = RegistrationDataImport.objects.create(
         name=f"Default RDI for Programme: {new_program.name}",
         status=RegistrationDataImport.MERGED,
-        deduplication_engine_status=RegistrationDataImport.DEDUP_ENGINE_PENDING
-        if new_program.biometric_deduplication_enabled
-        else None,
+        deduplication_engine_status=(
+            RegistrationDataImport.DEDUP_ENGINE_PENDING if new_program.biometric_deduplication_enabled else None
+        ),
         imported_by=User.objects.get(id=user_id),
         data_source=RegistrationDataImport.PROGRAM_POPULATION,
         number_of_individuals=copy_from_individuals.count(),
@@ -398,9 +398,9 @@ def enroll_households_to_program(households: QuerySet, program: Program, user_id
     error_messages = []
     rdi = RegistrationDataImport.objects.create(
         status=RegistrationDataImport.MERGED,
-        deduplication_engine_status=RegistrationDataImport.DEDUP_ENGINE_PENDING
-        if program.biometric_deduplication_enabled
-        else None,
+        deduplication_engine_status=(
+            RegistrationDataImport.DEDUP_ENGINE_PENDING if program.biometric_deduplication_enabled else None
+        ),
         imported_by=User.objects.get(id=user_id),
         data_source=RegistrationDataImport.ENROLL_FROM_PROGRAM,
         number_of_individuals=0,

src/hct_mis_api/config/env.py

@@ -1,13 +1,20 @@
-from environ import Env
+from smart_env import SmartEnv
 
 DEFAULTS = {
     "AURORA_SERVER": (str, ""),
     "AURORA_TOKEN": (str, ""),
     "AURORA_USER": (str, ""),
+    "DATABASE_URL": (
+        str,
+        "sqlite://",
+        "",
+        "",
+        "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url",
+    ),
     "DEBUG": (bool, False),
     "ENV": (str, "dev"),
     "DOMAIN": (str, "localhost:8000"),
-    "DJANGO_ALLOWED_HOSTS": (list, "*"),
+    "DJANGO_ALLOWED_HOSTS": (list, []),
     "HCT_MIS_FRONTEND_HOST": (str, ""),
     "ALERTS_EMAIL": (str, "[email protected]"),
     "SECRET_KEY": (str, ""),
@@ -20,16 +27,21 @@
     "EMAIL_HOST_USER": (str, ""),
     "EMAIL_HOST_PASSWORD": (str, ""),
     "EMAIL_USE_TLS": (bool, True),
+    "DEFAULT_EMAIL": (str, ""),
     "MAILJET_API_KEY": (str, ""),
     "MAILJET_SECRET_KEY": (str, ""),
+    "CATCH_ALL_EMAIL": (list, []),
+    "DEFAULT_EMAIL_DISPLAY": (list, []),
     "KOBO_KF_URL": (str, "https://kf-hope.unitst.org"),
     "KOBO_KC_URL": (str, "https://kc-hope.unitst.org"),
     "KOBO_MASTER_API_TOKEN": (str, "KOBO_TOKEN"),
     "AZURE_CLIENT_ID": (str, ""),
     "AZURE_CLIENT_SECRET": (str, ""),
     "AZURE_TENANT_KEY": (str, ""),
+    "CONSTANCE_REDIS_CONNECTION": (str, "redis://redis:6379/0"),
     "SANCTION_LIST_CC_MAIL": (str, "[email protected]"),
     "ELASTICSEARCH_HOST": (str, "http://elasticsearch:9200"),
+    "ELASTICSEARCH_INDEX_PREFIX": (str, ""),
     "RAPID_PRO_URL": (str, "https://rapidpro.io"),
     "DATAMART_USER": (str, ""),
     "DATAMART_URL": (str, "https://datamart-dev.unicef.io"),
@@ -54,6 +66,7 @@
     "SECURE_HSTS_SECONDS": (int, 3600),
     "FLOWER_ADDRESS": (str, "https://hope.unicef.org/flower"),
     "CACHE_ENABLED": (bool, True),
+    "CACHE_LOCATION": (str, "redis://redis:6379/1"),
     "CSP_REPORT_URI": (tuple, ("",)),
     "CSP_REPORT_ONLY": (bool, True),
     "CSP_REPORT_PERCENTAGE": (float, 0.1),
@@ -112,6 +125,34 @@
         str,
         "",
     ),
+    "SOCIAL_AUTH_REDIRECT_IS_HTTPS": (bool, True),
+    "LOG_LEVEL": (str, "ERROR"),
+    "POSTGRES_SSL": (bool, False),
+    "GIT_VERSION": (str, "UNKNOWN"),
+    "EXCHANGE_RATE_CACHE_EXPIRY": (int, 60 * 60 * 24),
+    "USE_DUMMY_EXCHANGE_RATES": (str, "no"),
+    "PROFILING": (str, "off"),
+    "LIBRARY_PATHS": (bool, False),
+    "STATIC_URL": (str, "/static/"),
+    "MEDIA_URL": (str, "/media/"),
+    "FILE_STORAGE_STATIC": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+    ),
+    "FILE_STORAGE_MEDIA": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+    ),
+    "STORAGE_AZURE_ACCOUNT_NAME": (str, ""),
+    "STORAGE_AZURE_ACCOUNT_KEY": (str, ""),
+    "MEDIA_STORAGE_AZURE_ACCOUNT_NAME": (str, ""),
+    "MEDIA_STORAGE_AZURE_ACCOUNT_KEY": (str, ""),
+    "STATIC_STORAGE_AZURE_ACCOUNT_NAME": (str, ""),
+    "STATIC_STORAGE_AZURE_ACCOUNT_KEY": (str, ""),
+    "STATIC_LOCATION": (str, "static"),
+    "MEDIA_LOCATION": (str, "media"),
+    "AZURE_ACCOUNT_NAME": (str, ""),
+    "AZURE_ACCOUNT_KEY": (str, ""),
 }
 
-env = Env(**DEFAULTS)
+env = SmartEnv(**DEFAULTS)

src/hct_mis_api/config/fragments/celery.py

@@ -1,14 +1,12 @@
 from hct_mis_api.apps.core.tasks_schedules import TASKS_SCHEDULES
 from hct_mis_api.config.env import env
-from hct_mis_api.config.settings import REDIS_INSTANCE
 
-CELERY_BROKER_URL = env("CELERY_BROKER_URL", default=f"redis://{REDIS_INSTANCE}/0")
+CELERY_BROKER_URL = env("CELERY_BROKER_URL")
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
 CELERY_RESULT_SERIALIZER = "json"
-CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND", default=f"redis://{REDIS_INSTANCE}/0")
+CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND")
 CELERY_TIMEZONE = "UTC"
 CELERY_TASK_TRACK_STARTED = True
 CELERY_TASK_TIME_LIMIT = 360 * 60
 CELERY_BEAT_SCHEDULE = TASKS_SCHEDULES
-CELERY_TASK_ALWAYS_EAGER = env.bool("CELERY_TASK_ALWAYS_EAGER")