From 3bf93214a0a52bd2a9d91e8b2a6ebeb4c8502db6 Mon Sep 17 00:00:00 2001 From: Vitali Yanushchyk Date: Tue, 26 May 2026 04:12:28 -0400 Subject: [PATCH 1/2] add new rdp statuses --- docs/src/flows/rdp_lifecycle.md | 82 ++++++++++--------- src/country_workspace/admin/rdp.py | 14 ++-- .../contrib/hope/push/orchestration.py | 10 +-- .../contrib/hope/push/policy.py | 11 ++- .../contrib/hope/push/repository.py | 14 +++- .../migrations/0055_alter_rdp_status.py | 60 ++++++++++++++ src/country_workspace/models/rdp.py | 5 +- src/country_workspace/workspaces/admin/rdp.py | 2 +- 8 files changed, 138 insertions(+), 60 deletions(-) create mode 100644 src/country_workspace/migrations/0055_alter_rdp_status.py diff --git a/docs/src/flows/rdp_lifecycle.md b/docs/src/flows/rdp_lifecycle.md index af9422b3..b03444c4 100644 --- a/docs/src/flows/rdp_lifecycle.md +++ b/docs/src/flows/rdp_lifecycle.md @@ -8,16 +8,17 @@ stateDiagram-v2 PENDING --> PENDING: Dedup via DE REST PENDING --> FAILURE: Push failed - PENDING --> SUCCESS: Push succeeded - PENDING --> CANCELLED: Admin cancel - PENDING --> CANCELLED: Reject DS - PENDING --> CANCELLED: Clone replaces source + PENDING --> PUSHED: Push succeeded + PENDING --> REJECTED: Reject DS + PENDING --> REJECTED: Clone replaces source - SUCCESS --> CANCELLED: Staff Reset after HOPE rejection + PUSHED --> MERGED: HOPE merge confirmed + PUSHED --> REJECTED: Staff Reset after HOPE rejection FAILURE --> [*]: Final failed - CANCELLED --> [*]: Final cancelled - SUCCESS --> [*]: Final successful + REJECTED --> [*]: Final rejected + PUSHED --> [*]: Final pushed + MERGED --> [*]: Final merged note right of PENDING Local active RDP status.
@@ -31,17 +32,20 @@ stateDiagram-v2 RDP was not successfully handed off to HOPE Core. end note - note right of SUCCESS - Final successful RDP status.
- CW pushed data to HOPE Core successfully. - HOPE Core automerge is assumed. + note right of PUSHED + CW pushed data to HOPE Core successfully.
If a DE set exists, CW calls DE approve after successful push. DE approve failure is recorded but does not change RDP status. end note - note right of CANCELLED - Final cancelled RDP status.
- Can be set by admin cancel, DE set reject, + note right of MERGED + Final merged RDP status.
+ HOPE-side merge was confirmed. + end note + + note right of REJECTED + Final rejected RDP status.
+ Can be set by DE set reject, Clone replacing PENDING source, or Staff Reset. end note ``` @@ -60,25 +64,24 @@ flowchart TB D1 -- "error" --> PENDING PENDING --> RDS["Reject Deduplication Set
DE REST API"] - RDS -->|rejected| CANCELLED["CANCELLED
final cancelled state"] + RDS -->|rejected| REJECTED["REJECTED
final rejected state"] RDS -->|error| PENDING PENDING --> HOPE_PUSH["Push to HOPE
create RDI / push / complete"] HOPE_PUSH -->|failed| FAILURE["FAILURE
push workflow failed"] - HOPE_PUSH -->|succeeded| SUCCESS["SUCCESS
pushed to HOPE Core
automerge assumed"] - - SUCCESS -. "post-push" .-> DE_APPROVE["Approve Deduplication Set
DE REST API"] - DE_APPROVE -. "status unchanged" .-> SUCCESS + HOPE_PUSH -->|succeeded| PUSHED["PUSHED
pushed to HOPE Core"] - SUCCESS --> RESET["Staff admin Reset"] - RESET -->|HOPE rejection confirmed| CANCELLED + PUSHED -. "post-push" .-> DE_APPROVE["Approve Deduplication Set
DE REST API"] + DE_APPROVE -. "status unchanged" .-> PUSHED - PENDING -->|admin cancels RDP| CANCELLED + PUSHED --> MERGED["MERGED
merged in HOPE Core"] + PUSHED --> RESET["Staff admin Reset"] + RESET -->|HOPE rejection confirmed| REJECTED CLONE["Clone RDP"] --> CL1{"source status"} - CL1 -- "PENDING" --> CL2["source becomes CANCELLED
child becomes PENDING"] - CL1 -- "FAILURE / CANCELLED" --> CL3["source unchanged
child becomes PENDING"] - CL1 -- "SUCCESS" --> CL4["clone blocked"] + CL1 -- "PENDING" --> CL2["source becomes REJECTED
child becomes PENDING"] + CL1 -- "FAILURE / REJECTED" --> CL3["source unchanged
child becomes PENDING"] + CL1 -- "PUSHED / MERGED" --> CL4["clone blocked"] CL2 --> PENDING CL3 --> PENDING @@ -86,31 +89,31 @@ flowchart TB subgraph STATUS_RULES["Status-based policy rules"] subgraph CREATE_RULES["Regular Create RDP"] A["BLOCK: Program has PENDING RDP"] - B["BLOCK: selected records are already linked to PENDING/SUCCESS RDP"] + B["BLOCK: selected records are already linked to PENDING/PUSHED/MERGED RDP"] end subgraph DEDUP_RULES["Run Deduplication"] H["ENSURE: CW-owned deduplication_set_id before new DE set creation"] - J["BLOCK: Deduplicate cannot be started twice for the same RDP"] + J["LOCK: Program dedup settings after deduplication is requested"] end subgraph CLONE_RULES["Clone RDP"] - C["ALLOW: clone PENDING; source becomes CANCELLED, child becomes PENDING"] - D["ALLOW: clone FAILURE/CANCELLED; source unchanged, child becomes PENDING"] - E["BLOCK: clone SUCCESS"] + C["ALLOW: clone PENDING; source becomes REJECTED, child becomes PENDING"] + D["ALLOW: clone FAILURE/REJECTED; source unchanged, child becomes PENDING"] + E["BLOCK: clone PUSHED/MERGED"] F["REQUIRE: DE state allows clone: failed encoding, failed dedup, deduplicated, or rejected"] K["REUSE: child may keep source deduplication_set_id when source DE set is deduplicated"] end subgraph SETTINGS_RULES["Update Dedup Settings"] - G["BLOCK: PENDING RDP after deduplication was requested, or SUCCESS RDP"] + G["BLOCK: PENDING locked by dedup request, PUSHED, or MERGED RDP"] end subgraph PUSH_RULES["Push to HOPE"] L["SEND: RDI country_workspace_id = deduplication_set_id when present"] M["SEND: beneficiary country_workspace_id as required by HOPE Core"] N["APPROVE: DE set after successful HOPE push, if present"] - O["KEEP SUCCESS: DE approve error is recorded only"] + O["KEEP PUSHED: DE approve error is recorded only"] end end @@ -124,14 +127,14 @@ flowchart TB classDef success fill:#E8F6EC,stroke:#2E8540,stroke-width:1.5px,color:#1F3D2A; classDef failed fill:#FBEAEA,stroke:#C43B3B,stroke-width:1.5px,color:#4A1F1F; classDef neutral fill:#F3F4F6,stroke:#6B7280,stroke-width:1.5px,color:#1F2937; - classDef cancelled fill:#F3E8FF,stroke:#7E22CE,stroke-width:1.5px,color:#3B0764; + classDef rejected fill:#F3E8FF,stroke:#7E22CE,stroke-width:1.5px,color:#3B0764; class PENDING,CREATE,CLONE,CL1,CL2,CL3,RESET,DEDUP_CLAIM local; class DEDUP,D1,RDS,HOPE_PUSH,DE_APPROVE,X1,X2 external; - class SUCCESS success; + class PUSHED,MERGED success; class FAILURE failed; - class CANCELLED cancelled; - class CL4,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O neutral; + class REJECTED rejected; + class CL4,A,B,C,D,E,F,G,H,J,K,L,M,N,O neutral; ``` ## Key rules @@ -140,6 +143,7 @@ flowchart TB - For a new DedupEngine set, `deduplication_set_id` is generated by CW before the create call. - RDI `country_workspace_id` is sent only when the RDP has `deduplication_set_id`; its value is `str(deduplication_set_id)`. - `deduplication_set_id` belongs to the selection owner / deduplication source, so clones may reuse it. -- `SUCCESS` means CW successfully completed the HOPE push; HOPE Core automerge is assumed. -- If a DedupEngine set exists, approve is called after successful HOPE push. Approval errors are recorded, but RDP status stays `SUCCESS`. -- Staff Reset can move `SUCCESS` to `CANCELLED` when HOPE-side rejection is confirmed manually. +- `PUSHED` means CW successfully completed the HOPE push. +- `MERGED` means HOPE-side merge was confirmed. +- If a DedupEngine set exists, approve is called after successful HOPE push. Approval errors are recorded, but RDP status stays `PUSHED`. +- Staff Reset can move `PUSHED` to `REJECTED` when HOPE-side rejection is confirmed manually. diff --git a/src/country_workspace/admin/rdp.py b/src/country_workspace/admin/rdp.py index 4a1af2ae..dc6399df 100644 --- a/src/country_workspace/admin/rdp.py +++ b/src/country_workspace/admin/rdp.py @@ -74,21 +74,21 @@ def view_in_workspace(self, btn: LinkButton) -> None: change_list=False, label="Reset", html_attrs={"class": "btn-warning"}, - enabled=lambda btn: btn.context["original"].status == Rdp.PushStatus.SUCCESS, + enabled=lambda btn: btn.context["original"].status == Rdp.PushStatus.PUSHED, ) def reset(self, request: HttpRequest, pk: int) -> HttpResponse: obj: Rdp = self.get_object(request, str(pk)) - if obj.status != Rdp.PushStatus.SUCCESS: - self.message_user(request, "Reset is only allowed for SUCCESS status.", level="error") + if obj.status != Rdp.PushStatus.PUSHED: + self.message_user(request, "Reset is only allowed for PUSHED status.", level="error") return HttpResponseRedirect(reverse("admin:country_workspace_rdp_change", args=[pk])) def _action(_: HttpRequest) -> HttpResponseRedirect: with transaction.atomic(): obj.households.all().update(removed=False) obj.individuals.all().update(removed=False) - obj.status = Rdp.PushStatus.CANCELLED - obj.save() + obj.status = Rdp.PushStatus.REJECTED + obj.save(update_fields=["status"]) return HttpResponseRedirect(reverse("admin:country_workspace_rdp_change", args=[pk])) return confirm_action( @@ -97,8 +97,8 @@ def _action(_: HttpRequest) -> HttpResponseRedirect: _action, "Are you sure you want to reset this RDP?", description=( - "This will set all related households and individuals to removed=False " - "and mark the RDP status as CANCELLED. This action cannot be undone." + "It will set all related households and individuals to removed=False " + "and mark the RDP status as REJECTED. This action cannot be undone." ), success_message="RDP reset successfully. Related beneficiaries marked as not removed.", pk=str(pk), diff --git a/src/country_workspace/contrib/hope/push/orchestration.py b/src/country_workspace/contrib/hope/push/orchestration.py index ea4cbe36..0c2bb204 100644 --- a/src/country_workspace/contrib/hope/push/orchestration.py +++ b/src/country_workspace/contrib/hope/push/orchestration.py @@ -191,8 +191,8 @@ def _clone_rdp_in_transaction( try: with transaction.atomic(): source = lock_rdp_for_update(pk=source.pk) - if source.status == Rdp.PushStatus.SUCCESS: - raise HopePushError({"errors": ["RDP: can not clone a successful RDP."]}) + if source.status == Rdp.PushStatus.MERGED: + raise HopePushError({"errors": ["RDP: can not clone a merged RDP."]}) owner = selection_owner_for_rdp(rdp=source) if owner.pk != source.pk: owner = lock_rdp_for_update(pk=owner.pk) @@ -212,7 +212,7 @@ def _clone_rdp_in_transaction( update_fields: list[str] = [] if source.status == Rdp.PushStatus.PENDING: - source.status = Rdp.PushStatus.CANCELLED + source.status = Rdp.PushStatus.REJECTED update_fields.append("status") if source.is_dedup_settings_locked: source.is_dedup_settings_locked = False @@ -253,7 +253,7 @@ def reject_deduplication_set_existing_rdp_core(job: AsyncJob) -> dict[str, Any]: locked = lock_rdp_for_update(pk=rdp.pk) set_rdp_push_status( rdp=locked, - status=Rdp.PushStatus.CANCELLED, + status=Rdp.PushStatus.REJECTED, hope_rdi_id=locked.hope_rdi_id or "N/A", is_dedup_settings_locked=False, ) @@ -321,7 +321,7 @@ def push_existing_rdp_core(job: AsyncJob) -> dict[str, Any]: _mark_rdp_beneficiaries_removed(locked, config["master_detail"]) set_rdp_push_status( rdp=locked, - status=Rdp.PushStatus.SUCCESS, + status=Rdp.PushStatus.PUSHED, hope_rdi_id=hope_processor.hope_rdi_id or "N/A", ) group_reference_id = locked.program.unicef_id diff --git a/src/country_workspace/contrib/hope/push/policy.py b/src/country_workspace/contrib/hope/push/policy.py index d84d7fa0..dfcced7b 100644 --- a/src/country_workspace/contrib/hope/push/policy.py +++ b/src/country_workspace/contrib/hope/push/policy.py @@ -77,7 +77,7 @@ def update_dedup_settings_check(self) -> ActionCheck: if self._has_locked_dedup_settings(): return ActionCheck( False, - "Deduplication settings cannot be updated after a successful RDP " + "Deduplication settings cannot be updated after an RDP was pushed or merged " "or while a pending RDP has requested a new deduplication run.", ) @@ -86,7 +86,10 @@ def update_dedup_settings_check(self) -> ActionCheck: def _has_locked_dedup_settings(self) -> bool: return ( Rdp.objects.filter(program=self.program) - .filter(Q(status=Rdp.PushStatus.SUCCESS) | Q(status=Rdp.PushStatus.PENDING, is_dedup_settings_locked=True)) + .filter( + Q(status__in=[Rdp.PushStatus.MERGED, Rdp.PushStatus.PUSHED]) + | Q(status=Rdp.PushStatus.PENDING, is_dedup_settings_locked=True) + ) .exists() ) @@ -222,8 +225,8 @@ def _clone_deduplication_check(self) -> ActionCheck: def clone_check(self) -> ActionCheck: if not self.is_biometric_deduplication_enabled: return ActionCheck(False, "DedupEngine: biometric deduplication is not enabled for this program.") - if self.rdp.status == Rdp.PushStatus.SUCCESS: - return ActionCheck(False, "RDP: can not clone a successful RDP.") + if self.rdp.status in [Rdp.PushStatus.PUSHED, Rdp.PushStatus.MERGED]: + return ActionCheck(False, f"RDP: can not clone in status={self.rdp.status}") exclude_ids = (self.rdp.pk,) if self.is_pending else () if has_other_pending_rdp(owner=self.owner, exclude_ids=exclude_ids): diff --git a/src/country_workspace/contrib/hope/push/repository.py b/src/country_workspace/contrib/hope/push/repository.py index 438008b8..d78d16b5 100644 --- a/src/country_workspace/contrib/hope/push/repository.py +++ b/src/country_workspace/contrib/hope/push/repository.py @@ -123,7 +123,7 @@ def preflight_errors( if not pks: return ["RDP: no beneficiaries selected"] - rdp_qs = Rdp.objects.filter(status__in=[Rdp.PushStatus.PENDING, Rdp.PushStatus.SUCCESS]) + rdp_qs = Rdp.objects.filter(status__in=[Rdp.PushStatus.PENDING, Rdp.PushStatus.PUSHED, Rdp.PushStatus.MERGED]) if excluded := tuple(exclude_rdp_ids): rdp_qs = rdp_qs.exclude(pk__in=excluded) @@ -134,7 +134,7 @@ def collect(rows: QuerySet, tag: str) -> list[str]: if not _is_valid_row(last_checked=last_checked, errors=obj_errors): errors.append(f"{base} invalid") if has_rdp: - errors.append(f"{base} already in another RDP(s) (pending/success)") + errors.append(f"{base} already in another RDP(s) (pending/pushed/merged)") return errors def individual_rows() -> QuerySet: @@ -190,3 +190,13 @@ def has_other_pending_rdp(*, owner: Rdp, exclude_ids: Iterable[int] = ()) -> boo if excluded := tuple(exclude_ids): qs = qs.exclude(pk__in=excluded) return qs.exists() + + +def has_other_active_rdp(*, owner: Rdp, exclude_ids: Iterable[int] = ()) -> bool: + qs = Rdp.objects.filter( + program_id=owner.program_id, + status__in=[Rdp.PushStatus.PENDING, Rdp.PushStatus.PUSHED], + ) + if excluded := tuple(exclude_ids): + qs = qs.exclude(pk__in=excluded) + return qs.exists() diff --git a/src/country_workspace/migrations/0055_alter_rdp_status.py b/src/country_workspace/migrations/0055_alter_rdp_status.py new file mode 100644 index 00000000..2fca4470 --- /dev/null +++ b/src/country_workspace/migrations/0055_alter_rdp_status.py @@ -0,0 +1,60 @@ +from collections.abc import Mapping +from django.db import migrations, models +from django.db.migrations.state import StateApps +from django.db.backends.base.schema import BaseDatabaseSchemaEditor + + +def _remap_rdp_statuses(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor, mapping: Mapping[str, str]) -> None: + Rdp = apps.get_model("country_workspace", "Rdp") + db_alias = schema_editor.connection.alias + for old, new in mapping.items(): + Rdp.objects.using(db_alias).filter(status=old).update(status=new) + + +def forward_rdp_statuses(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + _remap_rdp_statuses( + apps, + schema_editor, + { + "SUCCESS": "PUSHED", + "CANCELLED": "REJECTED", + }, + ) + + +def backward_rdp_statuses(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + _remap_rdp_statuses( + apps, + schema_editor, + { + "PUSHED": "SUCCESS", + "MERGED": "SUCCESS", + "REJECTED": "CANCELLED", + }, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("country_workspace", "0054_rdp_deduplication_snapshots_and_more"), + ] + + operations = [ + migrations.RunPython(forward_rdp_statuses, backward_rdp_statuses), + migrations.AlterField( + model_name="rdp", + name="status", + field=models.CharField( + blank=True, + choices=[ + ("PENDING", "Pending"), + ("FAILURE", "Failure"), + ("PUSHED", "Pushed"), + ("MERGED", "Merged"), + ("REJECTED", "Rejected"), + ], + default="PENDING", + max_length=10, + ), + ), + ] diff --git a/src/country_workspace/models/rdp.py b/src/country_workspace/models/rdp.py index 3d2c49f4..e706d153 100644 --- a/src/country_workspace/models/rdp.py +++ b/src/country_workspace/models/rdp.py @@ -11,9 +11,10 @@ class Rdp(BaseModel): class PushStatus(models.TextChoices): PENDING = "PENDING", _("Pending") - SUCCESS = "SUCCESS", _("Success") FAILURE = "FAILURE", _("Failure") - CANCELLED = "CANCELLED", _("Cancelled") + PUSHED = "PUSHED", _("Pushed") + MERGED = "MERGED", _("Merged") + REJECTED = "REJECTED", _("Rejected") class DedupTrackingState(models.TextChoices): NOT_RUN = "NOT_RUN", _("Not run yet") diff --git a/src/country_workspace/workspaces/admin/rdp.py b/src/country_workspace/workspaces/admin/rdp.py index 3089ad9e..65c503f2 100644 --- a/src/country_workspace/workspaces/admin/rdp.py +++ b/src/country_workspace/workspaces/admin/rdp.py @@ -300,7 +300,7 @@ def push(self, request: HttpRequest, pk: str) -> HttpResponse: @link(change_list=False, html_attrs={"title": "Shows related beneficiary records."}) def records(self, btn: LinkButton) -> None: obj = btn.context["original"] - if obj.status == CountryRdp.PushStatus.SUCCESS: + if obj.status == CountryRdp.PushStatus.MERGED: btn.visible = False return item = "countryhousehold" if obj.program.beneficiary_group.master_detail else "countryindividual" From deeaa43b53112f8e3c31718cdabe8bae12a9b378 Mon Sep 17 00:00:00 2001 From: Vitali Yanushchyk Date: Fri, 5 Jun 2026 04:30:32 -0400 Subject: [PATCH 2/2] Add authenticated HOPE RDI callbacks to finalize RDPs as merged or rejected. --- pyproject.toml | 1 + src/country_workspace/admin/__init__.py | 2 + src/country_workspace/admin/api_token.py | 49 ++++++++++++++ src/country_workspace/api/__init__.py | 0 src/country_workspace/api/authentication.py | 20 ++++++ src/country_workspace/api/permissions.py | 12 ++++ src/country_workspace/api/serializers.py | 15 +++++ src/country_workspace/api/urls.py | 11 +++ src/country_workspace/api/views.py | 61 +++++++++++++++++ src/country_workspace/config/settings.py | 3 + src/country_workspace/config/urls.py | 1 + .../contrib/hope/exceptions.py | 12 ++++ .../contrib/hope/push/__init__.py | 2 + .../contrib/hope/push/orchestration.py | 67 +++++++++++++++++-- .../contrib/hope/push/repository.py | 16 ++++- .../0056_rdp_uniq_rdp_hope_rdi_id.py | 39 +++++++++++ .../migrations/0057_apitoken.py | 45 +++++++++++++ src/country_workspace/models/__init__.py | 2 + src/country_workspace/models/api_token.py | 55 +++++++++++++++ src/country_workspace/models/rdp.py | 6 ++ 20 files changed, 411 insertions(+), 8 deletions(-) create mode 100644 src/country_workspace/admin/api_token.py create mode 100644 src/country_workspace/api/__init__.py create mode 100644 src/country_workspace/api/authentication.py create mode 100644 src/country_workspace/api/permissions.py create mode 100644 src/country_workspace/api/serializers.py create mode 100644 src/country_workspace/api/urls.py create mode 100644 src/country_workspace/api/views.py create mode 100644 src/country_workspace/migrations/0056_rdp_uniq_rdp_hope_rdi_id.py create mode 100644 src/country_workspace/migrations/0057_apitoken.py create mode 100644 src/country_workspace/models/api_token.py diff --git a/pyproject.toml b/pyproject.toml index 645d524c..adad7c79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "django-sysinfo>=2.6.2", "django-tailwind>=3.8", "djangorestframework>=3.15.1", + "drf-spectacular[sidecar]", "dukpy>=0.5", "flower>=2.0.1", "hope-flex-fields>=0.8.1", diff --git a/src/country_workspace/admin/__init__.py b/src/country_workspace/admin/__init__.py index 136c0737..52dbb077 100644 --- a/src/country_workspace/admin/__init__.py +++ b/src/country_workspace/admin/__init__.py @@ -5,6 +5,7 @@ from smart_admin.smart_auth.admin import ContentTypeAdmin, PermissionAdmin from ..cache.smart_panel import panel_cache +from .api_token import APITokenAdmin from .batch import BatchAdmin from .beneficiary_group import BeneficiaryGroupAdmin from .constance import ConstanceAdmin @@ -32,6 +33,7 @@ site.register_panel(panel_redis) __all__ = [ + "APITokenAdmin", "AreaAdmin", "AreaTypeAdmin", "AsyncJobAdmin", diff --git a/src/country_workspace/admin/api_token.py b/src/country_workspace/admin/api_token.py new file mode 100644 index 00000000..34cf0faf --- /dev/null +++ b/src/country_workspace/admin/api_token.py @@ -0,0 +1,49 @@ +from django.contrib.admin import ModelAdmin, display, register +from django.db.models import QuerySet +from django.http import HttpRequest +from django.utils import timezone + +from country_workspace.models import APIToken + + +@register(APIToken) +class APITokenAdmin(ModelAdmin): + list_display = ( + "masked_key", + "user", + "grant_type", + "office_names", + "valid_now", + "valid_from", + "valid_to", + "created", + ) + list_filter = ("grant_type", "offices", "valid_from", "valid_to") + search_fields = ("key", "user__email", "offices__name") + filter_horizontal = ("offices",) + ordering = ("-created",) + date_hierarchy = "created" + autocomplete_fields = ("user",) + + def get_queryset(self, request: HttpRequest) -> QuerySet[APIToken]: + return super().get_queryset(request).select_related("user").prefetch_related("offices") + + def get_fields(self, request: HttpRequest, obj: APIToken | None = None) -> tuple[str, ...]: + fields = ("user", "grant_type", "offices", "valid_from", "valid_to") + return (*fields, "key", "valid_now", "created") if obj else fields + + def get_readonly_fields(self, request: HttpRequest, obj: APIToken | None = None) -> tuple[str, ...]: + readonly = ("key", "valid_now", "created") + return (*readonly, "user", "grant_type", "valid_from") if obj else readonly + + @display(description="Key", ordering="key") + def masked_key(self, obj: APIToken) -> str: + return f"{obj.key[:8]}…{obj.key[-4:]}" + + @display(description="Offices") + def office_names(self, obj: APIToken) -> str: + return ", ".join(map(str, obj.offices.all())) or "—" + + @display(boolean=True, description="Valid now") + def valid_now(self, obj: APIToken) -> bool: + return obj.is_valid_at(timezone.now()) diff --git a/src/country_workspace/api/__init__.py b/src/country_workspace/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/country_workspace/api/authentication.py b/src/country_workspace/api/authentication.py new file mode 100644 index 00000000..ecc124a6 --- /dev/null +++ b/src/country_workspace/api/authentication.py @@ -0,0 +1,20 @@ +from typing import Any, cast + +from django.utils import timezone +from rest_framework.authentication import TokenAuthentication +from rest_framework.exceptions import AuthenticationFailed + +from country_workspace.models import APIToken + + +class APITokenAuthentication(TokenAuthentication): + model = APIToken + + def authenticate_credentials(self, key: str) -> tuple[Any, APIToken]: + user, token = super().authenticate_credentials(key) + token = cast("APIToken", token) + + if not token.is_valid_at(timezone.now()): + raise AuthenticationFailed("Token is expired or not active yet.") + + return user, token diff --git a/src/country_workspace/api/permissions.py b/src/country_workspace/api/permissions.py new file mode 100644 index 00000000..290bd471 --- /dev/null +++ b/src/country_workspace/api/permissions.py @@ -0,0 +1,12 @@ +from rest_framework.permissions import BasePermission +from rest_framework.request import Request +from rest_framework.views import APIView + +from country_workspace.models import APIToken + + +class CanCallHopeRdiCallback(BasePermission): + message = "Token does not grant access to the HOPE RDI callback." + + def has_permission(self, request: Request, view: APIView) -> bool: + return isinstance(request.auth, APIToken) and request.auth.grant_type == APIToken.GrantType.HOPE_RDI_CALLBACK diff --git a/src/country_workspace/api/serializers.py b/src/country_workspace/api/serializers.py new file mode 100644 index 00000000..79632f42 --- /dev/null +++ b/src/country_workspace/api/serializers.py @@ -0,0 +1,15 @@ +from rest_framework import serializers + +from country_workspace.models import Rdp + + +class HopeRdiCallbackSerializer(serializers.Serializer): + status = serializers.ChoiceField( + choices=( + Rdp.PushStatus.MERGED, + Rdp.PushStatus.REJECTED, + ) + ) + + def validate_status(self, value: str) -> Rdp.PushStatus: + return Rdp.PushStatus(value) diff --git a/src/country_workspace/api/urls.py b/src/country_workspace/api/urls.py new file mode 100644 index 00000000..e4717e0e --- /dev/null +++ b/src/country_workspace/api/urls.py @@ -0,0 +1,11 @@ +from rest_framework.routers import SimpleRouter + +from .views import HopeRdiViewSet + + +app_name = "api" + +router = SimpleRouter() +router.register("hope-rdis", HopeRdiViewSet, basename="hope-rdi") + +urlpatterns = router.urls diff --git a/src/country_workspace/api/views.py b/src/country_workspace/api/views.py new file mode 100644 index 00000000..6447644c --- /dev/null +++ b/src/country_workspace/api/views.py @@ -0,0 +1,61 @@ +from http import HTTPMethod +from typing import Any, cast, TYPE_CHECKING + +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.request import Request +from rest_framework.response import Response + +from country_workspace.contrib.hope.exceptions import ( + HopeRdiCallbackConflictError, + HopeRdiCallbackError, + HopeRdiCallbackNotFoundError, +) +from country_workspace.contrib.hope.push import apply_hope_rdi_final_status + +from .authentication import APITokenAuthentication +from .permissions import CanCallHopeRdiCallback +from .serializers import HopeRdiCallbackSerializer + +if TYPE_CHECKING: + from country_workspace.models import APIToken, Rdp + + +def _error_payload(exc: HopeRdiCallbackError) -> dict[str, Any]: + return exc.args[0] if exc.args and isinstance(exc.args[0], dict) else {"errors": [str(exc)]} + + +class HopeRdiViewSet(viewsets.GenericViewSet): + authentication_classes = (APITokenAuthentication,) + permission_classes = (CanCallHopeRdiCallback,) + serializer_class = HopeRdiCallbackSerializer + lookup_url_kwarg = "hope_rdi_id" + + @action(detail=True, methods=(HTTPMethod.POST,), url_path="callback") + def callback(self, request: Request, hope_rdi_id: str) -> Response: + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + token = cast("APIToken", request.auth) + callback_status: Rdp.PushStatus = serializer.validated_data["status"] + + try: + rdp = apply_hope_rdi_final_status( + hope_rdi_id=hope_rdi_id, + status=callback_status, + token=token, + ) + except HopeRdiCallbackNotFoundError as exc: + return Response(_error_payload(exc), status=status.HTTP_404_NOT_FOUND) + except HopeRdiCallbackConflictError as exc: + return Response(_error_payload(exc), status=status.HTTP_409_CONFLICT) + except HopeRdiCallbackError as exc: + return Response(_error_payload(exc), status=status.HTTP_400_BAD_REQUEST) + + return Response( + { + "rdp_id": rdp.pk, + "rdi_id": rdp.hope_rdi_id, + "status": rdp.status, + } + ) diff --git a/src/country_workspace/config/settings.py b/src/country_workspace/config/settings.py index c293e4d2..02c17966 100644 --- a/src/country_workspace/config/settings.py +++ b/src/country_workspace/config/settings.py @@ -41,10 +41,13 @@ "adminfilters", "adminfilters.depot", "constance", + "rest_framework", "jsoneditor", "django_celery_beat", "django_celery_results", "django_celery_boost", + "drf_spectacular", + "drf_spectacular_sidecar", "hope_flex_fields", "hope_smart_import", "hope_smart_export", diff --git a/src/country_workspace/config/urls.py b/src/country_workspace/config/urls.py index 3e44cbc7..6bd9ab01 100644 --- a/src/country_workspace/config/urls.py +++ b/src/country_workspace/config/urls.py @@ -9,6 +9,7 @@ urlpatterns = [ path(r"admin/", admin.site.urls), + path("api/", include("country_workspace.api.urls", namespace="api")), path(r"security/", include("unicef_security.urls", namespace="security")), path(r"social/", include("social_django.urls", namespace="social")), path(r"accounts/", include("django.contrib.auth.urls")), diff --git a/src/country_workspace/contrib/hope/exceptions.py b/src/country_workspace/contrib/hope/exceptions.py index 397f734a..9b27cea9 100644 --- a/src/country_workspace/contrib/hope/exceptions.py +++ b/src/country_workspace/contrib/hope/exceptions.py @@ -2,6 +2,18 @@ class HopePushError(Exception): """Exception raised for errors during the push process.""" +class HopeRdiCallbackError(Exception): + """Exception raised for errors during HOPE RDI callback processing.""" + + +class HopeRdiCallbackNotFoundError(HopeRdiCallbackError): + """Exception raised when no RDP matches the HOPE RDI ID.""" + + +class HopeRdiCallbackConflictError(HopeRdiCallbackError): + """Exception raised when the HOPE RDI callback conflicts with the current RDP state.""" + + class HopeSyncError(Exception): """Exception raised for errors during the synchronization process.""" diff --git a/src/country_workspace/contrib/hope/push/__init__.py b/src/country_workspace/contrib/hope/push/__init__.py index 81ececa8..df7850dd 100644 --- a/src/country_workspace/contrib/hope/push/__init__.py +++ b/src/country_workspace/contrib/hope/push/__init__.py @@ -1,5 +1,6 @@ from .config import CreateRdpConfig, PushExistingRdpConfig from .orchestration import ( + apply_hope_rdi_final_status, claim_rdp_deduplication, clone_rdp_core, create_rdp_core, @@ -16,6 +17,7 @@ "DedupEngineState", "PushExistingRdpConfig", "PushProcessor", + "apply_hope_rdi_final_status", "claim_rdp_deduplication", "clone_rdp_core", "create_rdp_core", diff --git a/src/country_workspace/contrib/hope/push/orchestration.py b/src/country_workspace/contrib/hope/push/orchestration.py index 0c2bb204..68ebf5a4 100644 --- a/src/country_workspace/contrib/hope/push/orchestration.py +++ b/src/country_workspace/contrib/hope/push/orchestration.py @@ -11,9 +11,14 @@ DeduplicationSetState, make_dedup_client, ) -from country_workspace.contrib.hope.exceptions import HopePushError +from country_workspace.contrib.hope.exceptions import ( + HopePushError, + HopeRdiCallbackConflictError, + HopeRdiCallbackError, + HopeRdiCallbackNotFoundError, +) from country_workspace.exceptions import RemoteError, RemoteUnavailableError -from country_workspace.models import AsyncJob, Rdp +from country_workspace.models import APIToken, AsyncJob, Rdp from .config import CreateRdpConfig, PushWorkflowConfig from .policy import ActionCheck, get_rdp_policy @@ -21,6 +26,7 @@ from .repository import ( has_other_pending_rdp, lock_rdp_for_update, + lock_rdp_for_hope_callback, preflight_errors, preflight_exclude_rdp_ids, qs_households, @@ -229,7 +235,6 @@ def _clone_rdp_in_transaction( status=Rdp.PushStatus.PENDING, deduplication_set_id=context.clone_deduplication_set_id, is_dedup_settings_locked=False, - hope_rdi_id="", ) except IntegrityError as e: raise HopePushError({"errors": [f"RDP: can not clone record: {e}"]}) from e @@ -254,7 +259,7 @@ def reject_deduplication_set_existing_rdp_core(job: AsyncJob) -> dict[str, Any]: set_rdp_push_status( rdp=locked, status=Rdp.PushStatus.REJECTED, - hope_rdi_id=locked.hope_rdi_id or "N/A", + hope_rdi_id=locked.hope_rdi_id, is_dedup_settings_locked=False, ) @@ -312,17 +317,20 @@ def push_existing_rdp_core(job: AsyncJob) -> dict[str, Any]: set_rdp_push_status( rdp=locked, status=Rdp.PushStatus.FAILURE, - hope_rdi_id=hope_processor.hope_rdi_id or "N/A", + hope_rdi_id=hope_processor.hope_rdi_id, ) raise HopePushError(hope_processor.total) + if not (hope_rdi_id := hope_processor.hope_rdi_id): + raise HopePushError({"errors": ["RDI id is not set after successful push."]}) + with transaction.atomic(): locked = lock_rdp_for_update(pk=rdp.pk) _mark_rdp_beneficiaries_removed(locked, config["master_detail"]) set_rdp_push_status( rdp=locked, status=Rdp.PushStatus.PUSHED, - hope_rdi_id=hope_processor.hope_rdi_id or "N/A", + hope_rdi_id=hope_rdi_id, ) group_reference_id = locked.program.unicef_id deduplication_set_id = locked.deduplication_set_id @@ -353,3 +361,50 @@ def _approve_deduplication_set_after_successful_push( client.approve() except (RemoteError, RemoteUnavailableError) as e: processor.fail("DedupEngine", f"approve failed. {e}") + + +def _mark_rdp_beneficiaries_not_removed(rdp: Rdp) -> None: + """Mark selection-owner beneficiaries as not removed.""" + owner = selection_owner_for_rdp(rdp=rdp) + hh_ids = list(owner.households.values_list("pk", flat=True)) + if hh_ids: + owner.households.update(removed=False) + qs_individuals_by_household_pks(hh_ids).update(removed=False) + return + owner.individuals.update(removed=False) + + +def apply_hope_rdi_final_status(*, hope_rdi_id: str, status: Rdp.PushStatus, token: APIToken) -> Rdp: + """Apply final HOPE Core RDI status to a pushed CW RDP.""" + if not hope_rdi_id: + raise HopeRdiCallbackError({"errors": ["RDI id is required."]}) + + if status not in {Rdp.PushStatus.MERGED, Rdp.PushStatus.REJECTED}: + raise HopeRdiCallbackError({"errors": ["Unsupported RDI final status."]}) + + with transaction.atomic(): + try: + rdp = lock_rdp_for_hope_callback( + hope_rdi_id=hope_rdi_id, + token=token, + ) + except Rdp.DoesNotExist as exc: + raise HopeRdiCallbackNotFoundError({"errors": ["RDP not found for RDI id."]}) from exc + + if rdp.status == status: + return rdp + + if rdp.status != Rdp.PushStatus.PUSHED: + raise HopeRdiCallbackConflictError({"errors": [f"RDP: can not be finalized from status={rdp.status}."]}) + + if status == Rdp.PushStatus.REJECTED: + _mark_rdp_beneficiaries_not_removed(rdp) + + set_rdp_push_status( + rdp=rdp, + status=status, + hope_rdi_id=hope_rdi_id, + is_dedup_settings_locked=False if status == Rdp.PushStatus.REJECTED else None, + ) + + return rdp diff --git a/src/country_workspace/contrib/hope/push/repository.py b/src/country_workspace/contrib/hope/push/repository.py index d78d16b5..5fea9449 100644 --- a/src/country_workspace/contrib/hope/push/repository.py +++ b/src/country_workspace/contrib/hope/push/repository.py @@ -4,7 +4,7 @@ from django.db.models import Exists, OuterRef, Prefetch, QuerySet from country_workspace.contrib.hope.constants import PUSH_BATCH_SIZE -from country_workspace.models import Program, Rdp +from country_workspace.models import APIToken, Program, Rdp from country_workspace.workspaces.models import CountryHousehold, CountryIndividual from .config import PushWorkflowConfig, Serializer @@ -163,7 +163,11 @@ def individual_rows() -> QuerySet: def set_rdp_push_status( - *, rdp: Rdp, status: Rdp.PushStatus, hope_rdi_id: str, is_dedup_settings_locked: bool | None = None + *, + rdp: Rdp, + status: Rdp.PushStatus, + hope_rdi_id: str | None, + is_dedup_settings_locked: bool | None = None, ) -> None: """Persist push status fields for an already-locked RDP.""" rdp.status = status @@ -200,3 +204,11 @@ def has_other_active_rdp(*, owner: Rdp, exclude_ids: Iterable[int] = ()) -> bool if excluded := tuple(exclude_ids): qs = qs.exclude(pk__in=excluded) return qs.exists() + + +def lock_rdp_for_hope_callback(*, hope_rdi_id: str, token: APIToken) -> Rdp: + """Return an office-scoped RDP locked for a HOPE callback.""" + return Rdp.objects.select_for_update().get( + hope_rdi_id=hope_rdi_id, + country_office__api_tokens=token, + ) diff --git a/src/country_workspace/migrations/0056_rdp_uniq_rdp_hope_rdi_id.py b/src/country_workspace/migrations/0056_rdp_uniq_rdp_hope_rdi_id.py new file mode 100644 index 00000000..aee50f81 --- /dev/null +++ b/src/country_workspace/migrations/0056_rdp_uniq_rdp_hope_rdi_id.py @@ -0,0 +1,39 @@ +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + + +def validate_unique_hope_rdi_id(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: + Rdp = apps.get_model("country_workspace", "Rdp") + db_alias = schema_editor.connection.alias + + duplicates = list( + Rdp.objects.using(db_alias) + .filter(hope_rdi_id__isnull=False) + .values("hope_rdi_id") + .annotate(count=models.Count("pk")) + .filter(count__gt=1) + .order_by("hope_rdi_id") + ) + if duplicates: + details = ", ".join(f"{item['hope_rdi_id']} ({item['count']})" for item in duplicates[:10]) + raise RuntimeError(f"Cannot add unique RDP hope_rdi_id constraint. Duplicates found: {details}") + + +class Migration(migrations.Migration): + dependencies = [ + ("country_workspace", "0055_alter_rdp_status"), + ] + + operations = [ + migrations.RunPython(validate_unique_hope_rdi_id, migrations.RunPython.noop), + migrations.AddConstraint( + model_name="rdp", + constraint=models.UniqueConstraint( + condition=models.Q(("hope_rdi_id__isnull", False)), + fields=("hope_rdi_id",), + name="uniq_rdp_hope_rdi_id", + violation_error_message="There is already an RDP for this HOPE RDI.", + ), + ), + ] diff --git a/src/country_workspace/migrations/0057_apitoken.py b/src/country_workspace/migrations/0057_apitoken.py new file mode 100644 index 00000000..47ca7e70 --- /dev/null +++ b/src/country_workspace/migrations/0057_apitoken.py @@ -0,0 +1,45 @@ +# Generated by Django 5.2.14 on 2026-06-05 07:34 + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("country_workspace", "0056_rdp_uniq_rdp_hope_rdi_id"), + ] + + operations = [ + migrations.CreateModel( + name="APIToken", + fields=[ + ("key", models.CharField(editable=False, max_length=40, primary_key=True, serialize=False)), + ("grant_type", models.CharField(choices=[("HOPE_RDI_CALLBACK", "HOPE RDI Callback")], max_length=50)), + ("created", models.DateTimeField(auto_now_add=True)), + ("valid_from", models.DateTimeField(default=django.utils.timezone.now)), + ("valid_to", models.DateTimeField(blank=True, null=True)), + ("offices", models.ManyToManyField(related_name="api_tokens", to="country_workspace.office")), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="auth_tokens", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "constraints": [ + models.CheckConstraint( + condition=models.Q( + ("valid_to__isnull", True), ("valid_to__gt", models.F("valid_from")), _connector="OR" + ), + name="api_token_valid_period", + violation_error_message="Valid to must be later than valid from.", + ) + ], + }, + ), + ] diff --git a/src/country_workspace/models/__init__.py b/src/country_workspace/models/__init__.py index cf1e5485..737996e0 100644 --- a/src/country_workspace/models/__init__.py +++ b/src/country_workspace/models/__init__.py @@ -1,3 +1,4 @@ +from .api_token import APIToken from .batch import Batch from .beneficiary_group import BeneficiaryGroup from .data_serializer import DataSerializer @@ -16,6 +17,7 @@ from .user import User __all__ = [ + "APIToken", "Area", "AreaType", "AsyncJob", diff --git a/src/country_workspace/models/api_token.py b/src/country_workspace/models/api_token.py new file mode 100644 index 00000000..cb7e3256 --- /dev/null +++ b/src/country_workspace/models/api_token.py @@ -0,0 +1,55 @@ +import secrets +from datetime import datetime +from typing import Any + +from django.conf import settings +from django.db import models +from django.db.models import F, Q +from django.utils import timezone + +from .office import Office + + +class APIToken(models.Model): + """Token model for API authentication.""" + + class GrantType(models.TextChoices): + """Types of token grants.""" + + HOPE_RDI_CALLBACK = "HOPE_RDI_CALLBACK", "HOPE RDI Callback" + + key = models.CharField(max_length=40, primary_key=True, editable=False) + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE) + grant_type = models.CharField(max_length=50, choices=GrantType.choices) + offices = models.ManyToManyField(Office, related_name="api_tokens") + created = models.DateTimeField(auto_now_add=True) + valid_from = models.DateTimeField(default=timezone.now) + valid_to = models.DateTimeField(blank=True, null=True) + + class Meta: + constraints = [ + models.CheckConstraint( + condition=Q(valid_to__isnull=True) | Q(valid_to__gt=F("valid_from")), + name="api_token_valid_period", + violation_error_message="Valid to must be later than valid from.", + ), + ] + + def __str__(self) -> str: + return f"{self.get_grant_type_display()} token for {self.user}" + + def save(self, *args: Any, **kwargs: Any) -> None: + if not self.key or self.key.isspace(): + self.key = self.generate_key() + if self._state.adding: + kwargs["force_insert"] = True + super().save(*args, **kwargs) + + def is_valid_at(self, value: datetime) -> bool: + """Return whether the token is valid at the given time.""" + return self.valid_from <= value and (self.valid_to is None or value < self.valid_to) + + @classmethod + def generate_key(cls) -> str: + """Generate a secure random key for the token.""" + return secrets.token_hex(20) diff --git a/src/country_workspace/models/rdp.py b/src/country_workspace/models/rdp.py index e706d153..dad55cca 100644 --- a/src/country_workspace/models/rdp.py +++ b/src/country_workspace/models/rdp.py @@ -59,6 +59,12 @@ class Meta: name="uniq_pending_rdp_per_program", violation_error_message=_("There is already an active (PENDING) RDP for this program."), ), + models.UniqueConstraint( + fields=["hope_rdi_id"], + condition=Q(hope_rdi_id__isnull=False), + name="uniq_rdp_hope_rdi_id", + violation_error_message=_("There is already an RDP for this HOPE RDI."), + ), ] permissions = [ ("reset_rdp", _("Can reset RDP")),