test: add live coverage manifest

Author: subzeroid

README.md

@@ -454,14 +454,20 @@ TEST_ACCOUNTS_URL="https://example.com/accounts" python3.13 -m pytest tests/live
 GitHub Actions also has a scheduled **Live Tests** workflow that runs nightly
 and can be launched manually. It starts the Docker Compose API service, creates
 a real session from `TEST_ACCOUNTS_URL`, sends it through `X-Session-ID`, and
-checks `/user/about`, `/media/comments`, and paginated read-list routes. A
-second nightly job runs the same HTTP smoke against the published Docker image
+checks `/user/about`, `/media/comments`, `/account/family`,
+`/user/featured/accounts`, `/user/fundraiser`, and paginated read-list routes.
+A second nightly job runs the same HTTP smoke against the published Docker image
 `subzeroid/aiograpi-rest:latest`, so the public `docker run` path is exercised
-with a real session too. Both the direct ASGI live smoke and the published
-image HTTP smoke upload a real JPEG to `/story/upload`, verify the created
-story through `/story`, `/user/stories`, and `/story/viewers`, download the
-media through `/story/download`, validate that it is an image, and delete the
-story.
+with a real session too.
+
+Every public OpenAPI operation is classified by the live coverage manifest as a
+read, download, session, reversible mutation, upload, cleanup, or guarded
+operation. Non-guarded mutations must name the read-back or cleanup endpoint
+that proves the Instagram state changed. Both the direct ASGI live smoke and
+the published image HTTP smoke upload a real JPEG to `/story/upload`, verify
+the created story through `/story`, `/user/stories`, and `/story/viewers`,
+download the media through `/story/download`, validate that it is an image, and
+delete the story.
 
 Generate and validate docs:
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aiograpi-rest"
-version = "5.15.0"
+version = "5.15.1"
 description = "RESTful API service for aiograpi"
 readme = "README.md"
 requires-python = ">=3.13"

tests/live/coverage_manifest.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class LivePolicy:
+    kind: str
+    reason: str = ""
+    verify_with: str = ""
+
+
+SYSTEM_PATHS = {
+    "/health",
+    "/ready",
+    "/metrics",
+    "/build",
+    "/deps",
+}
+
+SESSION_MUTATIONS = {
+    ("POST", "/auth/login"): "/account",
+    ("POST", "/auth/login/by/sessionid"): "/account",
+    ("PATCH", "/auth/relogin"): "/account",
+    ("PATCH", "/auth/settings"): "/auth/settings",
+}
+
+REVERSIBLE_MUTATIONS = {
+    ("POST", "/hashtag/follow"): "DELETE /hashtag/follow",
+    ("DELETE", "/hashtag/follow"): "GET /hashtag",
+    ("POST", "/media/like"): "DELETE /media/like",
+    ("DELETE", "/media/like"): "GET /media",
+    ("PATCH", "/media/seen"): "GET /media",
+    ("POST", "/media/comment"): "GET /media/comments",
+    ("DELETE", "/media/comment"): "GET /media/comments",
+    ("POST", "/media/comment/check/offensive"): "same response",
+    ("POST", "/media/comment/like"): "DELETE /media/comment/like",
+    ("DELETE", "/media/comment/like"): "GET /media/comment/likers",
+    ("POST", "/media/save"): "DELETE /media/save",
+    ("DELETE", "/media/save"): "GET /account/liked/media",
+    ("POST", "/note"): "GET /notes",
+    ("DELETE", "/note"): "GET /notes",
+    ("POST", "/note/music"): "GET /notes",
+    ("PATCH", "/notes/last-seen"): "GET /notes",
+    ("POST", "/story/like"): "DELETE /story/like",
+    ("DELETE", "/story/like"): "GET /story",
+    ("PATCH", "/story/seen"): "GET /story",
+    ("POST", "/user/follow"): "GET /user/friendship",
+    ("DELETE", "/user/follow"): "GET /user/friendship",
+    ("DELETE", "/user/follower"): "GET /user/friendship",
+    ("POST", "/user/close-friend"): "GET /user/friendship",
+    ("DELETE", "/user/close-friend"): "GET /user/friendship",
+    ("POST", "/user/mute/posts"): "GET /user/friendship",
+    ("DELETE", "/user/mute/posts"): "GET /user/friendship",
+    ("POST", "/user/mute/stories"): "GET /user/friendship",
+    ("DELETE", "/user/mute/stories"): "GET /user/friendship",
+    ("POST", "/user/notifications/posts"): "GET /user/friendship",
+    ("DELETE", "/user/notifications/posts"): "GET /user/friendship",
+    ("POST", "/user/notifications/stories"): "GET /user/friendship",
+    ("DELETE", "/user/notifications/stories"): "GET /user/friendship",
+    ("POST", "/user/notifications/reels"): "GET /user/friendship",
+    ("DELETE", "/user/notifications/reels"): "GET /user/friendship",
+    ("POST", "/user/notifications/videos"): "GET /user/friendship",
+    ("DELETE", "/user/notifications/videos"): "GET /user/friendship",
+    ("POST", "/user/block"): "GET /user/friendship",
+    ("DELETE", "/user/block"): "GET /user/friendship",
+}
+
+UPLOAD_MUTATIONS = {
+    ("POST", "/album/upload"): "GET /media",
+    ("POST", "/album/upload/with/music"): "GET /media",
+    ("POST", "/clip/upload"): "GET /media",
+    ("POST", "/clip/upload/by/url"): "GET /media",
+    ("POST", "/clip/upload/with/music"): "GET /media",
+    ("POST", "/igtv/upload"): "GET /media",
+    ("POST", "/igtv/upload/by/url"): "GET /media",
+    ("POST", "/photo/upload"): "GET /media",
+    ("POST", "/photo/upload/by/url"): "GET /media",
+    ("POST", "/photo/upload/with/music"): "GET /media",
+    ("POST", "/story/upload"): "GET /story + GET /user/stories + GET /story/download",
+    ("POST", "/story/upload/by/url"): "GET /story + GET /user/stories + GET /story/download",
+    ("POST", "/video/upload"): "GET /media",
+    ("POST", "/video/upload/by/url"): "GET /media",
+}
+
+GUARDED_PREFIX_REASONS = {
+    ("/account", "account mutation"): "changes authenticated account state or depends on inbound follow requests",
+    ("/auth/challenge", "challenge"): "requires a real active Instagram challenge",
+    ("/auth/totp", "totp state"): "changes two-factor authentication state",
+    ("/clip/pin", "owned reel"): "requires owned Reel fixture and profile cleanup",
+    ("/direct", "direct fixture"): "requires controlled Direct threads, users, and messages",
+    ("/highlight", "highlight fixture"): "requires owned story/highlight fixtures",
+    ("/media/archive", "owned media"): "requires owned media and archive state cleanup",
+    ("/media/livestream", "live broadcast"): "creates or mutates a real livestream",
+    ("/media/pin", "owned media"): "requires owned media and visible profile cleanup",
+    ("/media/comment/pin", "owned comment"): "requires owned media comments",
+    ("/notifications", "account settings"): "changes notification settings on the authenticated account",
+}
+
+GUARDED_EXACT_REASONS = {
+    ("DELETE", "/media"): "deletes real media and requires a dedicated upload fixture",
+    ("PATCH", "/media"): "edits real media caption and requires a dedicated upload fixture",
+}
+
+
+def operation_policy(method: str, path: str) -> LivePolicy:
+    method = method.upper()
+
+    if path in SYSTEM_PATHS:
+        return LivePolicy("system")
+    if method == "GET" and path.endswith(("/download", "/download/by/url", "/download/by/urls")):
+        return LivePolicy("download", verify_with="binary/media validation")
+    if method == "GET":
+        if path.startswith("/auth/"):
+            return LivePolicy("session-read")
+        return LivePolicy("read")
+
+    if (method, path) in SESSION_MUTATIONS:
+        return LivePolicy("session", verify_with=SESSION_MUTATIONS[(method, path)])
+    if (method, path) in REVERSIBLE_MUTATIONS:
+        return LivePolicy("reversible", verify_with=REVERSIBLE_MUTATIONS[(method, path)])
+    if (method, path) in UPLOAD_MUTATIONS:
+        return LivePolicy("upload", verify_with=UPLOAD_MUTATIONS[(method, path)])
+    if (method, path) == ("DELETE", "/story"):
+        return LivePolicy("cleanup", verify_with="GET /story returns 404 or missing story")
+
+    guarded_reason = GUARDED_EXACT_REASONS.get((method, path))
+    if guarded_reason:
+        return LivePolicy("guarded", reason=guarded_reason)
+    for prefix, reason in GUARDED_PREFIX_REASONS:
+        if path.startswith(prefix):
+            return LivePolicy("guarded", reason=reason)
+
+    return LivePolicy("unclassified")
+
+
+def guarded_operations() -> dict[tuple[str, str], LivePolicy]:
+    from aiograpi_rest.main import app
+
+    return {
+        (method.upper(), path): policy
+        for path, methods in app.openapi()["paths"].items()
+        for method in methods
+        if (policy := operation_policy(method, path)).kind == "guarded"
+    }

tests/live/test_live_http_smoke.py

@@ -3,6 +3,7 @@
 import os
 import ssl
 import time
+import urllib.error
 import urllib.parse
 import urllib.request
 from io import BytesIO
@@ -44,6 +45,18 @@ def _request_json(base_url, method, path, *, headers=None, data=None):
     return json.loads(payload)
 
 
+def _request_json_result(base_url, method, path, *, headers=None, data=None):
+    try:
+        return 200, _request_json(base_url, method, path, headers=headers, data=data)
+    except urllib.error.HTTPError as exc:
+        payload = exc.read()
+        try:
+            body = json.loads(payload)
+        except json.JSONDecodeError:
+            body = payload.decode(errors="replace")
+        return exc.code, body
+
+
 def _request_bytes(base_url, method, path, *, headers=None):
     req = urllib.request.Request(
         base_url.rstrip("/") + path,
@@ -183,6 +196,28 @@ def _assert_published_http_pagination(base_url, headers, public_user_id):
         )
 
 
+def _assert_published_http_v515_discovery(base_url, headers, public_user_id):
+    family = _request_json(base_url, "GET", "/account/family", headers=headers)
+    assert isinstance(family, dict), f"Unexpected /account/family response: {family!r}"
+
+    featured = _request_json(
+        base_url,
+        "GET",
+        f"/user/featured/accounts?user_id={public_user_id}",
+        headers=headers,
+    )
+    assert isinstance(featured, dict), f"Unexpected /user/featured/accounts response: {featured!r}"
+
+    status, fundraiser = _request_json_result(
+        base_url,
+        "GET",
+        f"/user/fundraiser?user_id={public_user_id}",
+        headers=headers,
+    )
+    assert status in {200, 400, 404}, f"Unexpected /user/fundraiser status {status}: {fundraiser!r}"
+    assert isinstance(fundraiser, dict), f"Unexpected /user/fundraiser response: {fundraiser!r}"
+
+
 def test_live_http_login_authorize_and_user_about_flow():
     accounts_url = os.environ.get("TEST_ACCOUNTS_URL")
     if not accounts_url:
@@ -221,6 +256,7 @@ def test_live_http_login_authorize_and_user_about_flow():
             assert "is_verified" in about
             assert isinstance(about["former_usernames"], str)
             _assert_published_http_pagination(base_url, headers, user["pk"])
+            _assert_published_http_v515_discovery(base_url, headers, user["pk"])
             return
         except Exception as exc:
             errors.append(f"{account.get('username', '?')}: {type(exc).__name__}: {exc}")

tests/test_live_coverage_manifest.py

@@ -0,0 +1,39 @@
+from aiograpi_rest.main import app
+from tests.live.coverage_manifest import guarded_operations, operation_policy
+
+
+def _openapi_operations() -> set[tuple[str, str]]:
+    return {
+        (method.upper(), path)
+        for path, methods in app.openapi()["paths"].items()
+        for method in methods
+    }
+
+
+def test_live_coverage_manifest_classifies_every_openapi_operation():
+    missing = [
+        f"{method} {path}"
+        for method, path in sorted(_openapi_operations())
+        if operation_policy(method, path).kind == "unclassified"
+    ]
+
+    assert missing == []
+
+
+def test_live_coverage_manifest_requires_post_mutation_verification():
+    missing_verification = []
+    for method, path in sorted(_openapi_operations()):
+        policy = operation_policy(method, path)
+        if method in {"POST", "PATCH", "DELETE"} and policy.kind != "guarded":
+            if not policy.verify_with:
+                missing_verification.append(f"{method} {path}")
+
+    assert missing_verification == []
+
+
+def test_live_coverage_manifest_documents_guarded_operations():
+    guarded = guarded_operations()
+
+    assert ("PATCH", "/account/password") in guarded
+    assert ("POST", "/auth/challenge/resolve") in guarded
+    assert all(policy.reason for policy in guarded.values())

tests/test_project_metadata.py

@@ -650,6 +650,9 @@ def test_live_tests_cover_published_image_and_paginated_read_lists():
     for endpoint in (
         "/user/posts",
         "/media/comments",
+        "/account/family",
+        "/user/featured/accounts",
+        "/user/fundraiser",
         "/hashtag/media/top",
         "/direct/inbox",
         "/story/upload",
@@ -659,6 +662,8 @@ def test_live_tests_cover_published_image_and_paginated_read_lists():
         assert endpoint in http_smoke
 
     assert "published Docker image" in readme
+    assert "Every public OpenAPI operation is classified by the live coverage manifest" in readme
+    assert "Non-guarded mutations must name the read-back or cleanup endpoint" in readme
     assert "paginated read-list routes" in readme