Retry ETTN creation on Nova Poshta rate limit instead of failing

Checkbox relays Nova Poshta's "To many requests" (20000401501) as a 4xx
third_party.generic error, not a 429, so bursts of receipt requests ended
up as failed receipts. Such responses are now CheckboxRateLimitedError:
the receipt stays pending and the worker job is retried with arq.Retry
after the "Try again after N seconds" delay plus backoff. NP timeouts
relayed the same way are treated as unavailable (unknown outcome).

The HTTP client also sends Checkbox requests one at a time with a
CHECKBOX_MIN_REQUEST_INTERVAL_MS pause and signs the cashier in once for
concurrent jobs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/tests/test_checkbox_client.py

@@ -2,7 +2,9 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
+import time
 import uuid
 
 import httpx
@@ -14,6 +16,7 @@ from app.core.config import Settings
 from app.services.checkbox.client import (
     CheckboxCredentials,
     CheckboxError,
+    CheckboxRateLimitedError,
     CheckboxUnavailableError,
 )
 from app.services.checkbox.http_client import HttpCheckboxClient
@@ -35,11 +38,12 @@ ETTN = {
 }
 
 
-def _client() -> HttpCheckboxClient:
+def _client(interval_ms: int = 0) -> HttpCheckboxClient:
     settings = Settings(
         secret_key="test-secret-key",
         encryption_key="dGVzdC1lbmNyeXB0aW9uLWtleS0zMi1ieXRlcyEh",
         checkbox_base_url=BASE,
+        checkbox_min_request_interval_ms=interval_ms,
     )  # type: ignore[arg-type]
     return HttpCheckboxClient(settings)
 
@@ -178,5 +182,78 @@ async def test_rate_limit_is_retryable() -> None:
     respx.post(f"{BASE}/api/v1/ettn").mock(
         return_value=Response(429, json={"message": "Занадто часто виконуються запити"})
     )
-    with pytest.raises(CheckboxUnavailableError):
+    with pytest.raises(CheckboxRateLimitedError):
         await _client().create_ettn(CREDS, {})
+
+
+def _np_error(message: str, info: list[str], codes: list[str]) -> Response:
+    """Ошибка Новой Почты в том виде, в каком её отдаёт боевой Checkbox."""
+    inner = {
+        "type": "ettn",
+        "message": message,
+        "code": 0,
+        "context": {"success": False, "errors": [message], "info": info, "errorCodes": codes},
+    }
+    return Response(400, json={"code": "third_party.generic", "message": json.dumps(inner)})
+
+
+@respx.mock
+async def test_np_too_many_requests_is_rate_limited() -> None:
+    _signin()
+    respx.post(f"{BASE}/api/v1/ettn").mock(
+        return_value=_np_error("To many requests", ["Try again after 3 seconds"], ["20000401501"])
+    )
+    with pytest.raises(CheckboxRateLimitedError) as exc_info:
+        await _client().create_ettn(CREDS, {})
+    assert exc_info.value.retry_after == 3
+
+
+@respx.mock
+async def test_np_zero_retry_after_waits_at_least_a_second() -> None:
+    _signin()
+    respx.post(f"{BASE}/api/v1/ettn").mock(
+        return_value=_np_error("To many requests", ["Try again after 0 seconds"], ["20000401501"])
+    )
+    with pytest.raises(CheckboxRateLimitedError) as exc_info:
+        await _client().create_ettn(CREDS, {})
+    assert exc_info.value.retry_after == 1
+
+
+@respx.mock
+async def test_np_timeout_is_unavailable_not_rate_limited() -> None:
+    _signin()
+    respx.post(f"{BASE}/api/v1/ettn").mock(
+        return_value=_np_error("cURL error 28: SSL connection timeout", [], [])
+    )
+    with pytest.raises(CheckboxUnavailableError) as exc_info:
+        await _client().create_ettn(CREDS, {})
+    assert not isinstance(exc_info.value, CheckboxRateLimitedError)
+
+
+@respx.mock
+async def test_concurrent_calls_sign_in_once() -> None:
+    signin = _signin()
+    respx.get(f"{BASE}/api/v1/ettn/e-1").mock(return_value=Response(200, json=ETTN))
+    client = _client()
+
+    await asyncio.gather(*(client.get_ettn(CREDS, "e-1") for _ in range(5)))
+
+    assert signin.call_count == 1
+
+
+@respx.mock
+async def test_requests_are_spaced_by_min_interval() -> None:
+    _signin()
+    sent_at: list[float] = []
+
+    def record(_: httpx.Request) -> Response:
+        sent_at.append(time.monotonic())
+        return Response(200, json=ETTN)
+
+    respx.get(f"{BASE}/api/v1/ettn/e-1").mock(side_effect=record)
+    client = _client(interval_ms=50)
+
+    await asyncio.gather(*(client.get_ettn(CREDS, "e-1") for _ in range(3)))
+
+    gaps = [b - a for a, b in zip(sent_at, sent_at[1:], strict=False)]
+    assert len(gaps) == 2 and all(gap >= 0.045 for gap in gaps)

backend/tests/test_receipts_service.py

@@ -18,7 +18,7 @@ from app.db.models.order import Order
 from app.db.models.receipt import Receipt, ReceiptStatus
 from app.schemas.checkbox import EttnStatus
 from app.services import receipts as svc
-from app.services.checkbox.client import CheckboxUnavailableError
+from app.services.checkbox.client import CheckboxRateLimitedError, CheckboxUnavailableError
 from app.services.checkbox.stub_client import StubCheckboxClient
 
 
@@ -266,6 +266,24 @@ class TestCreateEttn:
         assert receipt.status == ReceiptStatus.CREATED
         assert len(client.orders) == 1  # второго чека на ту же ТТН нет
 
+    async def test_rate_limit_keeps_pending_without_unknown_outcome(self) -> None:
+        order, register = _order(receipt_created_at=datetime.now(UTC)), _register()
+        receipt = _pending(order, register)
+        client = StubCheckboxClient()
+        session = FakeSession(order, register, receipt)
+
+        async def rate_limited(creds: Any, body: dict[str, Any]) -> Any:
+            raise CheckboxRateLimitedError("To many requests", retry_after=1)
+
+        client.create_ettn = rate_limited  # type: ignore[method-assign]
+        with pytest.raises(CheckboxRateLimitedError):
+            await svc.create_ettn_for_receipt(session, client, receipt.id)
+
+        assert receipt.status == ReceiptStatus.PENDING
+        assert receipt.error is None  # повтор не пойдёт через find_ettn
+        assert order.receipt_created_at is not None  # заказ не вернулся в очередь
+        assert session.commits == 1  # блокировка строки снята
+
 
 class TestApplyEttn:
     @pytest.mark.parametrize(