Retry ETTN creation on Nova Poshta rate limit instead of failing

Checkbox relays Nova Poshta's "To many requests" (20000401501) as a 4xx
third_party.generic error, not a 429, so bursts of receipt requests ended
up as failed receipts. Such responses are now CheckboxRateLimitedError:
the receipt stays pending and the worker job is retried with arq.Retry
after the "Try again after N seconds" delay plus backoff. NP timeouts
relayed the same way are treated as unavailable (unknown outcome).

The HTTP client also sends Checkbox requests one at a time with a
CHECKBOX_MIN_REQUEST_INTERVAL_MS pause and signs the cashier in once for
concurrent jobs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/checkbox/http_client.py

@@ -2,10 +2,17 @@
 
 Авторизация — токен кассира по PIN-коду (`/api/v1/cashier/signinPinCode`).
 Токен кэшируется в памяти процесса по кассе; на 401 — один повторный вход.
+
+Запросы из одного процесса идут строго по одному с паузой
+`CHECKBOX_MIN_REQUEST_INTERVAL_MS`: при создании ЕТТН Checkbox синхронно ходит
+в API Новой Почты, а та на частые вызовы отвечает «To many requests».
 """
 
 from __future__ import annotations
 
+import asyncio
+import re
+import time
 import uuid
 from datetime import UTC, datetime, timedelta
 from typing import Any
@@ -17,6 +24,7 @@ from app.schemas.checkbox import EttnOut, EttnStatus
 from app.services.checkbox.client import (
     CheckboxCredentials,
     CheckboxError,
+    CheckboxRateLimitedError,
     CheckboxUnavailableError,
 )
 
@@ -27,6 +35,15 @@ _TIMEOUT = httpx.Timeout(30, connect=10)
 _FIND_PAGES = 3
 _FIND_PAGE_SIZE = 50
 
+# Ошибки Новой Почты, которые Checkbox отдаёт 4xx с `code=third_party.*`:
+# {"message": "To many requests", "info": ["Try again after 1 seconds"],
+#  "errorCodes": ["20000401501"]} — лимит частоты НП;
+# "cURL error 28: SSL connection timeout" — Checkbox не дождался НП.
+_NP_RATE_LIMIT_RE = re.compile(r"20000401501|too? many requests", re.IGNORECASE)
+_NP_RETRY_AFTER_RE = re.compile(r"try again after (\d+) second", re.IGNORECASE)
+_NP_TIMEOUT_RE = re.compile(r"curl error|timed? ?out", re.IGNORECASE)
+_MIN_RETRY_AFTER = 1.0
+
 
 def _error_message(response: httpx.Response) -> str:
     try:
@@ -55,6 +72,29 @@ def _error_message(response: httpx.Response) -> str:
     return f"HTTP {response.status_code}"
 
 
+def _retry_after(response: httpx.Response) -> float:
+    seconds: float = 0
+    if match := _NP_RETRY_AFTER_RE.search(response.text):
+        seconds = float(match.group(1))
+    elif (header := response.headers.get("Retry-After", "")).isdigit():
+        seconds = float(header)
+    return max(seconds, _MIN_RETRY_AFTER)
+
+
+def _transient_error(response: httpx.Response) -> CheckboxUnavailableError | None:
+    """Ответ, после которого запрос можно повторить, или None, если ошибка окончательная."""
+    if response.status_code == 429:
+        return CheckboxRateLimitedError(_error_message(response), _retry_after(response))
+    if response.status_code >= 500:
+        return CheckboxUnavailableError(_error_message(response))
+    if response.status_code >= 400 and '"third_party.' in response.text:
+        if _NP_RATE_LIMIT_RE.search(response.text):
+            return CheckboxRateLimitedError(_error_message(response), _retry_after(response))
+        if _NP_TIMEOUT_RE.search(response.text):
+            return CheckboxUnavailableError(_error_message(response))
+    return None
+
+
 class HttpCheckboxClient:
     def __init__(self, settings: Settings) -> None:
         self._base_url = settings.checkbox_base_url.rstrip("/")
@@ -63,6 +103,10 @@ class HttpCheckboxClient:
             "X-Client-Version": settings.checkbox_client_version,
         }
         self._tokens: dict[uuid.UUID, str] = {}
+        self._min_interval = settings.checkbox_min_request_interval_ms / 1000
+        self._send_lock = asyncio.Lock()
+        self._sign_in_lock = asyncio.Lock()
+        self._last_sent_at = float("-inf")
 
     async def _send(
         self,
@@ -73,18 +117,36 @@ class HttpCheckboxClient:
         json: Any = None,
         params: dict[str, Any] | None = None,
     ) -> httpx.Response:
-        try:
-            async with httpx.AsyncClient(base_url=self._base_url, timeout=_TIMEOUT) as client:
-                response = await client.request(
-                    method, path, headers=headers, json=json, params=params
-                )
-        except httpx.HTTPError as exc:
-            raise CheckboxUnavailableError(f"Checkbox недоступен: {exc!r}") from exc
-        # 429 — «Занадто часто виконуються запити»: исход не ошибка данных, повторяемо.
-        if response.status_code >= 500 or response.status_code == 429:
-            raise CheckboxUnavailableError(_error_message(response))
+        # Строго по одному запросу с паузой — параллельные задачи worker'а
+        # иначе пачкой упираются в лимит Новой Почты.
+        async with self._send_lock:
+            delay = self._last_sent_at + self._min_interval - time.monotonic()
+            if delay > 0:
+                await asyncio.sleep(delay)
+            self._last_sent_at = time.monotonic()
+            try:
+                async with httpx.AsyncClient(base_url=self._base_url, timeout=_TIMEOUT) as client:
+                    response = await client.request(
+                        method, path, headers=headers, json=json, params=params
+                    )
+            except httpx.HTTPError as exc:
+                raise CheckboxUnavailableError(f"Checkbox недоступен: {exc!r}") from exc
+        if error := _transient_error(response):
+            raise error
         return response
 
+    async def _token(self, creds: CheckboxCredentials, *, stale: str | None = None) -> str:
+        """Токен кассира; `stale` — отвергнутый Checkbox'ом (401), его не переиспользуем.
+
+        Под блокировкой: параллельные задачи без токена входят один раз,
+        остальные берут токен, полученный первой.
+        """
+        async with self._sign_in_lock:
+            token = self._tokens.get(creds.cash_register_id)
+            if token is not None and token != stale:
+                return token
+            return await self._sign_in(creds)
+
     async def _sign_in(self, creds: CheckboxCredentials) -> str:
         response = await self._send(
             "POST",
@@ -107,7 +169,7 @@ class HttpCheckboxClient:
         json: Any = None,
         params: dict[str, Any] | None = None,
     ) -> httpx.Response:
-        token = self._tokens.get(creds.cash_register_id) or await self._sign_in(creds)
+        token = await self._token(creds)
         for attempt in range(2):
             response = await self._send(
                 method,
@@ -121,7 +183,7 @@ class HttpCheckboxClient:
                 params=params,
             )
             if response.status_code == 401 and attempt == 0:
-                token = await self._sign_in(creds)
+                token = await self._token(creds, stale=token)
                 continue
             break
         if response.status_code >= 400: