From 4cb8f2dd77cb4fc113376660003a50a9a92c5d8b Mon Sep 17 00:00:00 2001 From: Peter Xia Date: Mon, 15 Jul 2024 14:28:42 -0700 Subject: [PATCH 1/2] Updates retry logic. Adds more error handling, exponential backoff, and saves httpxAsyncClient in object. --- pikpakapi/__init__.py | 73 ++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/pikpakapi/__init__.py b/pikpakapi/__init__.py index db9c443..beccc61 100644 --- a/pikpakapi/__init__.py +++ b/pikpakapi/__init__.py @@ -1,6 +1,8 @@ import binascii from hashlib import md5 import json +import logging +import asyncio from base64 import b64decode, b64encode from typing import Any, Dict, List, Optional from .utils import ( @@ -73,7 +75,7 @@ def __init__( ) self.captcha_token = None - self.httpx_client_args = httpx_client_args + self.httpx_client = httpx.AsyncClient(**httpx_client_args) self._path_id_cache = {} @@ -118,30 +120,57 @@ def get_headers(self, access_token: Optional[str] = None) -> Dict[str, str]: return headers async def _make_request( - self, method: str, url: str, data=None, params=None, headers=None, retry=0 + self, method: str, url: str, data=None, params=None, headers=None ) -> Dict[str, Any]: - async with httpx.AsyncClient(**self.httpx_client_args) as client: - response = await client.request( - method, - url, - json=data, - params=params, - headers=self.get_headers() if not headers else headers, - ) + backoff_seconds = 3 + error_decription = "" + for i in range(3): # retries + # headers can be different for each request with captcha + if headers is None: + req_headers = self.get_headers() + else: + req_headers = headers + try: + response = await self.httpx_client.request( + method, + url, + json=data, + params=params, + headers=req_headers, + ) + except httpx.HTTPError as e: + logging.error(e) + await asyncio.sleep(backoff_seconds) + backoff_seconds *= 2 # exponential backoff + continue + except KeyboardInterrupt as e: + sys.exit(0) + except Exception as e: + logging.error(e) + await asyncio.sleep(backoff_seconds) + backoff_seconds *= 2 # exponential backoff + continue + json_data = response.json() + if "error" not in json_data: + # ok + return json_data + + if len(json_data) == 0: + error_decription = "empty json data" + await asyncio.sleep(backoff_seconds) + backoff_seconds *= 2 # exponential backoff + continue + elif json_data["error_code"] == 16: + await self.refresh_access_token() + continue + # goes to next iteration in retry loop + else: + await asyncio.sleep(backoff_seconds) + backoff_seconds *= 2 # exponential backoff + continue - if "error" in json_data: - if json_data.get("error_code") == 16: - await self.refresh_access_token() - return await self._make_request(method, url, data, params) - else: - while retry <= 3: - retry += 1 - await self._make_request( - method, url, data, params, headers, retry - ) - raise PikpakException(f"{json_data['error_description']}") - return json_data + raise PikpakException(error_decription) async def _request_get( self, From edfd2a2404bd10413133794e42ace99b65ec5905 Mon Sep 17 00:00:00 2001 From: Peter Xia Date: Mon, 15 Jul 2024 14:32:19 -0700 Subject: [PATCH 2/2] Fix json_data empty check --- pikpakapi/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pikpakapi/__init__.py b/pikpakapi/__init__.py index beccc61..411a34c 100644 --- a/pikpakapi/__init__.py +++ b/pikpakapi/__init__.py @@ -152,11 +152,11 @@ async def _make_request( continue json_data = response.json() - if "error" not in json_data: + if json_data and "error" not in json_data: # ok return json_data - if len(json_data) == 0: + if not json_data: error_decription = "empty json data" await asyncio.sleep(backoff_seconds) backoff_seconds *= 2 # exponential backoff