diff --git a/.github/workflows/test_deploy.yaml b/.github/workflows/test_deploy.yaml index 15fde40a..29b556ea 100644 --- a/.github/workflows/test_deploy.yaml +++ b/.github/workflows/test_deploy.yaml @@ -51,6 +51,8 @@ jobs: TOKEN_DIGIKEY: ${{ secrets.TOKEN_DIGIKEY }} DIGIKEY_CLIENT_ID: ${{ secrets.DIGIKEY_CLIENT_ID }} DIGIKEY_CLIENT_SECRET: ${{ secrets.DIGIKEY_CLIENT_SECRET }} + TME_API_TOKEN: ${{ secrets.TME_API_TOKEN }} + TME_API_SECRET: ${{ secrets.TME_API_SECRET }} continue-on-error: true strategy: diff --git a/kintree/common/tools.py b/kintree/common/tools.py index 363fae8d..bfa8f24b 100644 --- a/kintree/common/tools.py +++ b/kintree/common/tools.py @@ -67,17 +67,22 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= import urllib.request import requests + headers = { + 'User-Agent': 'Mozilla/5.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + } + # Set default timeout for download socket socket.setdefaulttimeout(timeout) - if enable_headers: + if enable_headers and not requests_lib: opener = urllib.request.build_opener() - opener.addheaders = [('User-agent', 'Mozilla/5.0')] + opener.addheaders = list(headers.items()) urllib.request.install_opener(opener) try: if filetype == 'Image' or filetype == 'PDF': # Enable use of requests library for downloading files (some URLs do NOT work with urllib) if requests_lib: - headers = {'User-agent': 'Mozilla/5.0'} response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True) if filetype.lower() not in response.headers['Content-Type'].lower(): cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent) @@ -99,7 +104,7 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= cprint(f'[INFO]\tWarning: {filetype} download socket timed out ({timeout}s)', silent=silent) except (urllib.error.HTTPError, requests.exceptions.ConnectionError): cprint(f'[INFO]\tWarning: {filetype} download failed (HTTP Error)', silent=silent) - except (urllib.error.URLError, ValueError): + except (urllib.error.URLError, ValueError, AttributeError): cprint(f'[INFO]\tWarning: {filetype} download failed (URL Error)', silent=silent) except requests.exceptions.SSLError: cprint(f'[INFO]\tWarning: {filetype} download failed (SSL Error)', silent=silent) @@ -114,7 +119,7 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str if not url: cprint('[INFO]\tError: Missing image URL', silent=silent) return False - + # Try without headers file = download(url, fileoutput=full_path, silent=silent, **kwargs) diff --git a/kintree/config/tme/tme_api.yaml b/kintree/config/tme/tme_api.yaml index 76645ddb..841ee909 100644 --- a/kintree/config/tme/tme_api.yaml +++ b/kintree/config/tme/tme_api.yaml @@ -1,4 +1,4 @@ TME_API_TOKEN: NULL TME_API_SECRET: NULL -TME_API_COUNTRY: NULL -TME_API_LANGUAGE: NULL \ No newline at end of file +TME_API_COUNTRY: US +TME_API_LANGUAGE: EN \ No newline at end of file diff --git a/kintree/database/inventree_interface.py b/kintree/database/inventree_interface.py index ce230171..4ca6a117 100644 --- a/kintree/database/inventree_interface.py +++ b/kintree/database/inventree_interface.py @@ -262,7 +262,11 @@ def translate_form_to_inventree(part_info: dict, category_tree: list, is_custom= inventree_part['supplier_link'] = part_info['supplier_link'].replace(' ', '%20') inventree_part['datasheet'] = part_info['datasheet'].replace(' ', '%20') # Image URL is not shown to user so force default key/value - inventree_part['image'] = part_info['image'].replace(' ', '%20') + try: + inventree_part['image'] = part_info['image'].replace(' ', '%20') + except AttributeError: + # Part image URL is null (no product picture) + pass inventree_part['pricing'] = part_info.get('pricing', {}) # Load parameters map @@ -575,12 +579,12 @@ def inventree_create(part_info: dict, kicad=False, symbol=None, footprint=None, cprint('[TREE]\tWarning: Failed to upload part image', silent=settings.SILENT) if inventree_part['datasheet'] and settings.DATASHEET_UPLOAD: # Upload datasheet - datasheet_link = inventree_api.upload_part_datasheet( - inventree_part['datasheet'], part_pk) + datasheet_link = inventree_api.upload_part_datasheet(inventree_part['datasheet'], part_pk) if not datasheet_link: cprint('[TREE]\tWarning: Failed to upload part datasheet', silent=settings.SILENT) - else: - inventree_part['datasheet'] = datasheet_link + # TODO: this is messing up with the datasheet download to local folder + # else: + # inventree_part['datasheet'] = datasheet_link if kicad: try: @@ -753,7 +757,7 @@ def inventree_create_alternate(part_info: dict, part_id='', part_ipn='', show_pr manufacturer_mpn = part_info.get('manufacturer_part_number', '') datasheet = part_info.get('datasheet', '') - # if datasheet upload is enabled and no attechment present yet upload + # if datasheet upload is enabled and no attachment present yet then upload if settings.DATASHEET_UPLOAD and not part.getAttachments(): if datasheet: inventree_api.upload_part_datasheet(part_id=part_pk, diff --git a/kintree/gui/views/main.py b/kintree/gui/views/main.py index f06bfff7..9b2b7b17 100644 --- a/kintree/gui/views/main.py +++ b/kintree/gui/views/main.py @@ -11,7 +11,7 @@ from .common import DropdownWithSearch, SwitchWithRefs from .common import handle_transition # Tools -from ...common.tools import cprint, download +from ...common.tools import cprint, download_with_retry # Settings from ...common import progress from ...config import settings, config_interface @@ -1321,7 +1321,7 @@ def create_part(self, e=None): f'{part_info.get("IPN", "datasheet")}.pdf', ) cprint('\n[MAIN]\tDownloading Datasheet') - if download(datasheet_url, filetype='PDF', fileoutput=filename, timeout=10): + if download_with_retry(datasheet_url, filename, filetype='PDF', timeout=10): cprint(f'[INFO]\tSuccess: Datasheet saved to {filename}') # Open browser if settings.ENABLE_INVENTREE: diff --git a/kintree/search/tme_api.py b/kintree/search/tme_api.py index 09951e77..612a264c 100644 --- a/kintree/search/tme_api.py +++ b/kintree/search/tme_api.py @@ -38,8 +38,8 @@ def check_environment() -> bool: def setup_environment(force=False) -> bool: if not check_environment() or force: tme_api_settings = config_interface.load_file(settings.CONFIG_TME_API) - os.environ['TME_API_TOKEN'] = tme_api_settings['TME_API_TOKEN'] - os.environ['TME_API_SECRET'] = tme_api_settings['TME_API_SECRET'] + os.environ['TME_API_TOKEN'] = tme_api_settings.get('TME_API_TOKEN', None) + os.environ['TME_API_SECRET'] = tme_api_settings.get('TME_API_SECRET', None) return check_environment() @@ -47,17 +47,25 @@ def setup_environment(force=False) -> bool: # Based on TME API snippets mentioned in API documentation: https://developers.tme.eu/documentation/download # https://github.com/tme-dev/TME-API/blob/master/Python/call.py def tme_api_request(endpoint, tme_api_settings, part_number, api_host='https://api.tme.eu', format='json'): + TME_API_TOKEN = tme_api_settings.get('TME_API_TOKEN', None) + TME_API_SECRET = tme_api_settings.get('TME_API_SECRET', None) + params = collections.OrderedDict() - params['Country'] = tme_api_settings['TME_API_COUNTRY'] - params['Language'] = tme_api_settings['TME_API_LANGUAGE'] + params['Country'] = tme_api_settings.get('TME_API_COUNTRY', 'US') + params['Language'] = tme_api_settings.get('TME_API_LANGUAGE', 'EN') params['SymbolList[0]'] = part_number - params['Token'] = tme_api_settings['TME_API_TOKEN'] + if not TME_API_TOKEN and not TME_API_SECRET: + TME_API_TOKEN = os.environ.get('TME_API_TOKEN', None) + TME_API_SECRET = os.environ.get('TME_API_SECRET', None) + if not TME_API_TOKEN and not TME_API_SECRET: + return None + params['Token'] = TME_API_TOKEN url = api_host + endpoint + '.' + format encoded_params = urllib.parse.urlencode(params, quote_via=urllib.parse.quote) signature_base = 'POST' + '&' + urllib.parse.quote(url, '') + '&' + urllib.parse.quote(encoded_params, '') hmac_value = hmac.new( - tme_api_settings['TME_API_SECRET'].encode(), + TME_API_SECRET.encode(), signature_base.encode(), hashlib.sha1 ).digest() diff --git a/run_tests.py b/run_tests.py index f639f8d6..dd23482f 100644 --- a/run_tests.py +++ b/run_tests.py @@ -2,7 +2,7 @@ import sys import kintree.config.settings as settings -from kintree.common.tools import cprint, create_library, download, download_with_retry +from kintree.common.tools import cprint, create_library, download_with_retry from kintree.config import config_interface from kintree.database import inventree_api, inventree_interface from kintree.kicad import kicad_interface @@ -386,20 +386,26 @@ def check_result(status: str, new_part: bool) -> bool: test_pdf_urllib = 'https://www.seielect.com/Catalog/SEI-CF_CFM.pdf' # Test different download methods for images if not download_with_retry(test_image_urllib, './image1.jpg', silent=True, filetype='Image'): + print(' [1] ') method_success = False - if not download_with_retry(test_image_requestslib, './image2.jpg', silent=True, filetype='Image',): + if not download_with_retry(test_image_requestslib, './image2.jpg', silent=True, filetype='Image'): + print(' [2] ') method_success = False # Test PDF - if not download(test_pdf_urllib, filetype='PDF', fileoutput='./datasheet.pdf', silent=True): + if not download_with_retry(test_pdf_urllib, './datasheet.pdf', silent=True, filetype='PDF'): + print(' [3] ') method_success = False # Wrong folder - if download(test_pdf_urllib, filetype='PDF', fileoutput='./myfolder/datasheet.pdf', silent=True): + if download_with_retry(test_pdf_urllib, './myfolder/datasheet.pdf', silent=True, filetype='PDF'): + print(' [4] ') method_success = False # Test erroneous URL if download_with_retry('http', '', silent=True): + print(' [5] ') method_success = False # Test empty URL if download_with_retry('', '', silent=True): + print(' [6] ') method_success = False elif method_idx == 9: diff --git a/tests/files/results.tgz b/tests/files/results.tgz index c08f4795..df05accc 100644 Binary files a/tests/files/results.tgz and b/tests/files/results.tgz differ