diff --git a/.gitignore b/.gitignore index 1c448974..04158a2d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ build/ kintree/tests/* .coverage htmlcov/ +.vscode/launch.json diff --git a/kintree/common/tools.py b/kintree/common/tools.py index 96f600ea..d28416de 100644 --- a/kintree/common/tools.py +++ b/kintree/common/tools.py @@ -60,17 +60,40 @@ def create_library(library_path: str, symbol: str, template_lib: str): copyfile(template_lib, new_kicad_sym_file) -def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, silent=False): +def get_image_with_retries(url, headers, retries=3, wait=5, silent=False): + """ Method to download image with cloudscraper library and retry attempts""" + import cloudscraper + import time + scraper = cloudscraper.create_scraper() + for attempt in range(retries): + try: + response = scraper.get(url, headers=headers) + if response.status_code == 200: + return response + else: + cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} failed with status code {response.status_code}. Retrying in {wait} seconds...', silent=silent) + except Exception as e: + cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} encountered an error: {e}. Retrying in {wait} seconds...', silent=silent) + time.sleep(wait) + cprint('[INFO]\tWarning: All Image download attempts failed. Could not retrieve the image.', silent=silent) + return None + + +def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, try_cloudscraper=False, silent=False): ''' Standard method to download URL content, with option to save to local file (eg. images) ''' import socket import urllib.request import requests + # A more detailed headers was needed for request to Jameco headers = { - 'User-Agent': 'Mozilla/5.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36', + 'Accept': 'applicaiton/json,image/webp,image/apng,image/*,*/*;q=0.8', + 'Accept-Encoding': 'Accept-Encoding: gzip, deflate, br', + 'Accept-Language': 'en-US,en;q=0.9', + 'Connection': 'keep-alive', + 'Cache-Control': 'no-cache', } # Set default timeout for download socket @@ -95,6 +118,13 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= return None with open(fileoutput, 'wb') as file: file.write(response.content) + elif try_cloudscraper: + response = get_image_with_retries(url, headers=headers) + if filetype.lower() not in response.headers['Content-Type'].lower(): + cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent) + return None + with open(fileoutput, 'wb') as file: + file.write(response.content) else: (file, headers) = urllib.request.urlretrieve(url, filename=fileoutput) if filetype.lower() not in headers['Content-Type'].lower(): @@ -102,10 +132,19 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= return None return file else: - url_data = urllib.request.urlopen(url) - data = url_data.read() - data_json = json.loads(data.decode('utf-8')) - return data_json + # some suppliers work with requests.get(), others need urllib.request.urlopen() + try: + response = requests.get(url) + data_json = response.json() + return data_json + except requests.exceptions.JSONDecodeError: + try: + url_data = urllib.request.urlopen(url) + data = url_data.read() + data_json = json.loads(data.decode('utf-8')) + return data_json + finally: + pass except (socket.timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout): cprint(f'[INFO]\tWarning: {filetype} download socket timed out ({timeout}s)', silent=silent) except (urllib.error.HTTPError, requests.exceptions.ConnectionError): @@ -136,9 +175,14 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str if not file: # Try with requests library file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=True, silent=silent, **kwargs) + + if not file: + # Try with cloudscraper + file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=False, try_cloudscraper=True, silent=silent, **kwargs) # Still nothing if not file: return False + cprint('[INFO]\tSuccess: Part image downloaded', silent=silent) return True diff --git a/kintree/config/inventree/suppliers.yaml b/kintree/config/inventree/suppliers.yaml index 231c7b5d..0dfd2773 100644 --- a/kintree/config/inventree/suppliers.yaml +++ b/kintree/config/inventree/suppliers.yaml @@ -10,6 +10,9 @@ Element14: Farnell: enable: true name: Farnell +Jameco: + enable: true + name: Jameco Newark: enable: true name: Newark diff --git a/kintree/config/jameco/jamco_config.yaml b/kintree/config/jameco/jamco_config.yaml new file mode 100644 index 00000000..906de132 --- /dev/null +++ b/kintree/config/jameco/jamco_config.yaml @@ -0,0 +1,11 @@ +SUPPLIER_INVENTREE_NAME: Jameco Electronics +SEARCH_NAME: null +SEARCH_DESCRIPTION: null +SEARCH_REVISION: null +SEARCH_KEYWORDS: null +SEARCH_SKU: null +SEARCH_MANUFACTURER: null +SEARCH_MPN: null +SEARCH_SUPPLIER_URL: null +SEARCH_DATASHEET: null +EXTRA_FIELDS: null \ No newline at end of file diff --git a/kintree/config/jameco/jameco_api.yaml b/kintree/config/jameco/jameco_api.yaml new file mode 100644 index 00000000..04e0a8bc --- /dev/null +++ b/kintree/config/jameco/jameco_api.yaml @@ -0,0 +1 @@ +JAMECO_API_URL: https://ahzbkf.a.searchspring.io/api/search/search.json?ajaxCatalog=v3&resultsFormat=native&siteId=ahzbkf&q= \ No newline at end of file diff --git a/kintree/config/settings.py b/kintree/config/settings.py index 03e746de..f7b8e8ad 100644 --- a/kintree/config/settings.py +++ b/kintree/config/settings.py @@ -178,6 +178,10 @@ def load_suppliers(): CONFIG_LCSC = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'lcsc_config.yaml')) CONFIG_LCSC_API = os.path.join(CONFIG_USER_FILES, 'lcsc_api.yaml') +# JAMECO user configuration +CONFIG_JAMECO = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'jameco_config.yaml')) +CONFIG_JAMECO_API = os.path.join(CONFIG_USER_FILES, 'jameco_api.yaml') + # TME user configuration CONFIG_TME = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'tme_config.yaml')) CONFIG_TME_API = os.path.join(CONFIG_USER_FILES, 'tme_api.yaml') diff --git a/kintree/database/inventree_api.py b/kintree/database/inventree_api.py index 2641c5f6..4ceb2671 100644 --- a/kintree/database/inventree_api.py +++ b/kintree/database/inventree_api.py @@ -512,8 +512,9 @@ def create_part(category_id: int, name: str, description: str, revision: str, ip 'component': True, 'purchaseable': True, }) - except Exception: + except Exception as e: cprint('[TREE]\tError: Part creation failed. Check if Ki-nTree settings match InvenTree part settings.', silent=settings.SILENT) + cprint(repr(e), silent=settings.SILENT) return 0 if part: diff --git a/kintree/database/inventree_interface.py b/kintree/database/inventree_interface.py index 8d97d8d2..5bca74c3 100644 --- a/kintree/database/inventree_interface.py +++ b/kintree/database/inventree_interface.py @@ -5,7 +5,7 @@ from ..common.tools import cprint from ..config import config_interface from ..database import inventree_api -from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, tme_api +from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, jameco_api, tme_api category_separator = '/' @@ -401,6 +401,8 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None) user_search_key = settings.CONFIG_ELEMENT14.get(user_key, None) elif supplier == 'LCSC': user_search_key = settings.CONFIG_LCSC.get(user_key, None) + elif supplier == 'Jameco': + user_search_key = settings.CONFIG_JAMECO.get(user_key, None) elif supplier == 'TME': user_search_key = settings.CONFIG_TME.get(user_key, None) else: @@ -425,6 +427,8 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None) default_search_keys = element14_api.get_default_search_keys() elif supplier == 'LCSC': default_search_keys = lcsc_api.get_default_search_keys() + elif supplier == 'Jameco': + default_search_keys = jameco_api.get_default_search_keys() elif supplier == 'TME': default_search_keys = tme_api.get_default_search_keys() else: @@ -478,6 +482,8 @@ def supplier_search(supplier: str, part_number: str, test_mode=False) -> dict: part_info = element14_api.fetch_part_info(part_number, supplier) elif supplier == 'LCSC': part_info = lcsc_api.fetch_part_info(part_number) + elif supplier == 'Jameco': + part_info = jameco_api.fetch_part_info(part_number) elif supplier == 'TME': part_info = tme_api.fetch_part_info(part_number) diff --git a/kintree/gui/views/settings.py b/kintree/gui/views/settings.py index da2a8438..4e261089 100644 --- a/kintree/gui/views/settings.py +++ b/kintree/gui/views/settings.py @@ -81,6 +81,13 @@ ft.TextField(), None, ] + elif supplier == 'Jameco': + jameco_api_settings = config_interface.load_file(global_settings.CONFIG_JAMECO_API) + supplier_settings[supplier]['API URL'] = [ + jameco_api_settings['JAMECO_API_URL'], + ft.TextField(), + None, + ] elif supplier == 'TME': tme_api_settings = config_interface.load_file(global_settings.CONFIG_TME_API) supplier_settings[supplier]['API Token'] = [ @@ -672,6 +679,15 @@ def save_s(self, e: ft.ControlEvent, supplier: str, show_dialog=True): } lcsc_settings = {**settings_from_file, **updated_settings} config_interface.dump_file(lcsc_settings, global_settings.CONFIG_LCSC_API) + elif supplier == 'Jameco': + # Load settings from file + settings_from_file = config_interface.load_file(global_settings.CONFIG_JAMECO_API) + # Update settings values + updated_settings = { + 'JAMECO_API_URL': SETTINGS[self.title][supplier]['API URL'][1].value, + } + jameco_settings = {**settings_from_file, **updated_settings} + config_interface.dump_file(jameco_settings, global_settings.CONFIG_JAMECO_API) elif supplier == 'TME': # Load settings from file settings_from_file = config_interface.load_file(global_settings.CONFIG_TME_API) @@ -708,6 +724,9 @@ def test_s(self, e: ft.ControlEvent, supplier: str): elif supplier == 'LCSC': from ...search import lcsc_api result = lcsc_api.test_api() + elif supplier == 'Jameco': + from ...search import jameco_api + result = jameco_api.test_api() elif supplier == 'TME': from ...search import tme_api result = tme_api.test_api() diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py new file mode 100644 index 00000000..c1431624 --- /dev/null +++ b/kintree/search/jameco_api.py @@ -0,0 +1,172 @@ +import html +import re +from ..common.tools import download + +SEARCH_HEADERS = [ + 'title', + 'name', + 'prod_id', + 'ss_attr_manufacturer', + 'manufacturer_part_number', + 'url', + 'imageUrl', + 'related_prod_id', + 'category', +] + +# Not really a map for Jameco. +# Parameters are listed at same level as the search keys, not in separate list +PARAMETERS_KEYS = [ + 'product_type_unigram', + 'ss_attr_voltage_rating', + 'ss_attr_multiple_order_quantity', +] + + +def get_default_search_keys(): + # order matters, linked with part_form[] order in inventree_interface.translate_supplier_to_form() + return [ + 'title', + 'name', + 'revision', + 'keywords', + 'prod_id', + 'ss_attr_manufacturer', + 'manufacturer_part_number', + 'url', + 'datasheet', + 'imageUrl', + ] + + +def find_categories(part_details: str): + ''' Find categories ''' + try: + return part_details['parentCatalogName'], part_details['catalogName'] + except: + return None, None + + +def fetch_part_info(part_number: str) -> dict: + ''' Fetch part data from API ''' + + # Load Jameco settings + from ..config import settings, config_interface + jameco_api_settings = config_interface.load_file(settings.CONFIG_JAMECO_API) + + part_info = {} + + def search_timeout(timeout=10): + url = jameco_api_settings.get('JAMECO_API_URL', '') + part_number + response = download(url, timeout=timeout) + return response + + # Query part number + try: + part = search_timeout() + # Extract results, select first in returned search List + part = part.get('results', None) + part = part[0] + except: + part = None + + if not part: + return part_info + + category, subcategory = find_categories(part) + try: + part_info['category'] = category + part_info['subcategory'] = subcategory + except: + part_info['category'] = '' + part_info['subcategory'] = '' + + headers = SEARCH_HEADERS + + for key in part: + if key in headers: + if key == 'imageUrl': + try: + part_info[key] = part['imageUrl'] + except IndexError: + pass + elif key in ['title', 'name', 'category']: + # Jameco title/name is often >100 chars, which causes an error later. Check for it here. + if (len(part[key]) > 100): + trimmed_value = str(part[key])[:100] + part_info[key] = html.unescape(trimmed_value) # Json data sometimes has HTML encoded chars, e.g. " + else: + part_info[key] = html.unescape(part[key]) + else: + part_info[key] = part[key] + + # Parameters + part_info['parameters'] = {} + + for i, parameter_key in enumerate(PARAMETERS_KEYS): + if part.get(parameter_key, ''): + parameter_name = parameter_key + parameter_value = part[parameter_key] + if isinstance(parameter_value, list): + parameter_string = ', '.join(parameter_value) + part_info['parameters'][parameter_name] = parameter_string + else: + # Append to parameters dictionary + part_info['parameters'][parameter_name] = parameter_value + + # Pricing + part_info['pricing'] = {} + + # Jameco returns price breaks as a string of HTML text + # Convert pricing string pattern to List, then dictionary for Ki-nTree + price_break_str = part['secondary_prices'] + price_break_str = price_break_str.replace(',', '') # remove comma + price_break_str = re.sub('(\<br\s\/&*gt)', '', price_break_str) # remove HTML + price_break_str = re.sub(';', ':', price_break_str) # remove ; char + price_break_str = re.sub('(\:\s+\$)|\;', ':', price_break_str) # remove $ char + price_break_list = price_break_str.split(':') # split on : + price_break_list.pop() # remove last empty element in List + + for i in range(0, len(price_break_list), 2): + quantity = int(price_break_list[i]) + price = float(price_break_list[i + 1]) + part_info['pricing'][quantity] = price + + part_info['currency'] = 'USD' + + # Extra search fields + if settings.CONFIG_JAMECO.get('EXTRA_FIELDS', None): + for extra_field in settings.CONFIG_JAMECO['EXTRA_FIELDS']: + if part.get(extra_field, None): + part_info['parameters'][extra_field] = part[extra_field] + else: + from ..common.tools import cprint + cprint(f'[INFO]\tWarning: Extra field "{extra_field}" not found in search results', silent=False) + + return part_info + + +def test_api() -> bool: + ''' Test method for API ''' + + test_success = True + + expected = { + 'manufacturer_part_number': 'PN2222ABU', + 'name': 'Transistor PN2222A NPN Silicon General Purpose TO-92', + 'prod_id': '178511', + } + + test_part = fetch_part_info('178511') + if not test_part: + test_success = False + + # Check content of response + if test_success: + for key, value in expected.items(): + if test_part[key] != value: + print(f'"{test_part[key]}" <> "{value}"') + test_success = False + break + + return test_success diff --git a/kintree/search/lcsc_api.py b/kintree/search/lcsc_api.py index f0e06bd5..406906c4 100644 --- a/kintree/search/lcsc_api.py +++ b/kintree/search/lcsc_api.py @@ -65,11 +65,11 @@ def search_timeout(timeout=10): except: part = None - # Extract result - part = part.get('result', None) - if not part: return part_info + + # Extract result + part = part.get('result', None) category, subcategory = find_categories(part) try: