From e69dc5312ea2d61a2f4c10ea04fa7515defb2d1a Mon Sep 17 00:00:00 2001 From: ccw Date: Thu, 20 Jun 2024 21:43:22 -0400 Subject: [PATCH 01/14] Jameco API test in GUI passes --- kintree/config/inventree/suppliers.yaml | 3 + kintree/config/jameco/jamco_config.yaml | 11 ++ kintree/config/jameco/jameco_api.yaml | 1 + kintree/config/settings.py | 4 + kintree/gui/views/settings.py | 19 +++ kintree/search/jameco_api.py | 157 ++++++++++++++++++++++++ 6 files changed, 195 insertions(+) create mode 100644 kintree/config/jameco/jamco_config.yaml create mode 100644 kintree/config/jameco/jameco_api.yaml create mode 100644 kintree/search/jameco_api.py diff --git a/kintree/config/inventree/suppliers.yaml b/kintree/config/inventree/suppliers.yaml index 231c7b5d..0dfd2773 100644 --- a/kintree/config/inventree/suppliers.yaml +++ b/kintree/config/inventree/suppliers.yaml @@ -10,6 +10,9 @@ Element14: Farnell: enable: true name: Farnell +Jameco: + enable: true + name: Jameco Newark: enable: true name: Newark diff --git a/kintree/config/jameco/jamco_config.yaml b/kintree/config/jameco/jamco_config.yaml new file mode 100644 index 00000000..906de132 --- /dev/null +++ b/kintree/config/jameco/jamco_config.yaml @@ -0,0 +1,11 @@ +SUPPLIER_INVENTREE_NAME: Jameco Electronics +SEARCH_NAME: null +SEARCH_DESCRIPTION: null +SEARCH_REVISION: null +SEARCH_KEYWORDS: null +SEARCH_SKU: null +SEARCH_MANUFACTURER: null +SEARCH_MPN: null +SEARCH_SUPPLIER_URL: null +SEARCH_DATASHEET: null +EXTRA_FIELDS: null \ No newline at end of file diff --git a/kintree/config/jameco/jameco_api.yaml b/kintree/config/jameco/jameco_api.yaml new file mode 100644 index 00000000..04e0a8bc --- /dev/null +++ b/kintree/config/jameco/jameco_api.yaml @@ -0,0 +1 @@ +JAMECO_API_URL: https://ahzbkf.a.searchspring.io/api/search/search.json?ajaxCatalog=v3&resultsFormat=native&siteId=ahzbkf&q= \ No newline at end of file diff --git a/kintree/config/settings.py b/kintree/config/settings.py index 03e746de..f7b8e8ad 100644 --- a/kintree/config/settings.py +++ b/kintree/config/settings.py @@ -178,6 +178,10 @@ def load_suppliers(): CONFIG_LCSC = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'lcsc_config.yaml')) CONFIG_LCSC_API = os.path.join(CONFIG_USER_FILES, 'lcsc_api.yaml') +# JAMECO user configuration +CONFIG_JAMECO = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'jameco_config.yaml')) +CONFIG_JAMECO_API = os.path.join(CONFIG_USER_FILES, 'jameco_api.yaml') + # TME user configuration CONFIG_TME = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'tme_config.yaml')) CONFIG_TME_API = os.path.join(CONFIG_USER_FILES, 'tme_api.yaml') diff --git a/kintree/gui/views/settings.py b/kintree/gui/views/settings.py index da2a8438..4e261089 100644 --- a/kintree/gui/views/settings.py +++ b/kintree/gui/views/settings.py @@ -81,6 +81,13 @@ ft.TextField(), None, ] + elif supplier == 'Jameco': + jameco_api_settings = config_interface.load_file(global_settings.CONFIG_JAMECO_API) + supplier_settings[supplier]['API URL'] = [ + jameco_api_settings['JAMECO_API_URL'], + ft.TextField(), + None, + ] elif supplier == 'TME': tme_api_settings = config_interface.load_file(global_settings.CONFIG_TME_API) supplier_settings[supplier]['API Token'] = [ @@ -672,6 +679,15 @@ def save_s(self, e: ft.ControlEvent, supplier: str, show_dialog=True): } lcsc_settings = {**settings_from_file, **updated_settings} config_interface.dump_file(lcsc_settings, global_settings.CONFIG_LCSC_API) + elif supplier == 'Jameco': + # Load settings from file + settings_from_file = config_interface.load_file(global_settings.CONFIG_JAMECO_API) + # Update settings values + updated_settings = { + 'JAMECO_API_URL': SETTINGS[self.title][supplier]['API URL'][1].value, + } + jameco_settings = {**settings_from_file, **updated_settings} + config_interface.dump_file(jameco_settings, global_settings.CONFIG_JAMECO_API) elif supplier == 'TME': # Load settings from file settings_from_file = config_interface.load_file(global_settings.CONFIG_TME_API) @@ -708,6 +724,9 @@ def test_s(self, e: ft.ControlEvent, supplier: str): elif supplier == 'LCSC': from ...search import lcsc_api result = lcsc_api.test_api() + elif supplier == 'Jameco': + from ...search import jameco_api + result = jameco_api.test_api() elif supplier == 'TME': from ...search import tme_api result = tme_api.test_api() diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py new file mode 100644 index 00000000..bf32b2ca --- /dev/null +++ b/kintree/search/jameco_api.py @@ -0,0 +1,157 @@ +import re +from ..common.tools import download + +SEARCH_HEADERS = [ + 'name', + 'prod_id', + 'brandNameEn', + 'productModel', + 'pdfUrl', + 'imageUrl', + 'price', + 'manufacturer_part_number', + 'url', + 'ss_attr_manufacturer', + +] +PARAMETERS_MAP = [ + 'product_type_unigram', #e.g. to92 +] + + +def get_default_search_keys(): + return [ + 'name', + 'prod_id', + 'brandNameEn', + 'productModel', + 'pdfUrl', + 'imageUrl', + 'price', + 'manufacturer_part_number', + 'url', + 'ss_attr_manufacturer', + ] + + +def find_categories(part_details: str): + ''' Find categories ''' + try: + return part_details['parentCatalogName'], part_details['catalogName'] + except: + return None, None + + +def fetch_part_info(part_number: str) -> dict: + ''' Fetch part data from API ''' + + # Load Jameco settings + from ..config import settings, config_interface + jameco_api_settings = config_interface.load_file(settings.CONFIG_JAMECO_API) + + part_info = {} + + def search_timeout(timeout=10): + url = jameco_api_settings.get('JAMECO_API_URL', '') + part_number + response = download(url, timeout=timeout) + return response + + # Query part number + try: + part = search_timeout() + except: + part = None + + # Extract results, select first in returned search List + part = part.get('results', None) + part = part[0] + + if not part: + return part_info + + category, subcategory = find_categories(part) + try: + part_info['category'] = category + part_info['subcategory'] = subcategory + except: + part_info['category'] = '' + part_info['subcategory'] = '' + + headers = SEARCH_HEADERS + + for key in part: + if key in headers: + if key == 'imageUrl': + try: + part_info[key] = part['imageUrl'][0] + except IndexError: + pass + else: + part_info[key] = part[key] + + # Parameters + part_info['parameters'] = {} + [parameter_key] = PARAMETERS_MAP + + if part.get(parameter_key, ''): + for parameter in range(len(part[parameter_key])): + parameter_name = parameter_key + parameter_value = str(part[parameter_key]).upper() + # Append to parameters dictionary + part_info['parameters'][parameter_name] = parameter_value + + # Pricing + part_info['pricing'] = {} + + # Jameco returns price breaks as a string of HTML text + # Convert pricing string pattern to List, then dictionary for Ki-nTree + price_break_str = part['secondary_prices'] + price_break_str = re.sub('(\<br\s\/&*gt)','', price_break_str) + price_break_str = re.sub(';', ':', price_break_str) + price_break_str = re.sub('(\:\s+\$)|\;', ':', price_break_str) + price_break_list = price_break_str.split(':') + price_break_list.pop() # remove last empty element in List + + for i in range(0, len(price_break_list), 2): + quantity = price_break_list[i] + price = price_break_list[i+1] + part_info['pricing'][quantity] = price + + part_info['currency'] = 'USD' + + # Extra search fields + if settings.CONFIG_JAMECO.get('EXTRA_FIELDS', None): + for extra_field in settings.CONFIG_JAMECO['EXTRA_FIELDS']: + if part.get(extra_field, None): + part_info['parameters'][extra_field] = part[extra_field] + else: + from ..common.tools import cprint + cprint(f'[INFO]\tWarning: Extra field "{extra_field}" not found in search results', silent=False) + + return part_info + + +def test_api() -> bool: + ''' Test method for API ''' + + test_success = True + + expected = { + 'manufacturer_part_number':'PN2222ABU', + 'name': 'Transistor PN2222A NPN Silicon General Purpose TO-92', + 'prod_id': '178511', + } + + test_part = fetch_part_info('178511') + if not test_part: + test_success = False + + # Check content of response + if test_success: + for key, value in expected.items(): + if test_part[key] != value: + print(f'"{test_part[key]}" <> "{value}"') + test_success = False + break + + return test_success From a31f73be54d163855af53ea2c56a23557ccbc6fa Mon Sep 17 00:00:00 2001 From: ccw Date: Thu, 20 Jun 2024 23:46:23 -0400 Subject: [PATCH 02/14] Part fields and parameters populate correctly. Part not inserting to Inventree, Param map for Electronics does not exist warning. Part creation failed error, check Ki-nTree settings match Inventree part settings error msg. --- kintree/database/inventree_interface.py | 8 +++- kintree/search/jameco_api.py | 60 +++++++++++++++---------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/kintree/database/inventree_interface.py b/kintree/database/inventree_interface.py index 418555e0..5f604059 100644 --- a/kintree/database/inventree_interface.py +++ b/kintree/database/inventree_interface.py @@ -5,7 +5,7 @@ from ..common.tools import cprint from ..config import config_interface from ..database import inventree_api -from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, tme_api +from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, jameco_api, tme_api category_separator = '/' @@ -401,6 +401,8 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None) user_search_key = settings.CONFIG_ELEMENT14.get(user_key, None) elif supplier == 'LCSC': user_search_key = settings.CONFIG_LCSC.get(user_key, None) + elif supplier == 'Jameco': + user_search_key = settings.CONFIG_JAMECO.get(user_key, None) elif supplier == 'TME': user_search_key = settings.CONFIG_TME.get(user_key, None) else: @@ -425,6 +427,8 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None) default_search_keys = element14_api.get_default_search_keys() elif supplier == 'LCSC': default_search_keys = lcsc_api.get_default_search_keys() + elif supplier == 'Jameco': + default_search_keys = jameco_api.get_default_search_keys() elif supplier == 'TME': default_search_keys = tme_api.get_default_search_keys() else: @@ -478,6 +482,8 @@ def supplier_search(supplier: str, part_number: str, test_mode=False) -> dict: part_info = element14_api.fetch_part_info(part_number, supplier) elif supplier == 'LCSC': part_info = lcsc_api.fetch_part_info(part_number) + elif supplier == 'Jameco': + part_info = jameco_api.fetch_part_info(part_number) elif supplier == 'TME': part_info = tme_api.fetch_part_info(part_number) diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index bf32b2ca..ec17dcbc 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -2,38 +2,41 @@ from ..common.tools import download SEARCH_HEADERS = [ + 'title', 'name', 'prod_id', - 'brandNameEn', - 'productModel', - 'pdfUrl', - 'imageUrl', - 'price', + 'ss_attr_manufacturer', 'manufacturer_part_number', 'url', - 'ss_attr_manufacturer', - + 'imageUrl', + 'related_prod_id', + 'category', ] -PARAMETERS_MAP = [ - 'product_type_unigram', #e.g. to92 + +# Not really a map for Jameco. +# Parameters are listed at same level as the search keys, not in separate list +PARAMETERS_KEYS = [ + 'product_type_unigram', + 'ss_attr_voltage_rating', + 'ss_attr_multiple_order_quantity', ] def get_default_search_keys(): + # order matters, linked with part_form[] order in inventree_interface.translate_supplier_to_form() return [ + 'title', 'name', + 'revision', + 'keywords', 'prod_id', - 'brandNameEn', - 'productModel', - 'pdfUrl', - 'imageUrl', - 'price', + 'ss_attr_manufacturer', 'manufacturer_part_number', 'url', - 'ss_attr_manufacturer', + 'datasheet', + 'imageUrl', ] - def find_categories(part_details: str): ''' Find categories ''' try: @@ -83,22 +86,33 @@ def search_timeout(timeout=10): if key in headers: if key == 'imageUrl': try: - part_info[key] = part['imageUrl'][0] + part_info[key] = part['imageUrl'] except IndexError: pass + elif key in ['title','name']: + # Jameco title/name is often >100 chars, which causes an error later. Check for it here. + if (len(part[key]) > 100): + trimmed_value = str(part[key])[:100] + part_info[key] = trimmed_value + else: + part_info[key] = part[key] else: part_info[key] = part[key] # Parameters part_info['parameters'] = {} - [parameter_key] = PARAMETERS_MAP - if part.get(parameter_key, ''): - for parameter in range(len(part[parameter_key])): + + for i, parameter_key in enumerate(PARAMETERS_KEYS): + if part.get(parameter_key, ''): parameter_name = parameter_key - parameter_value = str(part[parameter_key]).upper() - # Append to parameters dictionary - part_info['parameters'][parameter_name] = parameter_value + parameter_value = part[parameter_key] + if isinstance(parameter_value, list): + parameter_string = ', '.join(parameter_value) + part_info['parameters'][parameter_name] = parameter_string + else: + # Append to parameters dictionary + part_info['parameters'][parameter_name] = parameter_value # Pricing part_info['pricing'] = {} From 00367603a607be171287541284e208d0a6c2ec5b Mon Sep 17 00:00:00 2001 From: ccw Date: Fri, 21 Jun 2024 20:08:32 -0400 Subject: [PATCH 03/14] Correct style --- kintree/search/jameco_api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index ec17dcbc..9049f229 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -37,6 +37,7 @@ def get_default_search_keys(): 'imageUrl', ] + def find_categories(part_details: str): ''' Find categories ''' try: @@ -89,7 +90,7 @@ def search_timeout(timeout=10): part_info[key] = part['imageUrl'] except IndexError: pass - elif key in ['title','name']: + elif key in ['title', 'name']: # Jameco title/name is often >100 chars, which causes an error later. Check for it here. if (len(part[key]) > 100): trimmed_value = str(part[key])[:100] @@ -102,7 +103,6 @@ def search_timeout(timeout=10): # Parameters part_info['parameters'] = {} - for i, parameter_key in enumerate(PARAMETERS_KEYS): if part.get(parameter_key, ''): parameter_name = parameter_key @@ -120,15 +120,15 @@ def search_timeout(timeout=10): # Jameco returns price breaks as a string of HTML text # Convert pricing string pattern to List, then dictionary for Ki-nTree price_break_str = part['secondary_prices'] - price_break_str = re.sub('(\<br\s\/&*gt)','', price_break_str) + price_break_str = re.sub('(\<br\s\/&*gt)', '', price_break_str) price_break_str = re.sub(';', ':', price_break_str) price_break_str = re.sub('(\:\s+\$)|\;', ':', price_break_str) price_break_list = price_break_str.split(':') - price_break_list.pop() # remove last empty element in List + price_break_list.pop() # remove last empty element in List for i in range(0, len(price_break_list), 2): quantity = price_break_list[i] - price = price_break_list[i+1] + price = price_break_list[i + 1] part_info['pricing'][quantity] = price part_info['currency'] = 'USD' @@ -151,7 +151,7 @@ def test_api() -> bool: test_success = True expected = { - 'manufacturer_part_number':'PN2222ABU', + 'manufacturer_part_number': 'PN2222ABU', 'name': 'Transistor PN2222A NPN Silicon General Purpose TO-92', 'prod_id': '178511', } From 7bf34ab045f23a1ea4d3c0514482ccffc9ac85e0 Mon Sep 17 00:00:00 2001 From: ccw Date: Fri, 21 Jun 2024 20:50:57 -0400 Subject: [PATCH 04/14] Unescape Jemeco JSON data for name, title, category --- kintree/search/jameco_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index 9049f229..997fa7db 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -1,3 +1,4 @@ +import html import re from ..common.tools import download @@ -84,19 +85,19 @@ def search_timeout(timeout=10): headers = SEARCH_HEADERS for key in part: - if key in headers: + if key in headers: if key == 'imageUrl': try: part_info[key] = part['imageUrl'] except IndexError: pass - elif key in ['title', 'name']: + elif key in ['title', 'name', 'category']: # Jameco title/name is often >100 chars, which causes an error later. Check for it here. if (len(part[key]) > 100): trimmed_value = str(part[key])[:100] - part_info[key] = trimmed_value + part_info[key] = html.unescape(trimmed_value) # Json data sometimes has HTML encoded chars, e.g. " else: - part_info[key] = part[key] + part_info[key] = html.unescape(part[key]) else: part_info[key] = part[key] From 116a573aca2bc05b146f3d5a2e4a0ec19677bc3b Mon Sep 17 00:00:00 2001 From: ccw Date: Fri, 21 Jun 2024 23:34:40 -0400 Subject: [PATCH 05/14] Fix no-image from Jameco image link with cloudscraper library --- kintree/common/tools.py | 42 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/kintree/common/tools.py b/kintree/common/tools.py index bfa8f24b..fda6b082 100644 --- a/kintree/common/tools.py +++ b/kintree/common/tools.py @@ -60,7 +60,26 @@ def create_library(library_path: str, symbol: str, template_lib: str): copyfile(template_lib, new_kicad_sym_file) -def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, silent=False): +def get_image_with_retries(url, headers, retries=3, wait=5, silent=False): + """ Method to download image with cloudscraper library and retry attempts""" + import cloudscraper + import time + scraper = cloudscraper.create_scraper() + for attempt in range(retries): + try: + response = scraper.get(url, headers=headers) + if response.status_code == 200: + return response + else: + cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} failed with status code {response.status_code}. Retrying in {wait} seconds...', silent=silent) + except Exception as e: + cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} encountered an error: {e}. Retrying in {wait} seconds...', silent=silent) + time.sleep(wait) + cprint(f'[INFO]\tWarning: All Image download attempts failed. Could not retrieve the image.', silent=silent) + return None + + +def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, try_cloudscraper=False, silent=False): ''' Standard method to download URL content, with option to save to local file (eg. images) ''' import socket @@ -68,9 +87,12 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= import requests headers = { - 'User-Agent': 'Mozilla/5.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', + 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36', + 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', + 'Accept-Encoding':'Accept-Encoding: gzip, deflate, br', + 'Accept-Language': 'en-US,en;q=0.9', + 'Connection': 'keep-alive', + 'Cache-Control': 'no-cache', } # Set default timeout for download socket @@ -89,6 +111,13 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= return None with open(fileoutput, 'wb') as file: file.write(response.content) + elif try_cloudscraper: + response = get_image_with_retries(url, headers=headers) + if filetype.lower() not in response.headers['Content-Type'].lower(): + cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent) + return None + with open(fileoutput, 'wb') as file: + file.write(response.content) else: (file, headers) = urllib.request.urlretrieve(url, filename=fileoutput) if filetype.lower() not in headers['Content-Type'].lower(): @@ -130,9 +159,14 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str if not file: # Try with requests library file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=True, silent=silent, **kwargs) + + if not file: + # Try with cloudscraper + file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=False, try_cloudscraper=True, silent=silent, **kwargs) # Still nothing if not file: return False + cprint(f'[INFO]\tSuccess: Part image downloaded', silent=silent) return True From 1b17ba760c5ec2157fe437a3daac2f45308ea0a2 Mon Sep 17 00:00:00 2001 From: ccw Date: Sat, 22 Jun 2024 00:29:48 -0400 Subject: [PATCH 06/14] Fix pricing crash due to incorrect data type --- kintree/search/jameco_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index 997fa7db..c7d8e981 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -128,8 +128,8 @@ def search_timeout(timeout=10): price_break_list.pop() # remove last empty element in List for i in range(0, len(price_break_list), 2): - quantity = price_break_list[i] - price = price_break_list[i + 1] + quantity = int(price_break_list[i]) + price = float(price_break_list[i + 1]) part_info['pricing'][quantity] = price part_info['currency'] = 'USD' From da4e160ebafb52bb43eb14a9933264cccbd07379 Mon Sep 17 00:00:00 2001 From: ccw Date: Sat, 22 Jun 2024 12:24:33 -0400 Subject: [PATCH 07/14] Clean up style --- kintree/common/tools.py | 8 ++++---- kintree/search/jameco_api.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kintree/common/tools.py b/kintree/common/tools.py index fda6b082..b101922c 100644 --- a/kintree/common/tools.py +++ b/kintree/common/tools.py @@ -75,7 +75,7 @@ def get_image_with_retries(url, headers, retries=3, wait=5, silent=False): except Exception as e: cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} encountered an error: {e}. Retrying in {wait} seconds...', silent=silent) time.sleep(wait) - cprint(f'[INFO]\tWarning: All Image download attempts failed. Could not retrieve the image.', silent=silent) + cprint('[INFO]\tWarning: All Image download attempts failed. Could not retrieve the image.', silent=silent) return None @@ -87,9 +87,9 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= import requests headers = { - 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36', 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', - 'Accept-Encoding':'Accept-Encoding: gzip, deflate, br', + 'Accept-Encoding': 'Accept-Encoding: gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9', 'Connection': 'keep-alive', 'Cache-Control': 'no-cache', @@ -168,5 +168,5 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str if not file: return False - cprint(f'[INFO]\tSuccess: Part image downloaded', silent=silent) + cprint('[INFO]\tSuccess: Part image downloaded', silent=silent) return True diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index c7d8e981..ce84098c 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -85,7 +85,7 @@ def search_timeout(timeout=10): headers = SEARCH_HEADERS for key in part: - if key in headers: + if key in headers: if key == 'imageUrl': try: part_info[key] = part['imageUrl'] From 41803461dae200d11a8532cb744a15f03aad5e6d Mon Sep 17 00:00:00 2001 From: ccw Date: Sat, 22 Jun 2024 13:05:12 -0400 Subject: [PATCH 08/14] Fix request header to accept expected JSON data type --- kintree/common/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kintree/common/tools.py b/kintree/common/tools.py index b101922c..72c6c25b 100644 --- a/kintree/common/tools.py +++ b/kintree/common/tools.py @@ -88,7 +88,7 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36', - 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', + 'Accept': 'applicaiton/json,image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Encoding': 'Accept-Encoding: gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9', 'Connection': 'keep-alive', From 70e6d3ff913f29d99c685e08fbfbbcf389827dd1 Mon Sep 17 00:00:00 2001 From: ccw Date: Sat, 22 Jun 2024 13:05:40 -0400 Subject: [PATCH 09/14] Fix crash if searched part number is not found --- kintree/search/jameco_api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index ce84098c..4d7537ff 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -64,13 +64,12 @@ def search_timeout(timeout=10): # Query part number try: part = search_timeout() + # Extract results, select first in returned search List + part = part.get('results', None) + part = part[0] except: part = None - # Extract results, select first in returned search List - part = part.get('results', None) - part = part[0] - if not part: return part_info From 72a68db1d8db39f897921fd56fc364e7f93e7d57 Mon Sep 17 00:00:00 2001 From: ccw Date: Tue, 25 Jun 2024 07:33:03 -0400 Subject: [PATCH 10/14] Fix crash when LCSC API returns None --- kintree/search/lcsc_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kintree/search/lcsc_api.py b/kintree/search/lcsc_api.py index f0e06bd5..406906c4 100644 --- a/kintree/search/lcsc_api.py +++ b/kintree/search/lcsc_api.py @@ -65,11 +65,11 @@ def search_timeout(timeout=10): except: part = None - # Extract result - part = part.get('result', None) - if not part: return part_info + + # Extract result + part = part.get('result', None) category, subcategory = find_categories(part) try: From b7c0a17bdaa87c66294b2662b2306b4380f25df2 Mon Sep 17 00:00:00 2001 From: ccw Date: Tue, 25 Jun 2024 07:33:54 -0400 Subject: [PATCH 11/14] Fix crash on parsing quantity containing comma to int() --- kintree/search/jameco_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kintree/search/jameco_api.py b/kintree/search/jameco_api.py index 4d7537ff..c1431624 100644 --- a/kintree/search/jameco_api.py +++ b/kintree/search/jameco_api.py @@ -120,10 +120,11 @@ def search_timeout(timeout=10): # Jameco returns price breaks as a string of HTML text # Convert pricing string pattern to List, then dictionary for Ki-nTree price_break_str = part['secondary_prices'] - price_break_str = re.sub('(\<br\s\/&*gt)', '', price_break_str) - price_break_str = re.sub(';', ':', price_break_str) - price_break_str = re.sub('(\:\s+\$)|\;', ':', price_break_str) - price_break_list = price_break_str.split(':') + price_break_str = price_break_str.replace(',', '') # remove comma + price_break_str = re.sub('(\<br\s\/&*gt)', '', price_break_str) # remove HTML + price_break_str = re.sub(';', ':', price_break_str) # remove ; char + price_break_str = re.sub('(\:\s+\$)|\;', ':', price_break_str) # remove $ char + price_break_list = price_break_str.split(':') # split on : price_break_list.pop() # remove last empty element in List for i in range(0, len(price_break_list), 2): From 7693a0cc0d41f8137d150acda76df58b92d78699 Mon Sep 17 00:00:00 2001 From: ccw Date: Tue, 25 Jun 2024 07:35:30 -0400 Subject: [PATCH 12/14] Add requests.get() try when attempting to download data (some suppliers work with requests.get() and not urllib.request.urlopen() --- kintree/common/tools.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/kintree/common/tools.py b/kintree/common/tools.py index 72c6c25b..6080fbbc 100644 --- a/kintree/common/tools.py +++ b/kintree/common/tools.py @@ -86,6 +86,7 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= import urllib.request import requests + # A more detailed headers was needed for request to Jameco headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36', 'Accept': 'applicaiton/json,image/webp,image/apng,image/*,*/*;q=0.8', @@ -125,10 +126,19 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers= return None return file else: - url_data = urllib.request.urlopen(url) - data = url_data.read() - data_json = json.loads(data.decode('utf-8')) - return data_json + # some suppliers work with requests.get(), others need urllib.request.urlopen() + try: + response = requests.get(url) + data_json = response.json() + return data_json + except requests.exceptions.JSONDecodeError: + try: + url_data = urllib.request.urlopen(url) + data = url_data.read() + data_json = json.loads(data.decode('utf-8')) + return data_json + finally: + pass except (socket.timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout): cprint(f'[INFO]\tWarning: {filetype} download socket timed out ({timeout}s)', silent=silent) except (urllib.error.HTTPError, requests.exceptions.ConnectionError): From 663a9b3a5c0c3aa97a226681aa3ad2fddb00ba01 Mon Sep 17 00:00:00 2001 From: ccw Date: Fri, 28 Jun 2024 20:57:25 -0400 Subject: [PATCH 13/14] Add Exception error message when create_part fails --- kintree/database/inventree_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kintree/database/inventree_api.py b/kintree/database/inventree_api.py index 35eab299..58650d02 100644 --- a/kintree/database/inventree_api.py +++ b/kintree/database/inventree_api.py @@ -510,8 +510,9 @@ def create_part(category_id: int, name: str, description: str, revision: str, ip 'component': True, 'purchaseable': True, }) - except Exception: + except Exception as e: cprint('[TREE]\tError: Part creation failed. Check if Ki-nTree settings match InvenTree part settings.', silent=settings.SILENT) + cprint(repr(e), silent=settings.SILENT) return 0 if part: From 53810e1401bae51e8644ff191dbad0001836ac70 Mon Sep 17 00:00:00 2001 From: ccw Date: Fri, 28 Jun 2024 20:57:48 -0400 Subject: [PATCH 14/14] Update .gitignore with .vscode settings --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1c448974..04158a2d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ build/ kintree/tests/* .coverage htmlcov/ +.vscode/launch.json