Add supplier jameco electronics

.gitignore

@@ -9,3 +9,4 @@ build/
 kintree/tests/*
 .coverage
 htmlcov/
+.vscode/launch.json

kintree/common/tools.py

@@ -60,17 +60,40 @@ def create_library(library_path: str, symbol: str, template_lib: str):
         copyfile(template_lib, new_kicad_sym_file)
 
 
-def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, silent=False):
+def get_image_with_retries(url, headers, retries=3, wait=5, silent=False):
+    """ Method to download image with cloudscraper library and retry attempts"""
+    import cloudscraper
+    import time
+    scraper = cloudscraper.create_scraper()
+    for attempt in range(retries):
+        try:
+            response = scraper.get(url, headers=headers)
+            if response.status_code == 200:
+                return response
+            else:
+                cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} failed with status code {response.status_code}. Retrying in {wait} seconds...', silent=silent)
+        except Exception as e:
+            cprint(f'[INFO]\tWarning: Image download Attempt {attempt + 1} encountered an error: {e}. Retrying in {wait} seconds...', silent=silent)
+        time.sleep(wait)
+    cprint('[INFO]\tWarning: All Image download attempts failed. Could not retrieve the image.', silent=silent)
+    return None
+
+
+def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=False, requests_lib=False, try_cloudscraper=False, silent=False):
     ''' Standard method to download URL content, with option to save to local file (eg. images) '''
 
     import socket
     import urllib.request
     import requests
 
+    # A more detailed headers was needed for request to Jameco
     headers = {
-        'User-Agent': 'Mozilla/5.0',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.5',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36',
+        'Accept': 'applicaiton/json,image/webp,image/apng,image/*,*/*;q=0.8',
+        'Accept-Encoding': 'Accept-Encoding: gzip, deflate, br',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Connection': 'keep-alive',
+        'Cache-Control': 'no-cache',
     }
 
     # Set default timeout for download socket
@@ -95,17 +118,33 @@ def download(url, filetype='API data', fileoutput='', timeout=3, enable_headers=
                     return None
                 with open(fileoutput, 'wb') as file:
                     file.write(response.content)
+            elif try_cloudscraper:
+                response = get_image_with_retries(url, headers=headers)
+                if filetype.lower() not in response.headers['Content-Type'].lower():
+                    cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent)
+                    return None
+                with open(fileoutput, 'wb') as file:
+                    file.write(response.content)
             else:
                 (file, headers) = urllib.request.urlretrieve(url, filename=fileoutput)
                 if filetype.lower() not in headers['Content-Type'].lower():
                     cprint(f'[INFO]\tWarning: {filetype} download returned the wrong file type', silent=silent)
                     return None
             return file
         else:
-            url_data = urllib.request.urlopen(url)
-            data = url_data.read()
-            data_json = json.loads(data.decode('utf-8'))
-            return data_json
+            # some suppliers work with requests.get(), others need urllib.request.urlopen()
+            try:
+                response = requests.get(url)
+                data_json = response.json()
+                return data_json
+            except requests.exceptions.JSONDecodeError:
+                try:
+                    url_data = urllib.request.urlopen(url)
+                    data = url_data.read()
+                    data_json = json.loads(data.decode('utf-8'))
+                    return data_json
+                finally:
+                    pass
     except (socket.timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout):
         cprint(f'[INFO]\tWarning: {filetype} download socket timed out ({timeout}s)', silent=silent)
     except (urllib.error.HTTPError, requests.exceptions.ConnectionError):
@@ -136,9 +175,14 @@ def download_with_retry(url: str, full_path: str, silent=False, **kwargs) -> str
     if not file:
         # Try with requests library
         file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=True, silent=silent, **kwargs)
+
+    if not file:
+        # Try with cloudscraper
+        file = download(url, fileoutput=full_path, enable_headers=True, requests_lib=False, try_cloudscraper=True, silent=silent, **kwargs)
 
     # Still nothing
     if not file:
         return False
 
+    cprint('[INFO]\tSuccess: Part image downloaded', silent=silent)
     return True

kintree/config/inventree/suppliers.yaml

@@ -10,6 +10,9 @@ Element14:
 Farnell:
   enable: true
   name: Farnell
+Jameco:
+  enable: true
+  name: Jameco
 Newark:
   enable: true
   name: Newark

kintree/config/jameco/jamco_config.yaml

@@ -0,0 +1,11 @@
+SUPPLIER_INVENTREE_NAME: Jameco Electronics
+SEARCH_NAME: null
+SEARCH_DESCRIPTION: null
+SEARCH_REVISION: null
+SEARCH_KEYWORDS: null
+SEARCH_SKU: null
+SEARCH_MANUFACTURER: null
+SEARCH_MPN: null
+SEARCH_SUPPLIER_URL: null
+SEARCH_DATASHEET: null
+EXTRA_FIELDS: null

kintree/config/jameco/jameco_api.yaml

@@ -0,0 +1 @@
+JAMECO_API_URL: https://ahzbkf.a.searchspring.io/api/search/search.json?ajaxCatalog=v3&resultsFormat=native&siteId=ahzbkf&q=

kintree/config/settings.py

@@ -178,6 +178,10 @@ def load_suppliers():
 CONFIG_LCSC = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'lcsc_config.yaml'))
 CONFIG_LCSC_API = os.path.join(CONFIG_USER_FILES, 'lcsc_api.yaml')
 
+# JAMECO user configuration
+CONFIG_JAMECO = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'jameco_config.yaml'))
+CONFIG_JAMECO_API = os.path.join(CONFIG_USER_FILES, 'jameco_api.yaml')
+
 # TME user configuration
 CONFIG_TME = config_interface.load_file(os.path.join(CONFIG_USER_FILES, 'tme_config.yaml'))
 CONFIG_TME_API = os.path.join(CONFIG_USER_FILES, 'tme_api.yaml')

kintree/database/inventree_api.py

@@ -512,8 +512,9 @@ def create_part(category_id: int, name: str, description: str, revision: str, ip
             'component': True,
             'purchaseable': True,
         })
-    except Exception:
+    except Exception as e:
         cprint('[TREE]\tError: Part creation failed. Check if Ki-nTree settings match InvenTree part settings.', silent=settings.SILENT)
+        cprint(repr(e), silent=settings.SILENT)
         return 0
 
     if part:

kintree/database/inventree_interface.py

@@ -5,7 +5,7 @@
 from ..common.tools import cprint
 from ..config import config_interface
 from ..database import inventree_api
-from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, tme_api
+from ..search import search_api, digikey_api, mouser_api, element14_api, lcsc_api, jameco_api, tme_api
 
 category_separator = '/'
 
@@ -401,6 +401,8 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None)
             user_search_key = settings.CONFIG_ELEMENT14.get(user_key, None)
         elif supplier == 'LCSC':
             user_search_key = settings.CONFIG_LCSC.get(user_key, None)
+        elif supplier == 'Jameco':
+            user_search_key = settings.CONFIG_JAMECO.get(user_key, None)
         elif supplier == 'TME':
             user_search_key = settings.CONFIG_TME.get(user_key, None)
         else:
@@ -425,6 +427,8 @@ def get_value_from_user_key(user_key: str, default_key: str, default_value=None)
         default_search_keys = element14_api.get_default_search_keys()
     elif supplier == 'LCSC':
         default_search_keys = lcsc_api.get_default_search_keys()
+    elif supplier == 'Jameco':
+        default_search_keys = jameco_api.get_default_search_keys()
     elif supplier == 'TME':
         default_search_keys = tme_api.get_default_search_keys()
     else:
@@ -478,6 +482,8 @@ def supplier_search(supplier: str, part_number: str, test_mode=False) -> dict:
             part_info = element14_api.fetch_part_info(part_number, supplier)
         elif supplier == 'LCSC':
             part_info = lcsc_api.fetch_part_info(part_number)
+        elif supplier == 'Jameco':
+            part_info = jameco_api.fetch_part_info(part_number)
         elif supplier == 'TME':
             part_info = tme_api.fetch_part_info(part_number)
 

kintree/gui/views/settings.py

@@ -81,6 +81,13 @@
             ft.TextField(),
             None,
         ]
+    elif supplier == 'Jameco':
+        jameco_api_settings = config_interface.load_file(global_settings.CONFIG_JAMECO_API)
+        supplier_settings[supplier]['API URL'] = [
+            jameco_api_settings['JAMECO_API_URL'],
+            ft.TextField(),
+            None,
+        ]
     elif supplier == 'TME':
         tme_api_settings = config_interface.load_file(global_settings.CONFIG_TME_API)
         supplier_settings[supplier]['API Token'] = [
@@ -672,6 +679,15 @@ def save_s(self, e: ft.ControlEvent, supplier: str, show_dialog=True):
             }
             lcsc_settings = {**settings_from_file, **updated_settings}
             config_interface.dump_file(lcsc_settings, global_settings.CONFIG_LCSC_API)
+        elif supplier == 'Jameco':
+            # Load settings from file
+            settings_from_file = config_interface.load_file(global_settings.CONFIG_JAMECO_API)
+            # Update settings values
+            updated_settings = {
+                'JAMECO_API_URL': SETTINGS[self.title][supplier]['API URL'][1].value,
+            }
+            jameco_settings = {**settings_from_file, **updated_settings}
+            config_interface.dump_file(jameco_settings, global_settings.CONFIG_JAMECO_API)
         elif supplier == 'TME':
             # Load settings from file
             settings_from_file = config_interface.load_file(global_settings.CONFIG_TME_API)
@@ -708,6 +724,9 @@ def test_s(self, e: ft.ControlEvent, supplier: str):
         elif supplier == 'LCSC':
             from ...search import lcsc_api
             result = lcsc_api.test_api()
+        elif supplier == 'Jameco':
+            from ...search import jameco_api
+            result = jameco_api.test_api()
         elif supplier == 'TME':
             from ...search import tme_api
             result = tme_api.test_api()

kintree/search/jameco_api.py

@@ -0,0 +1,172 @@
+import html
+import re
+from ..common.tools import download
+
+SEARCH_HEADERS = [
+    'title',
+    'name',
+    'prod_id',
+    'ss_attr_manufacturer',
+    'manufacturer_part_number',
+    'url',
+    'imageUrl',
+    'related_prod_id',
+    'category',
+]
+
+# Not really a map for Jameco.
+# Parameters are listed at same level as the search keys, not in separate list
+PARAMETERS_KEYS = [
+    'product_type_unigram',
+    'ss_attr_voltage_rating',
+    'ss_attr_multiple_order_quantity',
+]
+
+
+def get_default_search_keys():
+    # order matters, linked with part_form[] order in inventree_interface.translate_supplier_to_form()
+    return [
+        'title',
+        'name',
+        'revision',
+        'keywords',
+        'prod_id',
+        'ss_attr_manufacturer',
+        'manufacturer_part_number',
+        'url',
+        'datasheet',
+        'imageUrl',
+    ]
+
+
+def find_categories(part_details: str):
+    ''' Find categories '''
+    try:
+        return part_details['parentCatalogName'], part_details['catalogName']
+    except:
+        return None, None
+
+
+def fetch_part_info(part_number: str) -> dict:
+    ''' Fetch part data from API '''
+
+    # Load Jameco settings
+    from ..config import settings, config_interface
+    jameco_api_settings = config_interface.load_file(settings.CONFIG_JAMECO_API)
+
+    part_info = {}
+
+    def search_timeout(timeout=10):
+        url = jameco_api_settings.get('JAMECO_API_URL', '') + part_number
+        response = download(url, timeout=timeout)
+        return response
+
+    # Query part number
+    try:
+        part = search_timeout()
+        # Extract results, select first in returned search List
+        part = part.get('results', None)
+        part = part[0]
+    except:
+        part = None
+
+    if not part:
+        return part_info
+
+    category, subcategory = find_categories(part)
+    try:
+        part_info['category'] = category
+        part_info['subcategory'] = subcategory
+    except:
+        part_info['category'] = ''
+        part_info['subcategory'] = ''
+
+    headers = SEARCH_HEADERS
+
+    for key in part:
+        if key in headers:
+            if key == 'imageUrl':
+                try:
+                    part_info[key] = part['imageUrl']
+                except IndexError:
+                    pass
+            elif key in ['title', 'name', 'category']:
+                # Jameco title/name is often >100 chars, which causes an error later. Check for it here.
+                if (len(part[key]) > 100):
+                    trimmed_value = str(part[key])[:100]
+                    part_info[key] = html.unescape(trimmed_value)  # Json data sometimes has HTML encoded chars, e.g. "
+                else:
+                    part_info[key] = html.unescape(part[key])
+            else:
+                part_info[key] = part[key]
+
+    # Parameters
+    part_info['parameters'] = {}
+
+    for i, parameter_key in enumerate(PARAMETERS_KEYS):
+        if part.get(parameter_key, ''):
+            parameter_name = parameter_key
+            parameter_value = part[parameter_key]
+            if isinstance(parameter_value, list):
+                parameter_string = ', '.join(parameter_value)
+                part_info['parameters'][parameter_name] = parameter_string
+            else:
+                # Append to parameters dictionary
+                part_info['parameters'][parameter_name] = parameter_value
+
+    # Pricing
+    part_info['pricing'] = {}
+
+    # Jameco returns price breaks as a string of HTML text
+    # Convert pricing string pattern to List, then  dictionary for Ki-nTree
+    price_break_str = part['secondary_prices']
+    price_break_str = price_break_str.replace(',', '')  # remove comma
+    price_break_str = re.sub('(\<br\s\/&*gt)', '', price_break_str)  # remove HTML
+    price_break_str = re.sub(';', ':', price_break_str)  # remove ; char
+    price_break_str = re.sub('(\:\s+\$)|\;', ':', price_break_str)  # remove $ char
+    price_break_list = price_break_str.split(':')  # split on :
+    price_break_list.pop()  # remove last empty element in List
+
+    for i in range(0, len(price_break_list), 2):
+        quantity = int(price_break_list[i])
+        price = float(price_break_list[i + 1])
+        part_info['pricing'][quantity] = price
+
+    part_info['currency'] = 'USD'
+
+    # Extra search fields
+    if settings.CONFIG_JAMECO.get('EXTRA_FIELDS', None):
+        for extra_field in settings.CONFIG_JAMECO['EXTRA_FIELDS']:
+            if part.get(extra_field, None):
+                part_info['parameters'][extra_field] = part[extra_field]
+            else:
+                from ..common.tools import cprint
+                cprint(f'[INFO]\tWarning: Extra field "{extra_field}" not found in search results', silent=False)
+
+    return part_info
+
+
+def test_api() -> bool:
+    ''' Test method for API '''
+
+    test_success = True
+
+    expected = {
+        'manufacturer_part_number': 'PN2222ABU',
+        'name': 'Transistor PN2222A NPN Silicon General Purpose TO-92',
+        'prod_id': '178511',
+    }
+
+    test_part = fetch_part_info('178511')
+    if not test_part:
+        test_success = False
+
+    # Check content of response
+    if test_success:
+        for key, value in expected.items():
+            if test_part[key] != value:
+                print(f'"{test_part[key]}" <> "{value}"')
+                test_success = False
+                break
+
+    return test_success