diff --git a/.vscode/launch.json b/.vscode/launch.json index 60f5b13..9c08377 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,6 +5,7 @@ "version": "0.2.0", "configurations": [ + { "name": "Python Debugger: Flask", diff --git a/Makefile b/Makefile index 3117c83..7b3cd08 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ reqs: $(SITE_PACKAGES) $(SITE_PACKAGES): requirements.txt pip install -r requirements.txt - touch requirements.txt + -sudo touch $(SITE_PACKAGES) ansible: sudo chown -R ${USER}:${USER} ~/ diff --git a/app/models/layout.py b/app/models/layout.py index 1c4f2f0..c56fe8f 100644 --- a/app/models/layout.py +++ b/app/models/layout.py @@ -28,8 +28,8 @@ def __init__(self, config_file: str = "configs/layout.yml", bookmarks_bar_file: self.bookmark_bar_path = pwd.joinpath(bookmarks_bar_file) try: - if not os.path.exists(pwd.joinpath('configs/bookmarks.json')): - with open(pwd.joinpath('configs/bookmarks.json'), 'w', encoding='utf-8') as f: + if not os.path.exists(pwd.joinpath(self.bookmark_bar_path)): + with open(pwd.joinpath(self.bookmark_bar_path), 'w', encoding='utf-8') as f: json.dump([], f) except Exception as ex: logger.error(f"Error: {ex} creating empty bookmark bar file at {self.bookmark_bar_path}") @@ -39,10 +39,10 @@ def __init__(self, config_file: str = "configs/layout.yml", bookmarks_bar_file: def load_bookmarks(self): try: - with open(pwd.joinpath('configs/bookmarks.json'), 'r', encoding='utf-8') as f: + with open(pwd.joinpath(self.bookmark_bar_path), 'r', encoding='utf-8') as f: return json.load(f) except Exception as ex: - logger.error(f"Error: {ex} loading bookmarks") + logger.error(f"Error: Loading bookmark bar file from {self.bookmark_bar_path}", ex) return None def stop_scheduler(self): diff --git a/app/processors/title_editor.py b/app/processors/title_editor.py index 21f025d..bdece9a 100644 --- a/app/processors/title_editor.py +++ b/app/processors/title_editor.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta import logging import os -from pathlib import Path from pytz import utc from models.feed_article import FeedArticle @@ -9,38 +8,39 @@ from langchain.prompts import ChatPromptTemplate, PromptTemplate, HumanMessagePromptTemplate from langchain_core.messages import SystemMessage from langchain.output_parsers import ResponseSchema, StructuredOutputParser -from langchain.callbacks.tracers import ConsoleCallbackHandler +# from langchain.callbacks.tracers import ConsoleCallbackHandler from models.utils import calculate_sha1_hash logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) + class TitleEditor: - def __init__(self): - self.ollama_url = os.getenv('OLLAMA_URL') - if self.ollama_url: - parser = StructuredOutputParser.from_response_schemas( - [ResponseSchema(name="title", description="title of the article")] - ) - - format_instructions = f""" - {parser.get_format_instructions()} - The contents of the markdown code snippet MUST be in VALID json format which includes proper quotes and escape characters! - JSON property names are case sensitive, and MUST ONLY include the defined schema properties! - """ + def __init__(self): + self.ollama_url = os.getenv('OLLAMA_URL') + if self.ollama_url: + parser = StructuredOutputParser.from_response_schemas( + [ResponseSchema(name="title", description="title of the article")] + ) + + format_instructions = f""" + {parser.get_format_instructions()} + The contents of the markdown code snippet MUST be in VALID json format which includes proper quotes and escape characters! + JSON property names are case sensitive, and MUST ONLY include the defined schema properties! + """ - prompt = PromptTemplate( - template=""" + prompt = PromptTemplate( + template=""" title: {title}, summary: {summary}, {format_instructions} """, - input_variables=["title", "summary"], - partial_variables={"format_instructions": format_instructions}, - ) + input_variables=["title", "summary"], + partial_variables={"format_instructions": format_instructions}, + ) - # format chat prompt - system_prompt = SystemMessage(content=(""" + # format chat prompt + system_prompt = SystemMessage(content=(""" You are an expert news article title editor. Use the provided title and summary to write a concise and accurate title that is informative and avoids sounding like clickbait. Do not include links or urls in the title. @@ -50,41 +50,42 @@ def __init__(self): title MUST NOT use words that are all capitalized. NO SHOUTING! Only return the title in the requested format! """)) - user_prompt = HumanMessagePromptTemplate(prompt=prompt) - - chat_prompt = ChatPromptTemplate.from_messages([system_prompt, user_prompt]) - - model_name = "dolphin-llama3" - model_temp = 0.0 - llama3_model = Ollama(base_url=self.ollama_url, model=model_name, keep_alive=5, temperature=model_temp) - self.llama3_chain = chat_prompt | llama3_model | parser + user_prompt = HumanMessagePromptTemplate(prompt=prompt) + + chat_prompt = ChatPromptTemplate.from_messages([system_prompt, user_prompt]) + + model_name = "dolphin-llama3" + model_temp = 0.0 + llama3_model = Ollama(base_url=self.ollama_url, model=model_name, keep_alive=5, temperature=model_temp) + self.llama3_chain = chat_prompt | llama3_model | parser + + model_name = "dolphin-mistral" + model_temp = 0.0 + mistral_model = Ollama(base_url=self.ollama_url, model=model_name, keep_alive=5, temperature=model_temp) + self.mistral_chain = chat_prompt | mistral_model | parser + + self.script_hash = calculate_sha1_hash(f"{system_prompt.content}{model_name}{model_temp}") + + def process(self, articles: list[FeedArticle]) -> list[FeedArticle]: + if self.ollama_url: - model_name = "dolphin-mistral" - model_temp = 0.0 - mistral_model = Ollama(base_url=self.ollama_url, model=model_name, keep_alive=5, temperature=model_temp) - self.mistral_chain = chat_prompt | mistral_model | parser - - self.script_hash = calculate_sha1_hash(f"{system_prompt.content}{model_name}{model_temp}") + needs_processed = list(filter(lambda article: article.processed != self.script_hash, articles)) + if len(needs_processed) > 10: + needs_processed = list(filter(lambda article: article.pub_date.replace(tzinfo=utc) >= + (datetime.now() - timedelta(days=1)).replace(tzinfo=utc), articles)) - def process(self, articles: list[FeedArticle]) -> list[FeedArticle]: - if self.ollama_url: - - needs_processed = list(filter(lambda article: article.processed != self.script_hash, articles)) - if len(needs_processed) > 10: - needs_processed = list(filter(lambda article: article.pub_date.replace(tzinfo=utc) >= (datetime.now() - timedelta(days=1)).replace(tzinfo=utc), articles)) + total = len(needs_processed) + for count, article in enumerate(needs_processed, start=1): + for chain in [self.llama3_chain, self.mistral_chain]: + try: + logger.info(f"Processing title {count}/{total}: {article.original_title}") + # , config={'callbacks': [ConsoleCallbackHandler()]}) + result = chain.invoke({"title": article.original_title, "summary": article.description}) + article.title = result['title'] + article.processed = self.script_hash + break + except Exception as ex: + logger.error(f"Error: {ex} for {article.original_title}") + # needs_processed.remove(article) - - total = len(needs_processed) - for count, article in enumerate(needs_processed, start=1): - for chain in [self.llama3_chain, self.mistral_chain]: - try: - logger.info(f"Processing title {count}/{total}: {article.original_title}") - result = chain.invoke({"title": article.original_title, "summary": article.description}) #, config={'callbacks': [ConsoleCallbackHandler()]}) - article.title = result['title'] - article.processed = self.script_hash - break - except Exception as ex: - logger.error(f"Error: {ex} for {article.original_title}") - #needs_processed.remove(article) - - return articles + return articles diff --git a/app/services/favicon_finder.py b/app/services/favicon_finder.py index 51fb76e..8be1a95 100644 --- a/app/services/favicon_finder.py +++ b/app/services/favicon_finder.py @@ -1,10 +1,14 @@ +import json import logging import os +import re import requests from urllib.parse import urljoin, urlparse from bs4 import BeautifulSoup from models.utils import pwd from models.scheduler import Scheduler +from PIL import Image +from io import BytesIO logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -15,54 +19,122 @@ def __init__(self, cache_dir='static/assets/icons'): self.full_cache_path = pwd.joinpath(cache_dir) self.full_cache_path.mkdir(parents=True, exist_ok=True) self.relative_cache_path = f"/{cache_dir}" + self.processed_domains = set() + self.processed_domains_file = pwd.joinpath('configs/processed_domains.json') + self.load_processed_domains() - def favicon_exists(self, url): - if not url: - return False - favicon_filename = self.get_favicon_filename(url) - favicon_path = os.path.join(self.full_cache_path, favicon_filename) - return os.path.exists(favicon_path) + def add_processed_domain(self, url, reason='completed'): + normalized_domain = self.normalize_domain(url) + self.processed_domains.append((normalized_domain, reason)) + self.save_processed_domains() - def favicon_relative_path(self, url): - return f"{self.relative_cache_path}/{self.get_favicon_filename(url)}" + def save_processed_domains(self): + try: + with open(self.processed_domains_file, 'w') as f: + json.dump(list(self.processed_domains), f, ensure_ascii=True, indent=2) + except Exception as ex: + logger.error(f"Error saving processed domains to disk: {ex}") - def get_favicon_filename(self, url): - domain_parts = urlparse(url).netloc.split('.')[-2:] - return '.'.join(domain_parts) + '.favicon.ico' + def load_processed_domains(self): + try: + if os.path.exists(self.processed_domains_file): + with open(self.processed_domains_file, 'r') as f: + self.processed_domains = json.load(f) + except Exception as ex: + logger.error(f"Error loading processed domains from disk: {ex}") @property def scheduler(self): return Scheduler.getScheduler() - def fetch_from_iterator(self, urls): - for url in urls: - self.scheduler.add_job(self._get_favicon, args=[url], misfire_grace_time=None, executor='processpool') + def normalize_domain(self, url): + parsed_url = urlparse(url) + domain_parts = parsed_url.netloc.split('.') + if domain_parts[0].startswith("www"): + domain_parts.pop(0) + return '.'.join(domain_parts) - def _get_favicon(self, url): - favicon_filename = self.get_favicon_filename(url) - favicon_path = os.path.join(self.full_cache_path, favicon_filename) + def make_request(self, url): + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'} + response = requests.get(url, headers=headers, allow_redirects=True) + return response - if not os.path.exists(favicon_path): - icon_url = self.find_favicon_url(url) + def favicon_exists(self, url): + if not url: + return False + try: + normalized_domain = self.normalize_domain(url) + favicon_filename = self.get_favicon_filename(normalized_domain) + favicon_path = os.path.join(self.full_cache_path, favicon_filename) + if os.path.exists(favicon_path): + return f"{self.relative_cache_path}/{favicon_filename}" + return None + except Exception as ex: + logger.error(f"Error checking if favicon exists for {url}: {ex}") + return None - if not icon_url: - # If favicon URL is not found for the original URL, try the base URL - base_url = self.get_base(url) - icon_url = self.find_favicon_url(base_url) + def get_favicon_filename(self, domain): + return domain + '.favicon.ico' - if icon_url: - self.download_favicon(url, icon_url) - else: - logger.warn(f'Favicon not found for {url} or {self.get_base(url)}') + def is_ip_address(self, url): + ip_pattern = re.compile( + r"^(?:(?:https?://)?(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}" + r"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?::\d{1,5})?(?:\/)?$" + ) + return bool(ip_pattern.match(url)) + + def is_domain_processed(self, url): + normalized_domain = self.normalize_domain(url) + domains = [domain for domain, _ in self.processed_domains] + return ( + normalized_domain in domains + or self.is_ip_address(url) + or self.favicon_exists(url) + or 'trivantis' in normalized_domain + or not url + ) def get_base(self, url): parsed_url = urlparse(url) base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" return base_url + def fetch_from_iterator(self, urls): + for url in urls: + if not self.is_domain_processed(url): + self.add_processed_domain(url) + self.scheduler.add_job( + self._get_favicon, + args=[url], + misfire_grace_time=None, + executor='processpool' + ) + + def _get_favicon(self, url): + normalized_domain = self.normalize_domain(url) + favicon_filename = self.get_favicon_filename(normalized_domain) + favicon_path = os.path.join(self.full_cache_path, favicon_filename) + + if os.path.exists(favicon_path): + return + + icon_url = self.find_favicon_url(url) + + if not icon_url: + # If favicon URL is not found for the original URL, try the base URL + base_url = self.get_base(url) + icon_url = self.find_favicon_url(base_url) + + if icon_url: + self.download_favicon(icon_url) + else: + logger.warn(f'Favicon not found for {normalized_domain}') + self.add_processed_domain(normalized_domain, reason='not found') + def find_favicon_url(self, url): try: - response = requests.get(url) + response = self.make_request(url) if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') icon_link = soup.find('link', rel=['icon', 'shortcut icon']) @@ -72,22 +144,42 @@ def find_favicon_url(self, url): icon_url = urljoin(url, icon_url) return icon_url else: + icon_url = f'http://www.google.com/s2/favicons?domain={self.normalize_domain(url)}' + response = self.make_request(icon_url) + if response.status_code == 200 and response.headers.get('Content-Type', '').startswith('image'): + return icon_url return None else: return None - except requests.exceptions.RequestException: + except Exception as ex: + logger.error(f'Error finding favicon with url: {url}: {ex}') + self.add_processed_domain(self.normalize_domain(url), reason=f'Error downloading favicon: {ex}') return None - def download_favicon(self, url, icon_url): + def download_favicon(self, icon_url): + if icon_url.startswith('http://www.google.com/s2/favicons?domain='): + # strip 'http://www.google.com/s2/favicons?domain=' from the URL + normalized_domain = icon_url[len('http://www.google.com/s2/favicons?domain='):] + else: + normalized_domain = self.normalize_domain(icon_url) + favicon_filename = self.get_favicon_filename(normalized_domain) + favicon_path = os.path.join(self.full_cache_path, favicon_filename) + try: - response = requests.get(icon_url) + response = self.make_request(icon_url) if response.status_code == 200: - favicon_filename = self.get_favicon_filename(url) - favicon_path = os.path.join(self.full_cache_path, favicon_filename) - with open(favicon_path, 'wb') as file: - file.write(response.content) - logger.debug(f'Favicon for {self.get_base(url)} downloaded and saved as {favicon_path}') + content_type = response.headers.get('content-type', '').lower() + if content_type.startswith('image/'): + + with open(favicon_path, 'wb') as file: + file.write(response.content) + logger.debug(f'Favicon for {normalized_domain} downloaded and saved as {favicon_path}') + else: + logger.warn(f'The downloaded file from {icon_url} is not a valid image') + self.add_processed_domain(normalized_domain, reason='not an image') else: - logger.warn(f'Failed to download the favicon for {self.get_base(url)}') - except requests.exceptions.RequestException as ex: - logger.error(f'An error occurred while downloading the favicon for {self.get_base(url)}', ex) + logger.warn(f'Failed to download the favicon for {self.get_base(icon_url)}') + self.add_processed_domain(normalized_domain, reason='not an image') + except Exception as ex: + logger.error(f'Error downloading favicon for {normalized_domain} with url: {icon_url}: {ex}') + self.add_processed_domain(normalized_domain, reason=f'Error downloading favicon: {ex}') diff --git a/app/static/css/bookmark_bar.css b/app/static/css/bookmark_bar.css index b9ac04a..75e669e 100644 --- a/app/static/css/bookmark_bar.css +++ b/app/static/css/bookmark_bar.css @@ -5,9 +5,6 @@ #bookmarkBar { background-color: #2c2c2c; width: 100%; - position: absolute; - top: 0; - left: 0; z-index: 1000; height: var(--bookmark-bar-height); font-size: 12px; diff --git a/app/templates/bookmark_bar.html b/app/templates/bookmark_bar.html index 5625db7..3f858a5 100644 --- a/app/templates/bookmark_bar.html +++ b/app/templates/bookmark_bar.html @@ -11,8 +11,9 @@ {% endwith %} {% else %} - {% if favicon_exists(bookmark.href) %} - {{ bookmark.name }} + {% set favicon_url = favicon_exists(bookmark.href) %} + {% if favicon_url %} + {{ bookmark.name }} {% else %} {% endif %} diff --git a/notebooks/bookmark-output.ipynb b/notebooks/bookmark-output.ipynb index 5181d15..14c2467 100644 --- a/notebooks/bookmark-output.ipynb +++ b/notebooks/bookmark-output.ipynb @@ -2,652 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "metadata": {} }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Gmail\n", - "Links\n", - " covid\n", - " I Have Been Through This Before - Tablet Magazine\n", - " Physical interventions to interrupt or reduce the spread of respiratory viruses - Jefferson, T - 2023 | Cochrane Library\n", - " Guess what DID NOT get discussed at today’s G7 meeting – twitchy.com\n", - " ‘SEE how it works?!’ Tweep pulls BACK curtain on woke gender experts refusing to admit women are WOMEN (and how to push back) in EPIC thread – twitchy.com\n", - " Global famines of 'biblical proportions' will be caused by coronavirus pandemic, UN warns - CNN\n", - " The dam is about to break on the COVID shots – Behind The Black – Robert Zimmerman\n", - " The Harm Caused by Masks | City Journal\n", - " news\n", - " Andrew Follett says ‘the media is telling you two major lies’ about mass shootings and gun control – twitchy.com\n", - " The Ukrainegate ‘Whisteblower’ Isn’t a Real Whistleblower – Rolling Stone\n", - " tech\n", - " Securing NGINX-ingress | cert-manager\n", - " Kubernetes Ingress with Nginx Example - Kubernetes Book\n", - " Basic usage - NGINX Ingress Controller\n", - " Install And Configure Traefik with Helm\n", - " kubectl Cheat Sheet | Kubernetes\n", - " The Ultimate Guide to Building Your Personal K3S Cluster | by Nima Mahmoudi | Nov, 2020 | ITNEXT\n", - " Hardening SSH With Ansible • Nathan Curry\n", - " zimmertr/Bootstrap-Kubernetes-with-QEMU: Bootstrap Kubernetes on Proxmox using QEMU, Ansible, and Debian\n", - " k8s@home\n", - " self-hosted-cookbook/wikijs.md at master · tborychowski/self-hosted-cookbook\n", - " Kubernetes: A curated list of learning material\n", - " hobby-kube/guide: Kubernetes clusters for the hobbyist.\n", - " ProfessorSalty/homelab-ansible\n", - " Your Microsoft Teams chats aren’t as private as you think.. | Infinite Logins\n", - " Restore pfsense configuration backup from console using USB drive - Linux Tutorials - Learn Linux Configuration\n", - " Log - Lenovo M720Q Tiny router/firewall build with aftermarket 4 port NIC | SFF.Network\n", - " programming\n", - " amos - Working with strings in Rust\n", - " Build a Simple API in Rust 2023\n", - " MeshMap | Meshtastic Nodes\n", - " Dev Container metadata reference\n", - " ZFS Snapshot Backups to an External Drive with LUKS -- λ ryan. himmelwright. net\n", - " (553) I Built a Keypad to Control My Entire Desk Setup - OLED TV, USB Switch, Lights, Desk, Macros... - YouTube\n", - " Making the $250 Proxmox HA Cluster Hyperconverged :: apalrd's adventures\n", - " Austin's Nerdy Things - Nerding out, one post at a time\n", - " Joshua Powers\n", - " What are some MUST HAVE apps or websites? : r/dndnext\n", - " The Alexandrian » Game Structures\n", - "Projects\n", - " Cluster\n", - " CI\n", - " Using Concourse CI to Deploy to Docker Swarm - Ruan Bekker's Blog\n", - " Concourse Pipeline to Build a Docker Image Automatically on Git Commit - Ruan Bekker's Blog\n", - " Janik Vonrotz - Setup GoCD environment using docker\n", - " factor-io/factor: Factor Server runs your continuous deployment workflows\n", - " Strider-CD/strider: Open Source Continuous Integration & Deployment Server\n", - " pressly/sup: Super simple deployment tool - think of it like 'make' for a network of servers\n", - " summerwind/actions-runner-controller: Kubernetes controller for GitHub Actions self-hosted runnners\n", - " Docker\n", - " Cheatsheets\n", - " Linux Performance\n", - " Rails\n", - " Dockerize Rails, the lean way | Georg Ledermann\n", - " Reduce your Docker images (an example with Ruby) - Code Pub\n", - " unixcharles/acme-client: A Ruby client for the letsencrypt's ACME protocol.\n", - " Wordpress\n", - " Docker WordPress in a subdirectory – SusaNET\n", - " docker - Setup wordpress container in subdirectory with nginx and mariadb - Server Fault\n", - " Linux Performance\n", - " Using Docker WordPress Cli to Manage WordPress Websites - Datanovia\n", - " How to Change WordPress URLs in MySQL Database Using phpMyAdmin\n", - " How to Fix the upload_max_filesize Error in WordPress\n", - " Restoring Your Database From Backup | WordPress.org\n", - " joyent/containerpilot: A service for autodiscovery and configuration of applications running in containers\n", - " huginn/huginn: Create agents that monitor and act on your behalf. Your agents are standing by!\n", - " Smashing/smashing: The exceptionally handsome dashboard framework in Ruby and Coffeescript.\n", - " n1trux/awesome-sysadmin: A curated list of amazingly awesome open source sysadmin resources.\n", - " Privoxy - Home Page\n", - " Integrate Rails logs with Elasticsearch, Logstash, and Kibana in Docker Compose - Eric London » Open Source » Software Blog\n", - " How To Migrate a Docker Compose Workflow to Kubernetes | DigitalOcean\n", - " Jacob Errington | Roll your own Ngrok with Nginx, Letsencrypt, and SSH reverse tunnelling\n", - " FreeDNS - Free DNS - Dynamic DNS - Static DNS subdomain and domain hosting\n", - " Docker containers log transport and aggregation at scale\n", - " A monitoring solution for Docker hosts, containers and containerized services\n", - " Dokku - The smallest PaaS implementation you've ever seen\n", - " Migrating From Heroku to Dokku | COREYJA\n", - " Nick Busey / HomelabOS · GitLab\n", - " Kibitzr\n", - " Apache Guacamole™\n", - " Mina\n", - " Security Onion\n", - " Network UPS Tools - Hardware compatibility list\n", - " Tikal Knowledge - Ofelia - cron tasks on steroids in Docker\n", - " docker-registry-pruner/examples.md at master · tumblr/docker-registry-pruner\n", - " jarischaefer/docker-librenms: Docker image for LibreNMS\n", - " sismics/docs: Lightweight document management system packed with all the features you can expect from big expensive solutions\n", - " Joplin - an open source note taking and to-do application with synchronisation capabilities\n", - " silverwind/droppy: Self-hosted file storage\n", - " Free self-hosted Zendesk & Help Scout alternative – FreeScout\n", - " Raneto - A free, open, simple Markdown powered Knowledgebase for Nodejs\n", - " rmountjoy92/DashMachine: Another web application bookmark dashboard, with fun features.\n", - " warner/magic-wormhole: get things from one computer to another, safely\n", - " blockstack/gaia: A decentralized high-performance storage system\n", - " Hashicorp\n", - " pete0emerson/hashipoc: A sample application using Vagrant, Consul, Vault, and Nomad\n", - " eldondev/pxecorenomad: A set of scripts for creating an oem cloud config for coreos to start serf and nomad clients\n", - " chrisjalinsky/vagrant-consul-nomad-xtradb: A lab environment to test Hashicorp Consul and Nomad managed VMs and Containers using Vagrant and Ansible as the deployment method to Linux Ubuntu with Libvirt/KVM or Mac OSX with Virtualbox\n", - " RabbitMQ clustering with Consul in Nomad\n", - " Running Alluxio On HashiCorp Nomad | Alluxio\n", - " Running Hashicorp Nomad, Consul, Pihole and Gitea on Raspberry Pi 3 B+ | by Georgijs Radovs | The Startup | Medium\n", - " Tyblog | Going Completely Overboard with a Clustered Homelab\n", - " jippi/awesome-nomad: A curated list of amazingly awesome Nomad tools and shiny things.\n", - " perrymanuk/hashi-homelab: small lightweight homelab based on nomad and consul from hashicorp\n", - " poseidon/matchbox: Network boot and provision Container Linux / Fedora CoreOS clusters\n", - " Kubernetes\n", - " Reading \"Ansible for Kubernetes\" | Leanpub\n", - " lokomotive-kubernetes/bare-metal.md at master · kinvolk/lokomotive-kubernetes\n", - " KubeWeekly\n", - " Metal³ - Metal Kubed\n", - " Building a Network Bootable Server Farm for Kubernetes with LTSP - Kubernetes\n", - " KubeVirt.io\n", - " Kubernetes using Interactive Browser-Based Labs | Katacoda\n", - " Introduction to Kubernetes :: Introduction to Kubernetes\n", - " lensapp/lens: Lens - The Kubernetes IDE\n", - " Infra App - Kubernetes made easy\n", - " Velero\n", - " Set Up Docker Registry Mirror as a Docker Hub Pull-through Cache for Minikube Local K8s Cluster | by Jacky Jiang | Jan, 2021 | ITNEXT\n", - " HowTO: Set up K8s or K3s so pods get IP from lan DHCP - Multus : rancher\n", - " HowTo: K8s, MetalLB and external DNS access for services : homelab\n", - " How To Migrate a Docker Compose Workflow to Kubernetes | DigitalOcean\n", - " bottlerocket-os/bottlerocket: An operating system designed for hosting containers\n", - " How To Provision VMs on KVM with Terraform - Computing for Geeks\n", - " Scaling out with CephFS and KVM - The Basics · Noah Bailey\n", - " How to setup a hypervisor in ubuntu 18.04 with KVM and Kimchi\n", - " Awesome-Kubernetes | awesome-kubernetes\n", - " 50+ Useful Kubernetes Tools | Caylent\n", - " 50+ Useful Kubernetes Tools List - Part 2 | Caylent\n", - " Kubernetes using Interactive Browser-Based Labs | Katacoda\n", - " Introduction to Kubernetes :: Introduction to Kubernetes\n", - " lokomotive-kubernetes/bare-metal.md at master · kinvolk/lokomotive-kubernetes\n", - " Reading \"Ansible for Kubernetes\" | Leanpub\n", - " Running a Windows VM on KubeVirt on K3s · cookies and containers\n", - " KVM\n", - " Setup Headless Virtualization Server Using KVM In Ubuntu 18.04 LTS - OSTechNix\n", - " KVM: Bare metal virtualization on Ubuntu with KVM – Fabian Lee : Software Architect\n", - " Complete Installation of KVM, QEMU and Virt Manager on Arch Linux and Manjaro - Computing for Geeks\n", - " dlford/ubuntu-vm-boilerplate: A simple bash script to handle boilerplate configurations for cloned Ubuntu VMs (Machine ID, SSH server keys, Hostname)\n", - " Run virt-manager on Windows 10 | MangoLassi\n", - " Scaling out with CephFS and KVM - The Basics · Noah Bailey\n", - " How to setup a hypervisor in ubuntu 18.04 with KVM and Kimchi\n", - " How To Provision VMs on KVM with Terraform - Computing for Geeks\n", - " How To Provision VMs on KVM with Terraform - Computing for Geeks\n", - " xcp-ng\n", - " Ezka77/xen-orchestra-ce: Docker & docker-compose files to deploy Xen Orchestra Community Edition (ie: from sources)\n", - " Using Ansible to automate VM creation on XenServer - Juri Rischel Jensen - Medium\n", - " XCP-ng documentation | XCP-ng documentation\n", - " Netdata Agent | Learn\n", - " Netdata package is now available in XCP-ng | XCP-ng forum\n", - " Migrated a Plex Intel NUC to XCP-ng Xenserver with iGPU Passthrough - Jason Loong\n", - " PXE\n", - " PXE booting – Diego Lemos' blog\n", - " Poor Man’s device discovery (DNS) - Myatu’s - Medium\n", - " Home Cluster (Part II): Machine Setup, PiXiEs!\n", - " plunder-app/plunder: A Modern automation platform\n", - " poseidon/matchbox: Network boot and provision Container Linux / Fedora CoreOS clusters\n", - " Hyper v PXE boot - how we do it easily\n", - " Bare-Metal - Typhoon\n", - " Bare-Metal - Typhoon\n", - " Is there any way to provision bare-metal with Packer? - Server Fault\n", - " Debian PXE boot image from scratch | Randy's Blog\n", - " Tinkerbell, a Provisioning Engine by Packet\n", - " Welcome to Linux From Scratch!\n", - " Building a Network Bootable Server Farm for Kubernetes with LTSP - Kubernetes\n", - " Build your own datacenter with PXE and Alpine\n", - " Cloud-init that works\n", - " Network setup\n", - " Ansible: Deploy VMs With PXE and Kickstart • Nathan Curry\n", - " Build your own network simulator using open-source DevOps tools | Open-Source Routing and Network Simulation\n", - " WtRPM: A Web-based (Wt) suite to power up/down your computers - mupuf.org\n", - " Tyblog | Going Completely Overboard with a Clustered Homelab\n", - " Infrastructure as code in the home - blog.skouf.com\n", - " Network setup\n", - " Dell Fan Noise Control - Silence Your Poweredge : homelab\n", - " Convert LSI 9211-8i HBA card to IT mode\n", - " One NGINX error page to rule them all · Adriaan's blog\n", - " The encrypted homelab\n", - " Build your own datacenter with PXE and Alpine\n", - " How to defend your website with ZIP bombs\n", - " bottlerocket-os/bottlerocket: An operating system designed for hosting containers\n", - " How To Create Infinite Windows Cloud Desktops for Your Nonprofit with BoXenLinux® – Jeffrey Hein\n", - " Running Hashicorp Nomad, Consul, Pihole and Gitea on Raspberry Pi 3 B+ | by Georgijs Radovs | The Startup | Medium\n", - " Reading \"Ansible for Kubernetes\" | Leanpub\n", - " poseidon/matchbox: Network boot and provision Container Linux / Fedora CoreOS clusters\n", - " pete0emerson/hashipoc: A sample application using Vagrant, Consul, Vault, and Nomad\n", - " eldondev/pxecorenomad: A set of scripts for creating an oem cloud config for coreos to start serf and nomad clients\n", - " chrisjalinsky/vagrant-consul-nomad-xtradb: A lab environment to test Hashicorp Consul and Nomad managed VMs and Containers using Vagrant and Ansible as the deployment method to Linux Ubuntu with Libvirt/KVM or Mac OSX with Virtualbox\n", - " lokomotive-kubernetes/bare-metal.md at master · kinvolk/lokomotive-kubernetes\n", - " RabbitMQ clustering with Consul in Nomad\n", - " Building a Network Bootable Server Farm for Kubernetes with LTSP - Kubernetes\n", - " Hyper v PXE boot - how we do it easily\n", - " Bare-Metal - Typhoon\n", - " How to avoid conflicts between dnsmasq and systemd-resolved? - Unix & Linux Stack Exchange\n", - " Running Alluxio On HashiCorp Nomad | Alluxio\n", - " How To Provision VMs on KVM with Terraform - Computing for Geeks\n", - " Scaling out with CephFS and KVM - The Basics · Noah Bailey\n", - " How to setup a hypervisor in ubuntu 18.04 with KVM and Kimchi\n", - " Run virt-manager on Windows 10 | MangoLassi\n", - " Welcome to Linux From Scratch!\n", - " Is there any way to provision bare-metal with Packer? - Server Fault\n", - " Hardware\n", - " IBM ServeRAID M1015 Part 4: Cross flashing to a LSI9211-8i in IT or IR mode\n", - " Dell Fan Noise Control - Silence Your Poweredge : homelab\n", - " [VGA Hack] How To Make a VGA Dummy Plug | Geeks3D\n", - " Convert LSI 9211-8i HBA card to IT mode\n", - " Network UPS Tools - Hardware compatibility list\n", - " Harnessing Your Data with Diskover\n", - " New Hard Drive rituals\n", - " Convert LSI 9211-8i HBA card to IT mode\n", - " bcache\n", - " Redmine\n", - " hacpaka/jira2redmine: The accurate data migration from JIRA to Redmine.\n", - " Windows\n", - " QTTabBar - QuizoApps\n", - " XMeters - Taskbar System Monitoring\n", - " https://www.highrez.co.uk/downloads/xmousebuttoncontrol.htm\n", - " KMSpico Activator Download | Official Website [2020]\n", - " Awesome-Windows/Awesome: 🎉 An awesome & curated list of best applications and tools for Windows.\n", - " sirredbeard/Awesome-WSL: Awesome list dedicated to Windows Subsystem for Linux\n", - " Archive button disabled or missing in Outlook 2016? – Gus Perez\n", - " Find which hardware can wake up Windows 10\n", - " Prevent & Disable Windows Update from Waking Up PC System - Tech Journey\n", - " How To Create Infinite Windows Cloud Desktops for Your Nonprofit with BoXenLinux® – Jeffrey Hein\n", - " Attention all Windows-AD admins: March 2020 will be a lot of fun! : sysadmin\n", - " Four PowerShell commands to help you track down insecure LDAP Bindings before March 2020 : sysadmin\n", - " Main | WinFsp\n", - " How to Transfer PuTTY Sessions To Another Windows Machine - nixCraft\n", - " Security\n", - " Securing SSH with the Vault SSH backend and GitHub authentication – Dicking with Docker\n", - " Build a Tiny Certificate Authority For Your Homelab\n", - " thinkst/opencanary: Modular and decentralised honeypot\n", - " Suricata | Open Source IDS / IPS / NSM engine\n", - " Secure Proxmox Install – Sudo, Firewall with IPv6, and more – How to Configure from Start to Finish « KiloRoot\n", - " Proxmox Notes\n", - " OpenSSL Certificate Authority — Jamie Nguyen\n", - " OpenSSL Certificate Authority — Jamie Nguyen\n", - " Cheatsheets\n", - " Linux Performance\n", - " Homelab Rat - Homelab Rat\n", - " Wasabi Cloud Storage Pricing | 80% Cheaper Than AWS S3\n", - " A personal Newspaper : selfhosted\n", - " anysync (AnySync Project)\n", - " iam4x/bobarr: 🍿 The all-in-one alternative for Sonarr, Radarr, Jackett... with a VPN and running in docker\n", - " papercups-io/papercups: Open-source live customer chat\n", - " bunkerity/bunkerized-nginx: nginx Docker image secure by default.\n", - " eerotal/LibreSignage: An open source digital signage solution\n", - " Requarks/wiki: Wiki.js | A modern, lightweight and powerful wiki app built on Node.js\n", - " Ananas Analytics Desktop · Build analytics in minutes\n", - " The low code framework to build internal tools\n", - " ciur/papermerge: Open Source Document Management System for Digital Archives (Scanned Documents)\n", - " frnmst/kalliope-docker: A Python script that downloads, runs, builds and handles the voice controlled personal assistant Kalliope inside a Debian Docker container.\n", - " Linux Performance\n", - " \n", - " floccus - browser bookmarks sync\n", - " Create a Docker dashboard with TypeScript, React and Socket.io\n", - " Healthchecks.io – Cron Job Monitoring\n", - " GoAccess - Visual Web Log Analyzer\n", - " Building a BitTorrent client from the ground up in Go | Jesse Li\n", - " ilikenwf/apt-fast: apt-fast: A shellscript wrapper for apt that speeds up downloading of packages.\n", - " Flyway by Redgate • Database Migrations Made Easy.\n", - " What I'm running\n", - " BorgBackup – Deduplicating archiver with compression and authenticated encryption\n", - " Cloud-init that works\n", - " Linux\n", - " bitwarden\n", - " Using the MySQL Backend · dani-garcia/bitwarden_rs Wiki\n", - " Bitwarden Duplicate Entries Remover : Bitwarden\n", - " Plex\n", - " HaveAGitGat/Tdarr: Tdarr Beta -Audio/Video library analytics + transcode automation using FFmpeg/HandBrake + video health checking (Windows, macOS, Linux & Docker)\n", - " Hardware Transcoding with Plex Docker : PleX\n", - " (2) UNLOCK NVIDIA Cards for More Plex Transcoding! - YouTube\n", - " linuxserver/plex - LinuxServer.io\n", - " Roku Recommended FFMPEG h264 Preset\n", - " Software Lists\n", - " Redmine\n", - " hacpaka/jira2redmine: The accurate data migration from JIRA to Redmine.\n", - " Windows\n", - " Ventoy\n", - " QTTabBar - QuizoApps\n", - " XMeters - Taskbar System Monitoring\n", - " https://www.highrez.co.uk/downloads/xmousebuttoncontrol.htm\n", - " KMSpico Activator Download | Official Website [2020]\n", - " casey/just: 🤖 Just a command runner\n", - " Redash helps you make sense of your data\n", - " Metabase\n", - " bradtraversy/design-resources-for-developers: Curated list of design and UI resources from stock photos, web templates, CSS frameworks, UI libraries, tools and much more\n", - " kenn/sunzi-recipes: Remote recipes for Sunzi\n", - " JanVanRyswyck/awesome-talks: Awesome online talks and screencasts\n", - " alexellis/awesome-baremetal: Bare-metal is awesome. Let's share our favourite tools.\n", - " silverwind/droppy: Self-hosted file storage\n", - " 50+ Useful Kubernetes Tools List - Part 2 | Caylent\n", - " 50+ Useful Kubernetes Tools | Caylent\n", - " \n", - " Nick Busey / HomelabOS · GitLab\n", - " Awesome Ruby\n", - " Local == Better ❤️ (My Dashboard) : selfhosted\n", - " Awesome-Kubernetes | awesome-kubernetes\n", - " Kickball/awesome-selfhosted: This is a list of Free Software network services and web applications which can be hosted locally. Selfhosting is the process of locally hosting and managing applications instead of renting from SaaS providers.\n", - " jippi/awesome-nomad: A curated list of amazingly awesome Nomad tools and shiny things.\n", - " perrymanuk/hashi-homelab: small lightweight homelab based on nomad and consul from hashicorp\n", - " n1trux/awesome-sysadmin: A curated list of amazingly awesome open source sysadmin resources.\n", - " awesome-selfhosted/awesome-selfhosted: A list of Free Software network services and web applications which can be hosted locally. Selfhosting is the process of hosting and managing applications instead of renting from Software-as-a-Service providers\n", - " Free self-hosted Zendesk & Help Scout alternative – FreeScout\n", - " rmountjoy92/DashMachine: Another web application bookmark dashboard, with fun features.\n", - " What are some smaller, not well known selfhosted items you've been using / watching that others should know about? : selfhosted\n", - " Harnessing Your Data with Diskover\n", - " Healthchecks.io – Cron Job Monitoring\n", - " GoAccess - Visual Web Log Analyzer\n", - " Raneto - A free, open, simple Markdown powered Knowledgebase for Nodejs\n", - " warner/magic-wormhole: get things from one computer to another, safely\n", - " blockstack/gaia: A decentralized high-performance storage system\n", - " Graphing PowerEdge r710 power usage using Telegraf, InfluxDB & Grafana\n", - " Easy Bash Prompt Generator\n", - " ilikenwf/apt-fast: apt-fast: A shellscript wrapper for apt that speeds up downloading of packages.\n", - " OpenSSL Certificate Authority — Jamie Nguyen\n", - " How to avoid conflicts between dnsmasq and systemd-resolved? - Unix & Linux Stack Exchange\n", - " New Hard Drive rituals\n", - " bcache\n", - " What is this - Funky Penguin's Geek Cookbook\n", - " Raspberry Pi Kiosk using Chromium - Pi My Life Up\n", - " [VGA Hack] How To Make a VGA Dummy Plug | Geeks3D\n", - " 3D Printing\n", - " Makerfarm\n", - " 10 Pegasus Printer.pdf - Google Drive\n", - " Firmware.pdf - Google Drive\n", - " Troubleshooting.pdf - Google Drive\n", - " Troubleshooting.pdf - Google Drive\n", - " lf-/reality: Documentation for physical projects\n", - " have a marlin 1.1.5 or 1.0.2-2 configuration.h for for a pegasus 12 w/Bl touch?\n", - " Best Creality 3D Printing Forum & Group | Creality 3D\n", - " Free 3D Printable Files and Designs | Pinshape\n", - " GrabCAD: Design Community, CAD Library, 3D Printing Software\n", - " CR6-SE\n", - " A Possible Cause For The USB Shorting Issue : CR6\n", - " (20+) Creality CR 6 SE / MAX Community | Facebook\n", - " Creality CR-6 SE: Start a print without drooping filament on the build plate while homing | Sebastiaan Dammann\n", - " Why upgrade firmware? : CR6\n", - " Gantry_Levelling_and_Wrong_Timing.pdf\n", - " I got the printer definition out of Creality Slicer : CR6\n", - " leveling - cr6\n", - " Teaching Tech 3D Printer Calibration\n", - " 3D Printer Auto Bed Leveling Mesh Visualizer – lokster | space\n", - " Mesh Visualizer – MKDev\n", - " Making\n", - " Laser\n", - " Upgrading a K40 Laser Cutter, Part 1: Control Board Upgrade » fox_robotics\n", - " Don's Things\n", - " pc build\n", - " 1m Computer Mesh 30CM DIY PVC PC Case Fan Cooler Black Dust Filter Network Net Case Dustproof Cover Chassis Dust Cover|Device Cleaners| - AliExpress\n", - " 5PCS 80mm 90mm 120mm 140mm Cuttable Black PVC PC Fan Dust Filter Dustproof Case Computer Mesh Cooling Ultra Fine Dustproof Cover|Fans| - AliExpress\n", - " 12CM Iron Net CPU Cooling Fan Cover Finger Guard Motherboard Protection Net Computer Fan for 12025 12038 AC/DC free shipping|computer table fan|computer fan rpmcomputer companions - AliExpress\n", - " Minecraft\n", - " minecraft java edition - How to automatically broadcast local server name? - Arqade\n", - " gameservers-docker/advertise.py at master · OpenSourceLAN/gameservers-docker\n", - " NPI\n", - " Avalara\n", - " AvaTax REST API\n", - " Sales Tax API - Avalara\n", - " AvaTax.NET client library available on NuGet - Avalara\n", - " avadev/AvaTax-REST-V2-DotNet-SDK: AvaTax v2 SDK for languages using the Dot Net Framework\n", - " avadev/AvaTax-REST-V2-Ruby-SDK: Sales Tax API SDK for Ruby and AvaTax REST\n", - " EbizCharge\n", - " AddInvoice\n", - " Programming\n", - " Unity\n", - " Use Unity's LineRenderer to draw a circle on a GameObject\n", - " SolarSystemSimulatorGame/OrbitRenderer.cs at master · sotos82/SolarSystemSimulatorGame\n", - " The Barnes-Hut Galaxy Simulator\n", - " Rendering a Galaxy with the density wave theory\n", - " Terrain Generation Using Procedural Models Based on Hydrology\n", - " Fortune's algorithm Archives - Alan Zucconi\n", - " List of gravitationally rounded objects of the Solar System - Wikipedia\n", - " ellioman/ShaderProject: A container for all sorts of handy shaders.\n", - " Best FREE Unity Assets - Over 200 Quality Curated Assets for Unity 3D\n", - " GUIText object jitters when scripted to follow a GameObject - Unity Forum\n", - " Building a BitTorrent client from the ground up in Go | Jesse Li\n", - " Flyway by Redgate • Database Migrations Made Easy.\n", - " Rust\n", - " GitHub - casey/just: 🤖 Just a command runner\n", - " rust-lang/rustlings: Small exercises to get you used to reading and writing Rust code!\n", - " rust-unofficial/awesome-rust: A curated list of Rust code and resources.\n", - " gist.cafe\n", - " Project Jupyter | Try Jupyter\n", - " RTSP\n", - " How to Generate Better Video Previews with ffmpeg\n", - " Scrub image playback demo\n", - " blakeblackshear/frigate: Realtime object detection on RTSP cameras with the Google Coral\n", - " VXG Media Player - Chrome Web Store\n", - " VXG Media Plugin for Chrome OS – Video Experts Group\n", - " Kerberos.io - an innovative video surveillance security system built for the community and enterprises.\n", - " Raspberry Pi + Deep Learning home security system\n", - " Capture/Lightning – FFmpeg\n", - " video capture - ffmpeg buffered recording - Video Production Stack Exchange\n", - " dewgenenny/trigger-sensor: Python script to trigger a binary sensor in Home Assistant via the API\n", - " BlueIris Successfully Running in a Docker Container : BlueIris\n", - " jrottenberg/ffmpeg - Docker Hub\n", - " ffmpeg/Dockerfile at master · opencoconut/ffmpeg\n", - " Home surveillance and motion detection with the Raspberry Pi, Python, OpenCV, and Dropbox - PyImageSearch\n", - " pynvr.py · master · Jack Strohm / PyNVR · GitLab\n", - " Process Monitoring with Systemd - Servers for Hackers\n", - " video - rtsp stream capturing - Stack Overflow\n", - " How to generate better video previews with ffmpeg - Tutorial - Binpress\n", - " Edit fiddle - JSFiddle\n", - " How to use ffmpeg to extract live stream into a sequence of mp4 - Super User\n", - " OpenCV detect movement in python - Stack Overflow\n", - " Scene change/shot detection/image extraction using ffmpeg from video - Stack Overflow\n", - " jooray/motion-detection: OpenCV motion detection for Raspberry Pi + Web interface\n", - " johmathe/Shotdetect: Automated shot detection software\n", - " Record and archive video from IP cameras | LucaTNT's\n", - " How to run a cron job inside a docker container? - Stack Overflow\n", - " Concatenate – FFmpeg\n", - " How To Install Ruby on Rails with rbenv on Ubuntu 16.04 | DigitalOcean\n", - " Multiple cameras with the Raspberry Pi and OpenCV - PyImageSearch\n", - " FFmpeg - ArchWiki\n", - " How to make an animated thumbnail | Webdesigner Depot\n", - " html video progress bar with js - Trivantis Community\n", - " A Plugin For Mediaelement.js For Preview Thumbnails on Hover Over the Time Rail Using WebVTT | Preliminary Inventory of Digital Collections by Jason Ronallo\n", - " Combine multiple images to form a strip of images ffmpeg - Super User\n", - " html5 - Create a image every XX seconds of the video [FFMPEG] - Super User\n", - " ffmpeg - Automatically split large .mov video files into smaller files at black frames (scene changes)? - Super User\n", - " On-Hover/Toggle Play HTML5 Video with only Vanilla JavaScript\n", - " OnHover HTML Video Vanilla JS\n", - " Kerberos.io - an innovative video surveillance security system built for the community and enterprises.\n", - " Windows\n", - " Archive button disabled or missing in Outlook 2016? – Gus Perez\n", - " Main | WinFsp\n", - " Find which hardware can wake up Windows 10\n", - " Prevent & Disable Windows Update from Waking Up PC System - Tech Journey\n", - " Attention all Windows-AD admins: March 2020 will be a lot of fun! : sysadmin\n", - " Four PowerShell commands to help you track down insecure LDAP Bindings before March 2020 : sysadmin\n", - " Laser\n", - " DIY CO2 LASER: R-LASER 6020 : 32 Steps (with Pictures) - Instructables\n", - " Parallels for Aligning Lasers and CNC Routers : 6 Steps (with Pictures) - Instructables\n", - " Upgrading a K40 Laser Cutter, Part 1: Control Board Upgrade » fox_robotics\n", - " Don's Things\n", - " RC\n", - " Hobbypro Conversion kit For Traxxas Revo | EuroRC.com\n", - " Makerfarm\n", - " have a marlin 1.1.5 or 1.0.2-2 configuration.h for for a pegasus 12 w/Bl touch?\n", - " Bluetooth\n", - " Triena Hi-Fi Bluetooth Headset : 7 Steps (with Pictures) - Instructables\n", - " Internet radio and music player with ESP8266 |\n", - " Playing audio files with CSR8645 Bluetooth chip |\n", - " esp32-snippets/Mijia-LYWSD03MMC-Client at master · karolkalinski/esp32-snippets\n", - " zewelor/bt-mqtt-gateway: A simple Python script which provides a Bluetooth to MQTT gateway, easily extensible via custom workers. See https://github.com/zewelor/bt-mqtt-gateway/wiki for more information.\n", - " algirdasc/xiaomi-ble-mqtt: Xiaomi BLE Temperature and Humidity Sensor Bluetooth To MQTT gateway\n", - " Plex\n", - " Hardware Transcoding with Plex Docker : PleX\n", - " (2) UNLOCK NVIDIA Cards for More Plex Transcoding! - YouTube\n", - " linuxserver/plex - LinuxServer.io\n", - " Roku Recommended FFMPEG h264 Preset\n", - " Quadro P400 Encoding 8 streams in Linux on Xeon E3-1226 (Lenovo TS140) : PleX\n", - " Tutorial on setting up unlimited transcodes for Nvidia GPUs : PleX\n", - " keylase/nvidia-patch: This patch removes restriction on maximum number of simultaneous NVENC video encoding sessions imposed by Nvidia to consumer-grade GPUs.\n", - " jantenhove/NvencSessionLimitBump: C++ program to disable NvEnc session limit for D3D on patched systems\n", - " nVidia Hardware Transcoding Calculator for Plex Estimates\n", - " Passthrough Intel iGPU with GVT-g to a VM and use Quick Sync with Plex in docker on Proxmox\n", - " [VGA Hack] How To Make a VGA Dummy Plug | Geeks3D\n", - " Unity\n", - " Use Unity's LineRenderer to draw a circle on a GameObject\n", - " SolarSystemSimulatorGame/OrbitRenderer.cs at master · sotos82/SolarSystemSimulatorGame\n", - " The Barnes-Hut Galaxy Simulator\n", - " Rendering a Galaxy with the density wave theory\n", - " Terrain Generation Using Procedural Models Based on Hydrology\n", - " Fortune's algorithm Archives - Alan Zucconi\n", - " List of gravitationally rounded objects of the Solar System - Wikipedia\n", - " ellioman/ShaderProject: A container for all sorts of handy shaders.\n", - " Best FREE Unity Assets - Over 200 Quality Curated Assets for Unity 3D\n", - " GUIText object jitters when scripted to follow a GameObject - Unity Forum\n", - " Software Lists\n", - " Redmine\n", - " New folder\n", - " hacpaka/jira2redmine: The accurate data migration from JIRA to Redmine.\n", - " Windows\n", - " QTTabBar - QuizoApps\n", - " XMeters - Taskbar System Monitoring\n", - " https://www.highrez.co.uk/downloads/xmousebuttoncontrol.htm\n", - " KMSpico Activator Download | Official Website [2020]\n", - " bookmark sync\n", - " New folder\n", - " floccus - browser bookmarks sync\n", - " Awesome Docker\n", - " binhnguyennus/awesome-scalability: The Patterns of Scalable, Reliable, and Performant Large-Scale Systems\n", - " veggiemonk/awesome-docker: A curated list of Docker resources and projects\n", - " Awesome-Windows/Awesome: 🎉 An awesome & curated list of best applications and tools for Windows.\n", - " sirredbeard/Awesome-WSL: Awesome list dedicated to Windows Subsystem for Linux\n", - " sismics/docs: Lightweight document management system packed with all the features you can expect from big expensive solutions\n", - " Joplin - an open source note taking and to-do application with synchronisation capabilities\n", - " NexClipper/exporterhub.io: A Curated List of Prometheus Exporters\n", - " silverwind/droppy: Self-hosted file storage\n", - " 50+ Useful Kubernetes Tools List - Part 2 | Caylent\n", - " 50+ Useful Kubernetes Tools | Caylent\n", - " \n", - " Nick Busey / HomelabOS · GitLab\n", - " Awesome Ruby\n", - " Local == Better ❤️ (My Dashboard) : selfhosted\n", - " Awesome-Kubernetes | awesome-kubernetes\n", - " Kickball/awesome-selfhosted: This is a list of Free Software network services and web applications which can be hosted locally. Selfhosting is the process of locally hosting and managing applications instead of renting from SaaS providers.\n", - " jippi/awesome-nomad: A curated list of amazingly awesome Nomad tools and shiny things.\n", - " perrymanuk/hashi-homelab: small lightweight homelab based on nomad and consul from hashicorp\n", - " n1trux/awesome-sysadmin: A curated list of amazingly awesome open source sysadmin resources.\n", - " awesome-selfhosted/awesome-selfhosted: A list of Free Software network services and web applications which can be hosted locally. Selfhosting is the process of hosting and managing applications instead of renting from Software-as-a-Service providers\n", - " Free self-hosted Zendesk & Help Scout alternative – FreeScout\n", - " rmountjoy92/DashMachine: Another web application bookmark dashboard, with fun features.\n", - " What are some smaller, not well known selfhosted items you've been using / watching that others should know about? : selfhosted\n", - " Awesome Lists\n", - " Awesome Ruby\n", - " gotify/server - Docker Hub\n", - " caronc/apprise - Docker Hub\n", - " veggiemonk/awesome-docker: A curated list of Docker resources and projects\n", - " Awesome Docker\n", - " n1trux/awesome-sysadmin: A curated list of amazingly awesome open source sysadmin resources.\n", - " awesome-selfhosted/awesome-selfhosted: A list of Free Software network services and web applications which can be hosted locally. Selfhosting is the process of hosting and managing applications instead of renting from Software-as-a-Service providers\n", - " binhnguyennus/awesome-scalability: The Patterns of Scalable, Reliable, and Performant Large-Scale Systems\n", - " NexClipper/exporterhub.io: A Curated List of Prometheus Exporters\n", - " Nick Busey / HomelabOS · GitLab\n", - " Local == Better ❤️ (My Dashboard) : selfhosted\n", - " What are some smaller, not well known selfhosted items you've been using / watching that others should know about? : selfhosted\n", - " Reverse proxy with Caddy 2 • Jordi Burgos\n", - " Pomerium — open source identity-aware access proxy — now supports TCP : devops\n", - " codemash 2020\n", - " Kubernetes\n", - " Kubernetes using Interactive Browser-Based Labs | Katacoda\n", - " Introduction to Kubernetes :: Introduction to Kubernetes\n", - " Home Automation\n", - " Teckin SB50 800lm RGBW Bulb Template for Tasmota\n", - " blakeblackshear/frigate at release-0.8.0\n", - " ct-Open-Source/tuya-convert: A collection of scripts to flash Tuya IoT devices to alternative firmwares\n", - " Split Single-phase Energy Meter | Crowd Supply\n", - " LYWSD03MMC Temp\n", - " Telink Flasher\n", - " atc1441/ATC_MiThermometer: Custom firmware for the Xiaomi Thermometer LYWSD03MMC and Telink Flasher via USB to Serial converter\n", - " LYWSD03MMC support (implemented since 0.6.0) · Issue #7 · custom-components/sensor.mitemp_bt\n", - " Create a Docker dashboard with TypeScript, React and Socket.io\n", - " How to Transfer PuTTY Sessions To Another Windows Machine - nixCraft\n", - " Archive button disabled or missing in Outlook 2016? – Gus Perez\n", - " Kerberos.io - an innovative video surveillance security system built for the community and enterprises.\n", - " [VGA Hack] How To Make a VGA Dummy Plug | Geeks3D\n", - " Main | WinFsp\n", - " Find which hardware can wake up Windows 10\n", - " Prevent & Disable Windows Update from Waking Up PC System - Tech Journey\n", - " Harnessing Your Data with Diskover\n", - " Healthchecks.io – Cron Job Monitoring\n", - " Convert LSI 9211-8i HBA card to IT mode\n", - " New Hard Drive rituals\n", - " Debian PXE boot image from scratch | Randy's Blog\n", - " GoAccess - Visual Web Log Analyzer\n", - " Raneto - A free, open, simple Markdown powered Knowledgebase for Nodejs\n", - " Building a BitTorrent client from the ground up in Go | Jesse Li\n", - " ilikenwf/apt-fast: apt-fast: A shellscript wrapper for apt that speeds up downloading of packages.\n", - " OpenSSL Certificate Authority — Jamie Nguyen\n", - " Attention all Windows-AD admins: March 2020 will be a lot of fun! : sysadmin\n", - " Four PowerShell commands to help you track down insecure LDAP Bindings before March 2020 : sysadmin\n", - " bcache\n", - " warner/magic-wormhole: get things from one computer to another, safely\n", - " blockstack/gaia: A decentralized high-performance storage system\n", - " Flyway by Redgate • Database Migrations Made Easy.\n", - " Premium Bootstrap Themes and Templates: Download @ Creative Tim\n", - " IronicBadger/docker-sanoid-builder: Builds Sanoid from source in a container\n", - " BookStack\n", - " WebOas.is | WebOasis | Web Oasis | v1.22.22\n", - " Self Hosted CICD with Gitea and Drone CI - DEV\n", - " We're sorry, but something went wrong (500)\n", - "Rearden\n", - " Homepage\n", - " firewall.ilude.com\n", - " Frigate\n", - " Stable Diffusion\n", - " Movies - Radarr\n", - " Plex\n", - " Prometheus\n", - " Prowlarr - Indexers\n", - " SABnzbd\n", - " TV Shows - Sonarr\n", - " unifi.ilude.com\n", - " Clark | Syncthing\n", - " Pi-hole - pihole\n", - " NZB\n", - " Sign In - Elitenzb.info\n", - " NZBgeek\n", - " Welcome - DS\n", - " NZBIndex - We index, you search\n", - " Support the community - abNZB\n", - "Machining\n", - " Bidspotter.com | Industrial, Commercial, Plant and Machinery auctions\n", - " Production Tool Supply® | Division of PTS® Group\n", - " Wholesale Tool | Industrial Machinery + Tool Supply\n", - " allindustrialtoolsupply on eBay\n", - " Shars\n", - " Sussex Tool & Supply\n", - " Find Specialty Carbide End Mills and Cutting Tools at Harvey Tool\n", - " Travers Tool Co. — Find Metalworking Tools, Machine Tools & More\n", - " Zoro.com: 1,000s of Brands, Millions of Products\n", - " Guy Lautard.Com Machinists Books & Supplies: Lathes, Milling, Drills, Gunsmithing and Clock making\n", - " Used Machinery & Industrial Equipment | HGR Industrial Surplus\n", - " www.hartlandauctions.com/Upcoming_Auctions.html\n", - " Laser Cutters | Advertising Signage Printing Industry | Toolots\n", - "Deals\n", - " shopping\n", - " Amazon.com: YITAHOME 5 Drawer Chest, Mobile File Cabinet with Wheels, Home Office Storage Dresser Cabinet, Black : Office Products\n", - " Amazon.com: QLLY 28 inch Adjustable Metal Desk Legs, Office Table Furniture Leg Set, Set of 4 (Black) : Tools & Home Improvement\n", - " Ebern Designs Ashendon Ergonomic Mesh Task Chair & Reviews | Wayfair\n", - " Harbor Freight \n", - " Harbor Freight Tools Coupon Database - Free coupons, 25 percent off coupons, 20 percent off coupons, No Purchase Required coupons, toolbox coupons\n", - " Struggleville tool deal website logo\n", - " Plastic Box Assortments\n", - "OnBoard\n", - "YouTube\n", - "Reddit\n", - "Twitch\n", - "Current\n", - " prebid\n", - " Prebid for beginners — with examples | by Emil Hein | Ad-tech | Medium\n", - " Basic Example for Header Bidding\n", - " Download Prebid.js for Header Bidding\n", - " Is the Topics API the future of digital advertising — With examples | by Emil Hein | Ad-tech | Medium\n", - " JSFiddle - url drop\n", - " Dev Container metadata reference\n", - " GetRSSFeed: RSS Feed Extractor & Finder for Websites, Blogs & Podcasts\n", - " Prompt Engineering Guide | Prompt Engineering Guide\n", - " rssfilter/backend/tests/test_recommend.py at 67e6befda4a7c987d2be1cced111232da2e08216 · m0wer/rssfilter\n", - " sunshine - selfhosted gaming\n", - " rssfilter/backend/app/models/article.py at 67e6befda4a7c987d2be1cced111232da2e08216 · m0wer/rssfilter\n", - " recommenders-team/recommenders: Best Practices on Recommendation Systems\n", - " Deep-Learning/T5-Fake-News-Detector/T5-fake-news-detector-github.ipynb at master · priya-dwivedi/Deep-Learning\n", - " buriy/python-readability: fast python port of arc90's readability tool, updated to match latest readability.js!\n", - " readeck/readeck: Readeck is a simple web application that lets you save the precious readable content of web pages you like and want to keep forever. - Codeberg.org\n", - " jenkins-docker/entrypoint.sh at main · sudo-bmitch/jenkins-docker\n", - " Build your Flask admin panel with ease following your guide\n", - " Integrating Bun: A Developer’s Journey | by Junior Felix | Medium\n", - "Kanban - agpm-demo\n" - ] - } - ], + "outputs": [], "source": [ "import json\n", "\n", @@ -659,11 +18,151 @@ "\n", " \n", "\n", - "with open('data/bookmarks.json', 'r', encoding='utf-8') as f:\n", + "with open('data/bookmarks_bar.json', 'r', encoding='utf-8') as f:\n", " bookmarks = json.load(f)\n", "\n", "process_bookmarks(bookmarks)" ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from PIL import Image\n", + "\n", + "def is_image(file_path):\n", + " try:\n", + " img = Image.open(file_path)\n", + " img.close()\n", + " return True\n", + " except Exception as e:\n", + " return False\n", + "\n", + "def check_directory(directory):\n", + " for file_name in os.listdir(directory):\n", + " file_path = os.path.join(directory, file_name)\n", + " if os.path.isfile(file_path) and file_path.endswith('.ico'):\n", + " if not is_image(file_path):\n", + " print(f\"{file_name} is not an image file. Deleting...\")\n", + " os.remove(file_path)\n", + "\n", + "directory_path = \"../app/static/assets/icons\"\n", + "check_directory(directory_path)\n", + "print (\"Cleanup completed!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "True\n", + "False\n" + ] + } + ], + "source": [ + "import re\n", + "\n", + "def is_ip_address(url):\n", + " ip_pattern = re.compile(\n", + " r\"^(?:(?:https?://)?(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}\"\n", + " r\"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?::\\d{1,5})?(?:\\/)?$\"\n", + " )\n", + " return bool(ip_pattern.match(url))\n", + "\n", + "# Example usage:\n", + "url1 = \"192.168.1.1\"\n", + "url2 = \"http://192.168.16.208:7860/\"\n", + "print(is_ip_address(url1)) # Output: True\n", + "print(is_ip_address(url2)) # Output: True\n", + "print(is_ip_address('https://example.com')) # Output: False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "def check_content_type_starts_with_image(domain):\n", + " url = f'http://www.google.com/s2/favicons?domain={domain}'\n", + " response = requests.get(url)\n", + " \n", + " if response.status_code == 200:\n", + " content_type = response.headers.get('Content-Type', '')\n", + " if content_type.startswith('image'):\n", + " return True\n", + " else:\n", + " return False\n", + " else:\n", + " print(f\"Failed to fetch URL: {url}\")\n", + " return False\n", + "\n", + "# Example usage:\n", + "domain = \"amazon.com\"\n", + "print(check_content_type_starts_with_image(domain))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "http://www.google.com/s2/favicons?domain=https://www.amazon.com\n" + ] + } + ], + "source": [ + "from urllib.parse import urljoin, urlparse\n", + "from bs4 import BeautifulSoup\n", + "import requests\n", + "\n", + "def get_base( url):\n", + " parsed_url = urlparse(url)\n", + " base_url = f\"{parsed_url.scheme}://{parsed_url.netloc}\"\n", + " return base_url\n", + "\n", + "\n", + "def find_favicon_url(url):\n", + " try:\n", + " headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'}\n", + " response = requests.get(url, headers=headers)\n", + " if response.status_code == 200:\n", + " soup = BeautifulSoup(response.text, 'html.parser')\n", + " icon_link = soup.find('link', rel=['icon', 'shortcut icon'])\n", + " if icon_link:\n", + " icon_url = icon_link['href']\n", + " if not icon_url.startswith('http'):\n", + " icon_url = urljoin(url, icon_url)\n", + " return icon_url\n", + " else:\n", + " domain = get_base(url)\n", + " response = requests.get(f'http://www.google.com/s2/favicons?domain={domain}', headers=headers)\n", + " if response.status_code == 200 and response.headers.get('Content-Type', '').startswith('image'):\n", + " return f'http://www.google.com/s2/favicons?domain={domain}'\n", + " return None\n", + " else:\n", + " return None\n", + " except Exception as ex:\n", + " print(f'Error finding favicon with url: {url}: {ex}')\n", + "\n", + "print(find_favicon_url('https://www.amazon.com/'))" + ] } ], "metadata": { diff --git a/notebooks/bookmark-processing-1.ipynb b/notebooks/bookmark-processing-1.ipynb index 0e3f27e..b37529a 100644 --- a/notebooks/bookmark-processing-1.ipynb +++ b/notebooks/bookmark-processing-1.ipynb @@ -24,7 +24,7 @@ " bookmarks_json = json.dumps(bookmarks_data, indent=2)\n", "\n", "# Write the JSON data to a file\n", - " with open('data/bookmarks.json', 'w') as file:\n", + " with open('data/bookmarks_bar.json', 'w') as file:\n", " file.write(bookmarks_json)" ] }, @@ -72,7 +72,7 @@ "soup = BeautifulSoup(html_content, 'html.parser')\n", "bookmarks = parse_bookmarks(soup.find('dl'))\n", "with open('./data/bookmarks.json', 'w', encoding='utf-8') as f:\n", - " json.dump(bookmarks, f, ensure_ascii=False, indent=2)" + " json.dump(bookmarks, f, ensure_ascii=True, indent=2)" ] } ],