updates

.vscode/settings.json

@@ -2,14 +2,17 @@
   "cSpell.words": [
     "cssless",
     "Instapundit",
+    "JSONIFY",
     "langchain",
     "llms",
     "Monero",
+    "mustexist",
     "ollama",
     "Ollama",
     "OLLAMA",
     "Slickdeals",
     "Tailscale",
-    "Thingiverse"
+    "Thingiverse",
+    "yamlpath"
   ]
 }

app/app.py

@@ -1,18 +1,19 @@
 import os
-import yaml
 import asyncio
 from datetime import datetime
 
-from flask import Flask, request, render_template
+from flask import Flask, Response, request, render_template
 from flask_caching import Cache
 
-from utils import copy_default_to_configs, load_file
+from utils import copy_default_to_configs
 from rss import rss
+from yaml_parser import yaml_parser
+import json
 
 copy_default_to_configs()
 
 app = Flask(__name__)
-
+app.config['JSONIFY_PRETTYPRINT_REGULAR'] = True
 if os.environ.get("FLASK_DEBUG", "False") == "True":
 	cache_config={
 		'CACHE_TYPE': 'null'
@@ -41,8 +42,8 @@ def save_tab_name():
 	column_count = data.get('column_count')
 
 	if tab_name and column_count >= 1 and column_count <= 6:
-		with open('configs/layout.yml', 'r') as file:
-			layout = yaml.safe_load(file)
+
+		layout = yaml_parser.load_layout()
 
 		tabs = layout['tabs']
 
@@ -54,8 +55,8 @@ def save_tab_name():
 			# Add a new tab
 			tabs.append({'name': tab_name, 'columns': column_count, 'widgets': []})
 
-		with open('configs/layout.yml', 'w') as file:
-			yaml.safe_dump(layout, file)
+		# with open('configs/layout.yml', 'w') as file:
+		# 	yaml.safe_dump(layout, file)
 
 		return {'message': f'Tab name "{tab_name}" with {column_count} columns saved successfully'}
 	else:
@@ -67,7 +68,7 @@ def save_tab_name():
 @cache.cached(timeout=600)
 async def index(tab_name=None):
 	# Load feeds and bookmarks
-	layout = load_file('layout.yml', cache)
+	layout = yaml_parser.load_layout()
 	headers = layout['headers']
 
 	tabs = layout['tabs']
@@ -96,6 +97,16 @@ async def index(tab_name=None):
 	# Pass column data to the template
 	return render_template('index.html', tabs=tabs, columns=columns, headers=headers, current_tab_name=current_tab['name'])
 
+
+@app.route('/widget/<widget_name>')
+async def widget(widget_name):
+	widget = yaml_parser.find_widget(widget_name)
+	widget = await rss.load_feed(widget)
+	return Response(
+        response=json.dumps(widget, indent=2),
+        status=200,
+        mimetype='application/json'
+    )
 if __name__ == '__main__':
 	port = int(os.environ.get("ONBOARD_PORT", 9830))
 	if os.environ.get("FLASK_DEBUG", "False") == "True":

app/file_data.py

@@ -0,0 +1,7 @@
+class FileData:
+	def __init__(self, last_modified=0, contents=None):
+		self.last_modified = last_modified
+		self.contents = contents
+
+	def __getitem__(self, key, default=None):
+		return self.contents.get(key, default)

app/post_processor.py

@@ -27,9 +27,9 @@ def to_snake_case(self, input_string):
 		return snake_case_string
 
 	def process(self, widget):
-		if 'processed' in widget and widget['processed'] and not bool(os.environ.get('FLASK_DEBUG')):
-			print (f"Widget {widget['name']} already processed.")
-			return widget
+		# if 'processed' in widget and widget['processed'] and not bool(os.environ.get('FLASK_DEBUG')):
+		# 	print (f"Widget {widget['name']} already processed.")
+		# 	return widget
 
 		self.normalize(widget)
 

app/processors/instapundit.py

@@ -31,7 +31,7 @@ def __init__(self):
 
 	def process(self, widget):
 		for article in widget['articles'][:]:
-			if '#CommissionEarned' in article['title'] or re.search('Open Thread', article['title'], re.IGNORECASE):
+			if article['title'] and ('#CommissionEarned' in article['title'] or re.search('Open Thread', article['title'], re.IGNORECASE)):
 				widget['articles'].remove(article)
 				next
 			if self.chain:

app/rss.py

@@ -51,7 +51,7 @@ async def load_feed(self, widget):
 		post_processor.process(widget)
 		self.feed_cache.set(widget['name'], widget)
 
-		return (time.time() - start_time)
+		return widget
 
 	def find_feed_links(self, url):
 		response = requests.get(url)

app/utils.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import yaml
 
 def copy_default_to_configs():
 	pwd = os.path.dirname(os.path.abspath(__file__))
@@ -23,27 +22,3 @@ def copy_default_to_configs():
 	else:
 		print(f"No files copied from {default_dir} to {config_dir}.")    
 
-class FileData:
-		def __init__(self, last_modified=0, contents=None):
-				self.last_modified = last_modified
-				self.contents = contents
-
-		def __getitem__(self, key, default=None):
-				return self.contents.get(key, default)
-
-file_cache = {}
-
-def load_file(file_name, cache=None):
-		current_working_directory = os.path.dirname(os.path.realpath(__file__))
-		file_path = os.path.join(current_working_directory, 'configs', file_name)
-
-		# Check the last modification time of the file
-		current_modified_time = os.path.getmtime(file_path)
-
-		# Only load the file if it has been modified since the last check or if there is no value for that file in the dict
-		if current_modified_time > file_cache.get(file_path, FileData()).last_modified or file_path not in file_cache:
-				with open(file_path, 'r') as file:
-						contents = yaml.safe_load(file)
-				file_cache[file_path] = FileData(current_modified_time, contents)
-
-		return file_cache[file_path].contents

app/yaml_parser.py

@@ -0,0 +1,62 @@
+
+import os
+from file_data import FileData
+import yaml
+from types import SimpleNamespace
+from yamlpath.common import Parsers
+from yamlpath.wrappers import ConsolePrinter
+from yamlpath import Processor
+from yamlpath import YAMLPath
+from yamlpath.exceptions import YAMLPathException
+
+class YamlParser:
+	def __init__(self):
+		self.file_cache = {}
+		self.pwd = os.path.dirname(os.path.realpath(__file__))
+
+		logging_args = SimpleNamespace(quiet=True, verbose=False, debug=False)
+		log = ConsolePrinter(logging_args)
+		parser = Parsers.get_yaml_editor()
+
+		# At this point, you'd load or parse your YAML file, stream, or string.  This
+		# example demonstrates loading YAML data from an external file.  You could also
+		# use the same function to load data from STDIN or even a String variable.  See
+		# the Parser class for more detail.
+		self.layout = os.path.join(self.pwd, "configs/layout.yml")
+		print(self.layout)
+		(yaml_data, doc_loaded) = Parsers.get_yaml_data(parser, log, self.layout)
+		if not doc_loaded:
+				# There was an issue loading the file; an error message has already been
+				# printed via ConsolePrinter.
+				exit(1)
+
+		# Pass the logging facility and parsed YAML data to the YAMLPath Processor
+		self.processor = Processor(log, yaml_data)
+
+	def find_widget(self, widget_name):
+		yaml_path = YAMLPath(f"/tabs/*/widgets[name = '{widget_name}']")
+
+		try:
+			for node in self.processor.get_nodes(yaml_path, return_coordinates=True, return_node=True, mustexist=True):
+				return node.node
+		except YAMLPathException as ex:
+				print(ex)
+
+		return None
+
+
+	def load_layout(self):
+		file_path = self.layout
+
+		# Check the last modification time of the file
+		current_modified_time = os.path.getmtime(file_path)
+
+		# Only load the file if it has been modified since the last check or if there is no value for that file in the dict
+		if current_modified_time > self.file_cache.get(file_path, FileData()).last_modified or file_path not in self.file_cache:
+				with open(file_path, 'r') as file:
+						contents = yaml.safe_load(file)
+				self.file_cache[file_path] = FileData(current_modified_time, contents)
+
+		return self.file_cache[file_path].contents
+
+yaml_parser = YamlParser()

requirements.txt

@@ -1,15 +1,15 @@
+aiohttp[speedups]
 argcomplete
 bs4
 docker
-feedparser
+Flask-Minify
 flask[async]
 flask-caching
-Flask-Minify
+hypercorn==0.15.0
 langchain
 langchain-community
 lxml
 python-dotenv
-pyyaml 
+pyyaml
 requests
-hypercorn==0.15.0
-aiohttp[speedups]
+yamlpath