diff --git a/.vscode/settings.json b/.vscode/settings.json index bf35d8d..ce1d2fe 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,14 +2,17 @@ "cSpell.words": [ "cssless", "Instapundit", + "JSONIFY", "langchain", "llms", "Monero", + "mustexist", "ollama", "Ollama", "OLLAMA", "Slickdeals", "Tailscale", - "Thingiverse" + "Thingiverse", + "yamlpath" ] } \ No newline at end of file diff --git a/app/app.py b/app/app.py index 439f20f..bfb4e01 100644 --- a/app/app.py +++ b/app/app.py @@ -1,18 +1,19 @@ import os -import yaml import asyncio from datetime import datetime -from flask import Flask, request, render_template +from flask import Flask, Response, request, render_template from flask_caching import Cache -from utils import copy_default_to_configs, load_file +from utils import copy_default_to_configs from rss import rss +from yaml_parser import yaml_parser +import json copy_default_to_configs() app = Flask(__name__) - +app.config['JSONIFY_PRETTYPRINT_REGULAR'] = True if os.environ.get("FLASK_DEBUG", "False") == "True": cache_config={ 'CACHE_TYPE': 'null' @@ -41,8 +42,8 @@ def save_tab_name(): column_count = data.get('column_count') if tab_name and column_count >= 1 and column_count <= 6: - with open('configs/layout.yml', 'r') as file: - layout = yaml.safe_load(file) + + layout = yaml_parser.load_layout() tabs = layout['tabs'] @@ -54,8 +55,8 @@ def save_tab_name(): # Add a new tab tabs.append({'name': tab_name, 'columns': column_count, 'widgets': []}) - with open('configs/layout.yml', 'w') as file: - yaml.safe_dump(layout, file) + # with open('configs/layout.yml', 'w') as file: + # yaml.safe_dump(layout, file) return {'message': f'Tab name "{tab_name}" with {column_count} columns saved successfully'} else: @@ -67,7 +68,7 @@ def save_tab_name(): @cache.cached(timeout=600) async def index(tab_name=None): # Load feeds and bookmarks - layout = load_file('layout.yml', cache) + layout = yaml_parser.load_layout() headers = layout['headers'] tabs = layout['tabs'] @@ -96,6 +97,16 @@ async def index(tab_name=None): # Pass column data to the template return render_template('index.html', tabs=tabs, columns=columns, headers=headers, current_tab_name=current_tab['name']) + +@app.route('/widget/') +async def widget(widget_name): + widget = yaml_parser.find_widget(widget_name) + widget = await rss.load_feed(widget) + return Response( + response=json.dumps(widget, indent=2), + status=200, + mimetype='application/json' + ) if __name__ == '__main__': port = int(os.environ.get("ONBOARD_PORT", 9830)) if os.environ.get("FLASK_DEBUG", "False") == "True": diff --git a/app/file_data.py b/app/file_data.py new file mode 100644 index 0000000..bcc8a45 --- /dev/null +++ b/app/file_data.py @@ -0,0 +1,7 @@ +class FileData: + def __init__(self, last_modified=0, contents=None): + self.last_modified = last_modified + self.contents = contents + + def __getitem__(self, key, default=None): + return self.contents.get(key, default) \ No newline at end of file diff --git a/app/post_processor.py b/app/post_processor.py index 4c4fdfd..f5cd895 100644 --- a/app/post_processor.py +++ b/app/post_processor.py @@ -27,9 +27,9 @@ def to_snake_case(self, input_string): return snake_case_string def process(self, widget): - if 'processed' in widget and widget['processed'] and not bool(os.environ.get('FLASK_DEBUG')): - print (f"Widget {widget['name']} already processed.") - return widget + # if 'processed' in widget and widget['processed'] and not bool(os.environ.get('FLASK_DEBUG')): + # print (f"Widget {widget['name']} already processed.") + # return widget self.normalize(widget) diff --git a/app/processors/instapundit.py b/app/processors/instapundit.py index 61bbe02..8e6272c 100644 --- a/app/processors/instapundit.py +++ b/app/processors/instapundit.py @@ -31,7 +31,7 @@ def __init__(self): def process(self, widget): for article in widget['articles'][:]: - if '#CommissionEarned' in article['title'] or re.search('Open Thread', article['title'], re.IGNORECASE): + if article['title'] and ('#CommissionEarned' in article['title'] or re.search('Open Thread', article['title'], re.IGNORECASE)): widget['articles'].remove(article) next if self.chain: diff --git a/app/rss.py b/app/rss.py index 515921b..26f2dc6 100644 --- a/app/rss.py +++ b/app/rss.py @@ -51,7 +51,7 @@ async def load_feed(self, widget): post_processor.process(widget) self.feed_cache.set(widget['name'], widget) - return (time.time() - start_time) + return widget def find_feed_links(self, url): response = requests.get(url) diff --git a/app/utils.py b/app/utils.py index 2c5a4b9..5f71cca 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,6 +1,5 @@ import os import shutil -import yaml def copy_default_to_configs(): pwd = os.path.dirname(os.path.abspath(__file__)) @@ -23,27 +22,3 @@ def copy_default_to_configs(): else: print(f"No files copied from {default_dir} to {config_dir}.") -class FileData: - def __init__(self, last_modified=0, contents=None): - self.last_modified = last_modified - self.contents = contents - - def __getitem__(self, key, default=None): - return self.contents.get(key, default) - -file_cache = {} - -def load_file(file_name, cache=None): - current_working_directory = os.path.dirname(os.path.realpath(__file__)) - file_path = os.path.join(current_working_directory, 'configs', file_name) - - # Check the last modification time of the file - current_modified_time = os.path.getmtime(file_path) - - # Only load the file if it has been modified since the last check or if there is no value for that file in the dict - if current_modified_time > file_cache.get(file_path, FileData()).last_modified or file_path not in file_cache: - with open(file_path, 'r') as file: - contents = yaml.safe_load(file) - file_cache[file_path] = FileData(current_modified_time, contents) - - return file_cache[file_path].contents diff --git a/app/yaml_parser.py b/app/yaml_parser.py new file mode 100644 index 0000000..f4500c7 --- /dev/null +++ b/app/yaml_parser.py @@ -0,0 +1,62 @@ + +import os +from file_data import FileData +import yaml +from types import SimpleNamespace +from yamlpath.common import Parsers +from yamlpath.wrappers import ConsolePrinter +from yamlpath import Processor +from yamlpath import YAMLPath +from yamlpath.exceptions import YAMLPathException + +class YamlParser: + def __init__(self): + self.file_cache = {} + self.pwd = os.path.dirname(os.path.realpath(__file__)) + + logging_args = SimpleNamespace(quiet=True, verbose=False, debug=False) + log = ConsolePrinter(logging_args) + parser = Parsers.get_yaml_editor() + + # At this point, you'd load or parse your YAML file, stream, or string. This + # example demonstrates loading YAML data from an external file. You could also + # use the same function to load data from STDIN or even a String variable. See + # the Parser class for more detail. + self.layout = os.path.join(self.pwd, "configs/layout.yml") + print(self.layout) + (yaml_data, doc_loaded) = Parsers.get_yaml_data(parser, log, self.layout) + if not doc_loaded: + # There was an issue loading the file; an error message has already been + # printed via ConsolePrinter. + exit(1) + + # Pass the logging facility and parsed YAML data to the YAMLPath Processor + self.processor = Processor(log, yaml_data) + + def find_widget(self, widget_name): + yaml_path = YAMLPath(f"/tabs/*/widgets[name = '{widget_name}']") + + try: + for node in self.processor.get_nodes(yaml_path, return_coordinates=True, return_node=True, mustexist=True): + return node.node + except YAMLPathException as ex: + print(ex) + + return None + + + def load_layout(self): + file_path = self.layout + + # Check the last modification time of the file + current_modified_time = os.path.getmtime(file_path) + + # Only load the file if it has been modified since the last check or if there is no value for that file in the dict + if current_modified_time > self.file_cache.get(file_path, FileData()).last_modified or file_path not in self.file_cache: + with open(file_path, 'r') as file: + contents = yaml.safe_load(file) + self.file_cache[file_path] = FileData(current_modified_time, contents) + + return self.file_cache[file_path].contents + +yaml_parser = YamlParser() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a04beef..3b71b9b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ +aiohttp[speedups] argcomplete bs4 docker -feedparser +Flask-Minify flask[async] flask-caching -Flask-Minify +hypercorn==0.15.0 langchain langchain-community lxml python-dotenv -pyyaml +pyyaml requests -hypercorn==0.15.0 -aiohttp[speedups] +yamlpath