Merge pull request #276 from NekoAria/2.0

v2.5.6

.env

@@ -1,2 +1,2 @@
 ENVIRONMENT=prod
-VERSION='v2.5.4'
+VERSION='v2.5.6'

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ELF_RSS"
-version = "2.5.5"
+version = "2.5.6"
 description = "QQ机器人 RSS订阅 插件，订阅源建议选择 RSSHub"
 authors = ["Quan666 <[email protected]>"]
 license = "GPL-3.0-only"

src/plugins/ELF_RSS2/RSS/qbittorrent_download.py

@@ -72,7 +72,7 @@ async def get_qb_client() -> Optional[Client]:
         qb.get_default_save_path()
     except Exception:
         bot = nonebot.get_bot()
-        msg = f"❌ 无法连登录到 qbittorrent ，请检查是否勾选“对本地主机上的客户端跳过身份验证”"
+        msg = "❌ 无法连登录到 qbittorrent ，请检查是否勾选“对本地主机上的客户端跳过身份验证”"
         logger.exception(msg)
         await bot.send_private_msg(user_id=str(list(config.superusers)[0]), message=msg)
         return None

src/plugins/ELF_RSS2/RSS/routes/Parsing/__init__.py

@@ -147,7 +147,7 @@ def _handler_filter(_handler_list: List[ParsingItem], _url: str) -> List[Parsing
         (h.func.__name__, "(.*)", h.priority) for h in _result if h.rex != "(.*)"
     ]
     _result = [
-        h for h in _result if not ((h.func.__name__, h.rex, h.priority) in _delete)
+        h for h in _result if (h.func.__name__, h.rex, h.priority) not in _delete
     ]
     return _result
 
@@ -177,7 +177,7 @@ async def start(self, rss_name: str, new_rss: Dict[str, Any]) -> None:
         # 前置处理
         rss_title = new_rss["feed"]["title"]
         new_data = new_rss["entries"]
-        _file = DATA_PATH / (rss_name + ".json")
+        _file = DATA_PATH / f"{rss_name}.json"
         db = TinyDB(
             _file,
             storage=CachingMiddleware(JSONStorage),  # type: ignore
@@ -241,7 +241,7 @@ async def start(self, rss_name: str, new_rss: Dict[str, Any]) -> None:
 @ParsingBase.append_before_handler(priority=10)
 async def handle_check_update(rss: Rss, state: Dict[str, Any]):
     db = state.get("tinydb")
-    change_data = await check_update(db, state.get("new_data"))
+    change_data = check_update(db, state.get("new_data"))
     return {"change_data": change_data}
 
 
@@ -299,7 +299,7 @@ async def handle_check_update(rss: Rss, state: Dict[str, Any]):
         conn = sqlite3.connect(str(DATA_PATH / "cache.db"))
         conn.set_trace_callback(logger.debug)
 
-    await cache_db_manage(conn)
+    cache_db_manage(conn)
 
     delete = []
     for index, item in enumerate(change_data):
@@ -398,13 +398,13 @@ async def handle_summary(
     tmp_state: Dict[str, Any],
 ) -> str:
     try:
-        tmp += await handle_html_tag(html=Pq(get_summary(item)))
+        tmp += handle_html_tag(html=Pq(get_summary(item)))
     except Exception as e:
         logger.warning(f"{rss.name} 没有正文内容！{e}")
     return tmp
 
 
-# 处理正文 移出指定内容
+# 处理正文 移除指定内容
 @ParsingBase.append_handler(parsing_type="summary", priority=11)
 async def handle_summary(
     rss: Rss,
@@ -418,6 +418,10 @@ async def handle_summary(
     if rss.content_to_remove:
         for pattern in rss.content_to_remove:
             tmp = re.sub(pattern, "", tmp)
+        # 去除多余换行
+        while "\n\n\n" in tmp:
+            tmp = tmp.replace("\n\n\n", "\n\n")
+        tmp = tmp.strip()
     return tmp
 
 
@@ -548,7 +552,7 @@ async def handle_message(
     if await send_msg(rss=rss, msg=item_msg, item=item):
 
         if rss.duplicate_filter_mode:
-            await insert_into_cache_db(
+            insert_into_cache_db(
                 conn=state["conn"], item=item, image_hash=item["image_hash"]
             )
 
@@ -583,7 +587,7 @@ async def after_handler(rss: Rss, state: Dict[str, Any]) -> Dict[str, Any]:
         conn.close()
 
     new_data_length = len(state["new_data"])
-    await cache_json_manage(db, new_data_length)
+    cache_json_manage(db, new_data_length)
     db.close()
 
     return {}

src/plugins/ELF_RSS2/RSS/routes/Parsing/cache_manage.py

@@ -35,7 +35,7 @@ def cache_filter(data: Dict[str, Any]) -> Dict[str, Any]:
 
 
 # 对去重数据库进行管理
-async def cache_db_manage(conn: Connection) -> None:
+def cache_db_manage(conn: Connection) -> None:
     cursor = conn.cursor()
     # 用来去重的 sqlite3 数据表如果不存在就创建一个
     cursor.execute(
@@ -62,7 +62,7 @@ async def cache_db_manage(conn: Connection) -> None:
 
 
 # 对缓存 json 进行管理
-async def cache_json_manage(db: TinyDB, new_data_length: int) -> None:
+def cache_json_manage(db: TinyDB, new_data_length: int) -> None:
     # 只保留最多 config.limit + new_data_length 条的记录
     limit = config.limit + new_data_length
     retains = db.all()
@@ -116,7 +116,7 @@ async def duplicate_exists(
         if mode == "link":
             sql += " AND link=?"
             args.append(link)
-        if mode == "title":
+        elif mode == "title":
             sql += " AND title=?"
             args.append(title)
     if "or" in rss.duplicate_filter_mode:
@@ -136,7 +136,7 @@ async def duplicate_exists(
 
 
 # 消息发送后存入去重数据库
-async def insert_into_cache_db(
+def insert_into_cache_db(
     conn: Connection, item: Dict[str, Any], image_hash: str
 ) -> None:
     cursor = conn.cursor()

src/plugins/ELF_RSS2/RSS/routes/Parsing/check_update.py

@@ -16,7 +16,7 @@ def dict_hash(dictionary: Dict[str, Any]) -> str:
 
 
 # 检查更新
-async def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
 
     # 发送失败超过 3 次的消息不再发送
     to_send_list: List[Dict[str, Any]] = db.search(
@@ -40,8 +40,7 @@ async def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str,
 
 
 def get_item_date(item: Dict[str, Any]) -> Arrow:
-    date = item.get("published", item.get("updated"))
-    if date:
+    if date := item.get("published", item.get("updated")):
         try:
             date = parsedate_to_datetime(date)
         except TypeError:

src/plugins/ELF_RSS2/RSS/routes/Parsing/handle_html_tag.py

@@ -3,12 +3,13 @@
 
 import bbcode
 from pyquery import PyQuery as Pq
+from yarl import URL
 
 from ....config import config
 
 
 # 处理 bbcode
-async def handle_bbcode(html: Pq) -> str:
+def handle_bbcode(html: Pq) -> str:
     rss_str = html_unescape(str(html))
 
     # issue 36 处理 bbcode
@@ -43,7 +44,7 @@ async def handle_bbcode(html: Pq) -> str:
 
     # 检查正文是否为 bbcode ，没有成对的标签也当作不是，从而不进行处理
     bbcode_search = re.search(r"\[/(\w+)]", rss_str)
-    if bbcode_search and re.search(rf"\[{bbcode_search.group(1)}", rss_str):
+    if bbcode_search and re.search(f"\\[{bbcode_search[1]}", rss_str):
         parser = bbcode.Parser()
         parser.escape_html = False
         rss_str = parser.format(rss_str)
@@ -52,21 +53,21 @@ async def handle_bbcode(html: Pq) -> str:
 
 
 # HTML标签等处理
-async def handle_html_tag(html: Pq) -> str:
+def handle_html_tag(html: Pq) -> str:
     rss_str = html_unescape(str(html))
 
     # 有序/无序列表 标签处理
     for ul in html("ul").items():
         for li in ul("li").items():
             li_str_search = re.search("<li>(.+)</li>", repr(str(li)))
             rss_str = rss_str.replace(
-                str(li), f"\n- {li_str_search.group(1)}"  # type: ignore
+                str(li), f"\n- {li_str_search[1]}"  # type: ignore
             ).replace("\\n", "\n")
     for ol in html("ol").items():
         for index, li in enumerate(ol("li").items()):
             li_str_search = re.search("<li>(.+)</li>", repr(str(li)))
             rss_str = rss_str.replace(
-                str(li), f"\n{index + 1}. {li_str_search.group(1)}"  # type: ignore
+                str(li), f"\n{index + 1}. {li_str_search[1]}"  # type: ignore
             ).replace("\\n", "\n")
     rss_str = re.sub("</(ul|ol)>", "\n", rss_str)
     # 处理没有被 ul / ol 标签包围的 li 标签
@@ -85,11 +86,17 @@ async def handle_html_tag(html: Pq) -> str:
             ):
                 rss_str = rss_str.replace(a_str, "")
             # 去除微博话题对应链接 及 微博用户主页链接，只保留文本
-            elif ("weibo.cn" in a.attr("href") and a.children("span.surl-text")) or (
-                "weibo.com" in a.attr("href") and a.text().startswith("@")
+            elif (
+                a.attr("href").startswith("https://m.weibo.cn/search?containerid=")
+                and re.search("#.+#", a.text())
+            ) or (
+                a.attr("href").startswith("https://weibo.com/n/")
+                and a.text().startswith("@")
             ):
                 rss_str = rss_str.replace(a_str, a.text())
             else:
+                if a.attr("href").startswith("https://weibo.cn/sinaurl?u="):
+                    a.attr("href", URL(a.attr("href")).query["u"])
                 rss_str = rss_str.replace(a_str, f" {a.text()}: {a.attr('href')}\n")
         else:
             rss_str = rss_str.replace(a_str, f" {a.attr('href')}\n")
@@ -150,6 +157,6 @@ async def handle_html_tag(html: Pq) -> str:
     rss_str = rss_str.strip()
 
     if 0 < config.max_length < len(rss_str):
-        rss_str = rss_str[: config.max_length] + "..."
+        rss_str = f"{rss_str[: config.max_length]}..."
 
     return rss_str

src/plugins/ELF_RSS2/RSS/routes/Parsing/handle_images.py

@@ -132,7 +132,7 @@ async def zip_pic(url: str, content: bytes) -> Union[Image.Image, bytes, None]:
 
 
 # 将图片转化为 base64
-async def get_pic_base64(content: Union[Image.Image, bytes, None]) -> str:
+def get_pic_base64(content: Union[Image.Image, bytes, None]) -> str:
     if not content:
         return ""
     if isinstance(content, Image.Image):
@@ -212,18 +212,16 @@ async def handle_img_combo(url: str, img_proxy: bool) -> str:
     content = await download_image(url, img_proxy)
     if content:
         resize_content = await zip_pic(url, content)
-        img_base64 = await get_pic_base64(resize_content)
-        if img_base64:
+        if img_base64 := get_pic_base64(resize_content):
             return f"[CQ:image,file=base64://{img_base64}]"
     return f"\n图片走丢啦: {url}\n"
 
 
 async def handle_img_combo_with_content(gif_url: str, content: bytes) -> str:
     resize_content = await zip_pic(gif_url, content)
-    img_base64 = await get_pic_base64(resize_content)
-    if img_base64:
+    if img_base64 := get_pic_base64(resize_content):
         return f"[CQ:image,file=base64://{img_base64}]"
-    return f"\n图片走丢啦\n"
+    return "\n图片走丢啦\n"
 
 
 # 处理图片、视频
@@ -245,8 +243,7 @@ async def handle_img(item: Dict[str, Any], img_proxy: bool, img_num: int) -> str
         img_str += await handle_img_combo(url, img_proxy)
 
     # 处理视频
-    doc_video = html("video")
-    if doc_video:
+    if doc_video := html("video"):
         img_str += "\n视频封面："
         for video in doc_video.items():
             url = video.attr("poster")

src/plugins/ELF_RSS2/RSS/routes/Parsing/handle_translation.py

@@ -46,9 +46,7 @@ async def handle_translation(content: str) -> str:
                 )
                 data = await resp.json()
                 try:
-                    content = ""
-                    for i in data["trans_result"]:
-                        content += i["dst"] + "\n"
+                    content = "".join(i["dst"] + "\n" for i in data["trans_result"])
                     text = "\n百度翻译：\n" + content[:-1]
                 except Exception:
                     logger.warning(f"使用百度翻译错误：{data['error_msg']}，开始尝试使用谷歌翻译")

src/plugins/ELF_RSS2/RSS/routes/Parsing/send_message.py

@@ -27,7 +27,7 @@ async def send_msg(rss: Rss, msg: str, item: Dict[str, Any]) -> bool:
                 )
                 continue
             try:
-                await bot.send_private_msg(user_id=int(user_id), message=str(msg))
+                await bot.send_private_msg(user_id=int(user_id), message=msg)
                 flag = True
             except Exception as e:
                 logger.error(f"E: {repr(e)} 链接：[{item.get('link')}]")
@@ -45,7 +45,7 @@ async def send_msg(rss: Rss, msg: str, item: Dict[str, Any]) -> bool:
                 )
                 continue
             try:
-                await bot.send_group_msg(group_id=int(group_id), message=str(msg))
+                await bot.send_group_msg(group_id=int(group_id), message=msg)
                 flag = True
             except Exception as e:
                 logger.error(f"E: {repr(e)} 链接：[{item.get('link')}]")
@@ -80,7 +80,7 @@ async def send_msg(rss: Rss, msg: str, item: Dict[str, Any]) -> bool:
 
             try:
                 await bot.send_guild_channel_msg(
-                    message=str(msg), guild_id=guild_id, channel_id=channel_id
+                    message=msg, guild_id=guild_id, channel_id=channel_id
                 )
                 flag = True
             except Exception as e:

src/plugins/ELF_RSS2/RSS/routes/danbooru.py

@@ -66,16 +66,15 @@ async def handle_img(item: Dict[str, Any], img_proxy: bool) -> str:
     async with aiohttp.ClientSession() as session:
         resp = await session.get(item["link"], proxy=get_proxy(img_proxy))
         d = Pq(await resp.text())
-        img = d("img#image")
-        if img:
+        if img := d("img#image"):
             url = img.attr("src")
         else:
             img_str += "视频预览："
             url = d("video#image").attr("src")
             try:
                 url = await get_preview_gif_from_video(url)
             except RetryError:
-                logger.warning(f"视频预览获取失败，将发送原视频封面")
+                logger.warning("视频预览获取失败，将发送原视频封面")
                 url = d("meta[property='og:image']").attr("content")
         img_str += await handle_img_combo(url, img_proxy)
 
@@ -97,7 +96,7 @@ async def handle_check_update(rss: Rss, state: Dict[str, Any]) -> Dict[str, Any]
         conn = sqlite3.connect(str(DATA_PATH / "cache.db"))
         conn.set_trace_callback(logger.debug)
 
-    await cache_db_manage(conn)
+    cache_db_manage(conn)
 
     delete = []
     for index, item in enumerate(change_data):
@@ -139,8 +138,7 @@ async def get_summary(item: Dict[str, Any], img_proxy: bool) -> str:
     async with aiohttp.ClientSession() as session:
         resp = await session.get(item["link"], proxy=get_proxy(img_proxy))
         d = Pq(await resp.text())
-        img = d("img#image")
-        if img:
+        if img := d("img#image"):
             summary_doc("img").attr("src", img.attr("src"))
         else:
             summary_doc.remove("img")

src/plugins/ELF_RSS2/RSS/routes/nga.py

@@ -17,12 +17,12 @@ async def handle_check_update(rss: Rss, state: Dict[str, Any]) -> Dict[str, Any]
     for i in new_data:
         i["link"] = re.sub(r"&rand=\d+", "", i["link"])
 
-    change_data = await check_update(db, new_data)
+    change_data = check_update(db, new_data)
     return {"change_data": change_data}
 
 
 # 检查更新
-async def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
 
     # 发送失败超过 3 次的消息不再发送
     to_send_list: List[Dict[str, Any]] = db.search(

src/plugins/ELF_RSS2/RSS/routes/pixiv.py

@@ -40,7 +40,7 @@ async def handle_check_update(rss: Rss, state: Dict[str, Any]) -> Dict[str, Any]
         conn = sqlite3.connect(str(DATA_PATH / "cache.db"))
         conn.set_trace_callback(logger.debug)
 
-    await cache_db_manage(conn)
+    cache_db_manage(conn)
 
     delete = []
     for index, item in enumerate(change_data):
@@ -174,12 +174,12 @@ async def handle_source(
 @ParsingBase.append_before_handler(rex="pixiv/ranking", priority=10)
 async def handle_check_update(rss: Rss, state: Dict[str, Any]) -> Dict[str, Any]:
     db = state["tinydb"]
-    change_data = await check_update(db, state["new_data"])
+    change_data = check_update(db, state["new_data"])
     return {"change_data": change_data}
 
 
 # 检查更新
-async def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def check_update(db: TinyDB, new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
 
     # 发送失败超过 3 次的消息不再发送
     to_send_list: List[Dict[str, Any]] = db.search(