Merge pull request #462 from FloRainRJY/xiaohongshu_comment_number_re…

…strict

All_platform_comments_restrict

config/base_config.py

@@ -11,7 +11,7 @@
 
 # 基础配置
 PLATFORM = "xhs"
-KEYWORDS = "编程副业,编程兼职" # 关键词搜索配置，以英文逗号分隔
+KEYWORDS = "城市" # 关键词搜索配置，以英文逗号分隔
 LOGIN_TYPE = "qrcode"  # qrcode or phone or cookie
 COOKIES = ""
 # 具体值参见media_platform.xxx.field下的枚举值，暂时只支持小红书
@@ -48,7 +48,7 @@
 START_PAGE = 1
 
 # 爬取视频/帖子的数量控制
-CRAWLER_MAX_NOTES_COUNT = 200
+CRAWLER_MAX_NOTES_COUNT = 10
 
 # 并发爬虫数量控制
 MAX_CONCURRENCY_NUM = 1

media_platform/bilibili/client.py

@@ -222,20 +222,23 @@ async def get_video_comments(self,
         return await self.get(uri, post_data)
 
     async def get_video_all_comments(self, video_id: str, crawl_interval: float = 1.0, is_fetch_sub_comments=False,
-                                     callback: Optional[Callable] = None, ):
+                                     callback: Optional[Callable] = None,
+                                     max_count: int = 10,):
         """
         get video all comments include sub comments
         :param video_id:
         :param crawl_interval:
         :param is_fetch_sub_comments:
         :param callback:
+        max_count: 一次笔记爬取的最大评论数量
+
         :return:
         """
 
         result = []
         is_end = False
         next_page = 0
-        while not is_end:
+        while not is_end and len(result) < max_count:
             comments_res = await self.get_video_comments(video_id, CommentOrderType.DEFAULT, next_page)
             cursor_info: Dict = comments_res.get("cursor")
             comment_list: List[Dict] = comments_res.get("replies", [])
@@ -249,6 +252,8 @@ async def get_video_all_comments(self, video_id: str, crawl_interval: float = 1.
                             await self.get_video_all_level_two_comments(
                                 video_id, comment_id, CommentOrderType.DEFAULT, 10, crawl_interval,  callback)
                         }
+            if len(result) + len(comment_list) > max_count:
+                comment_list = comment_list[:max_count - len(result)]
             if callback:  # 如果有回调函数，就执行回调函数
                 await callback(video_id, comment_list)
             await asyncio.sleep(crawl_interval)

media_platform/bilibili/core.py

@@ -182,7 +182,8 @@ async def get_comments(self, video_id: str, semaphore: asyncio.Semaphore):
                     video_id=video_id,
                     crawl_interval=random.random(),
                     is_fetch_sub_comments=config.ENABLE_GET_SUB_COMMENTS,
-                    callback=bilibili_store.batch_update_bilibili_video_comments
+                    callback=bilibili_store.batch_update_bilibili_video_comments,
+                    max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
                 )
 
             except DataFetchError as ex:

media_platform/douyin/client.py

@@ -230,25 +230,29 @@ async def get_aweme_all_comments(
             crawl_interval: float = 1.0,
             is_fetch_sub_comments=False,
             callback: Optional[Callable] = None,
+            max_count: int = 10,
     ):
         """
         获取帖子的所有评论，包括子评论
         :param aweme_id: 帖子ID
         :param crawl_interval: 抓取间隔
         :param is_fetch_sub_comments: 是否抓取子评论
         :param callback: 回调函数，用于处理抓取到的评论
+        :param max_count: 一次帖子爬取的最大评论数量
         :return: 评论列表
         """
         result = []
         comments_has_more = 1
         comments_cursor = 0
-        while comments_has_more:
+        while comments_has_more and len(result) < max_count:
             comments_res = await self.get_aweme_comments(aweme_id, comments_cursor)
             comments_has_more = comments_res.get("has_more", 0)
             comments_cursor = comments_res.get("cursor", 0)
             comments = comments_res.get("comments", [])
             if not comments:
                 continue
+            if len(result) + len(comments) > max_count:
+                comments = comments[:max_count - len(result)]
             result.extend(comments)
             if callback:  # 如果有回调函数，就执行回调函数
                 await callback(aweme_id, comments)

media_platform/douyin/core.py

@@ -179,7 +179,8 @@ async def get_comments(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Non
                     aweme_id=aweme_id,
                     crawl_interval=random.random(),
                     is_fetch_sub_comments=config.ENABLE_GET_SUB_COMMENTS,
-                    callback=douyin_store.batch_update_dy_aweme_comments
+                    callback=douyin_store.batch_update_dy_aweme_comments,
+                    max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES
                 )
                 utils.logger.info(
                     f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} comments have all been obtained and filtered ...")

media_platform/kuaishou/client.py

@@ -189,27 +189,29 @@ async def get_video_all_comments(
         photo_id: str,
         crawl_interval: float = 1.0,
         callback: Optional[Callable] = None,
+        max_count: int = 10,
     ):
         """
         get video all comments include sub comments
         :param photo_id:
         :param crawl_interval:
         :param callback:
+        :param max_count:
         :return:
         """
 
         result = []
         pcursor = ""
 
-        while pcursor != "no_more":
+        while pcursor != "no_more" and len(result) < max_count:
             comments_res = await self.get_video_comments(photo_id, pcursor)
             vision_commen_list = comments_res.get("visionCommentList", {})
             pcursor = vision_commen_list.get("pcursor", "")
             comments = vision_commen_list.get("rootComments", [])
-
+            if len(result) + len(comments) > max_count:
+                comments = comments[:max_count - len(result)]
             if callback:  # 如果有回调函数，就执行回调函数
                 await callback(photo_id, comments)
-
             result.extend(comments)
             await asyncio.sleep(crawl_interval)
             sub_comments = await self.get_comments_all_sub_comments(

media_platform/kuaishou/core.py

@@ -186,7 +186,8 @@ async def get_comments(self, video_id: str, semaphore: asyncio.Semaphore):
                 await self.ks_client.get_video_all_comments(
                     photo_id=video_id,
                     crawl_interval=random.random(),
-                    callback=kuaishou_store.batch_update_ks_video_comments
+                    callback=kuaishou_store.batch_update_ks_video_comments,
+                    max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES
                 )
             except DataFetchError as ex:
                 utils.logger.error(f"[KuaishouCrawler.get_comments] get video_id: {video_id} comment error: {ex}")

media_platform/tieba/client.py

@@ -204,21 +204,23 @@ async def get_note_by_id(self, note_id: str) -> TiebaNote:
         return self._page_extractor.extract_note_detail(page_content)
 
     async def get_note_all_comments(self, note_detail: TiebaNote, crawl_interval: float = 1.0,
-                                    callback: Optional[Callable] = None) -> List[TiebaComment]:
+                                    callback: Optional[Callable] = None,
+                                    max_count: int = 10,
+                                    ) -> List[TiebaComment]:
         """
         获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
         Args:
             note_detail: 帖子详情对象
             crawl_interval: 爬取一次笔记的延迟单位（秒）
             callback: 一次笔记爬取结束后
-
+            max_count: 一次帖子爬取的最大评论数量
         Returns:
 
         """
         uri = f"/p/{note_detail.note_id}"
         result: List[TiebaComment] = []
         current_page = 1
-        while note_detail.total_replay_page >= current_page:
+        while note_detail.total_replay_page >= current_page and len(result) < max_count:
             params = {
                 "pn": current_page
             }
@@ -227,6 +229,8 @@ async def get_note_all_comments(self, note_detail: TiebaNote, crawl_interval: fl
                                                                                 note_id=note_detail.note_id)
             if not comments:
                 break
+            if len(result) + len(comments) > max_count:
+                comments = comments[:max_count - len(result)]
             if callback:
                 await callback(note_detail.note_id, comments)
             result.extend(comments)

media_platform/tieba/core.py

@@ -226,7 +226,8 @@ async def get_comments_async_task(self, note_detail: TiebaNote, semaphore: async
             await self.tieba_client.get_note_all_comments(
                 note_detail=note_detail,
                 crawl_interval=random.random(),
-                callback=tieba_store.batch_update_tieba_note_comments
+                callback=tieba_store.batch_update_tieba_note_comments,
+                max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES
             )
 
     async def get_creators_and_notes(self) -> None:

media_platform/weibo/client.py

@@ -149,23 +149,28 @@ async def get_note_comments(self, mid_id: str, max_id: int) -> Dict:
         return await self.get(uri, params, headers=headers)
 
     async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0,
-                                    callback: Optional[Callable] = None, ):
+                                    callback: Optional[Callable] = None,
+                                    max_count: int = 10,
+                                    ):
         """
         get note all comments include sub comments
         :param note_id:
         :param crawl_interval:
         :param callback:
+        :param max_count:
         :return:
         """
 
         result = []
         is_end = False
         max_id = -1
-        while not is_end:
+        while not is_end and len(result) < max_count:
             comments_res = await self.get_note_comments(note_id, max_id)
             max_id: int = comments_res.get("max_id")
             comment_list: List[Dict] = comments_res.get("data", [])
             is_end = max_id == 0
+            if len(result) + len(comment_list) > max_count:
+                comment_list = comment_list[:max_count - len(result)]
             if callback:  # 如果有回调函数，就执行回调函数
                 await callback(note_id, comment_list)
             await asyncio.sleep(crawl_interval)

media_platform/weibo/core.py

@@ -206,7 +206,8 @@ async def get_note_comments(self, note_id: str, semaphore: asyncio.Semaphore):
                 await self.wb_client.get_note_all_comments(
                     note_id=note_id,
                     crawl_interval=random.randint(1,3), # 微博对API的限流比较严重，所以延时提高一些
-                    callback=weibo_store.batch_update_weibo_note_comments
+                    callback=weibo_store.batch_update_weibo_note_comments,
+                    max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES
                 )
             except DataFetchError as ex:
                 utils.logger.error(f"[WeiboCrawler.get_note_comments] get note_id: {note_id} comment error: {ex}")

media_platform/xhs/client.py

@@ -297,7 +297,7 @@ async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0,
             note_id: 笔记ID
             crawl_interval: 爬取一次笔记的延迟单位（秒）
             callback: 一次笔记爬取结束后
-
+            max_count: 一次笔记爬取的最大评论数量
         Returns:
 
         """