From 166661fb624357af04133bc78c86c28933ee639b Mon Sep 17 00:00:00 2001 From: Li Xuetao Date: Tue, 8 Aug 2023 03:49:06 +0800 Subject: [PATCH 1/4] fix(route/ximalaya): categoryInfo is undefined (#12940) * fix(route/ximalaya): categoryInfo is undefined * fix(route/ximalaya): categoryInfo is undefined * refactor: migrate to v2 --------- --- assets/radar-rules.js | 15 ----------- lib/radar-rules.js | 15 ----------- lib/router.js | 4 +-- lib/{routes => v2}/ximalaya/album.js | 39 ++++++++++++++-------------- lib/v2/ximalaya/maintainer.js | 4 +++ lib/v2/ximalaya/radar.js | 17 ++++++++++++ lib/v2/ximalaya/router.js | 4 +++ lib/{routes => v2}/ximalaya/utils.js | 11 ++++---- 8 files changed, 53 insertions(+), 56 deletions(-) rename lib/{routes => v2}/ximalaya/album.js (82%) create mode 100644 lib/v2/ximalaya/maintainer.js create mode 100644 lib/v2/ximalaya/radar.js create mode 100644 lib/v2/ximalaya/router.js rename lib/{routes => v2}/ximalaya/utils.js (89%) diff --git a/assets/radar-rules.js b/assets/radar-rules.js index 4eceeeb24f841c..64ebc195579189 100644 --- a/assets/radar-rules.js +++ b/assets/radar-rules.js @@ -27,21 +27,6 @@ { title: '仓库 Contributors', docs: 'https://docs.rsshub.app/programming.html#github', source: ['/:user/:repo/graphs/contributors', '/:user/:repo'], target: '/github/contributors/:user/:repo' }, ], }, - 'ximalaya.com': { - _name: '喜马拉雅', - '.': [ - { - title: '专辑', - docs: 'https://docs.rsshub.app/multimedia.html#xi-ma-la-ya', - source: '/:type/:id', - target: (params) => { - if (parseInt(params.id) + '' === params.id) { - return '/ximalaya/:type/:id/'; - } - }, - }, - ], - }, 'algocasts.io': { _name: 'AlgoCasts', '.': [{ title: '视频更新', docs: 'https://docs.rsshub.app/programming.html#algocasts', source: '/episodes', target: '/algocasts' }] }, 'soulapp.cn': { _name: 'Soul', '.': [{ title: '瞬间更新', docs: 'https://docs.rsshub.app/social-media.html#soul' }] }, 'anime1.me': { diff --git a/lib/radar-rules.js b/lib/radar-rules.js index a6108a586e36de..be8d25579424c0 100644 --- a/lib/radar-rules.js +++ b/lib/radar-rules.js @@ -1,19 +1,4 @@ module.exports = { - 'ximalaya.com': { - _name: '喜马拉雅', - '.': [ - { - title: '专辑', - docs: 'https://docs.rsshub.app/multimedia.html#xi-ma-la-ya', - source: '/:type/:id', - target: (params) => { - if (parseInt(params.id) + '' === params.id) { - return '/ximalaya/:type/:id/'; - } - }, - }, - ], - }, 'algocasts.io': { _name: 'AlgoCasts', '.': [ diff --git a/lib/router.js b/lib/router.js index 11524bf9612abc..1e91db8ab63b09 100644 --- a/lib/router.js +++ b/lib/router.js @@ -246,8 +246,8 @@ router.get('/yande.re/post/popular_recent/:period', lazyloadRouteHandler('./rout // router.get('/3dm/news', lazyloadRouteHandler('./routes/3dm/news_center')); // 喜马拉雅 -router.get('/ximalaya/:type/:id/:all?', lazyloadRouteHandler('./routes/ximalaya/album')); -router.get('/ximalaya/:type/:id/:all/:shownote?', lazyloadRouteHandler('./routes/ximalaya/album')); +// router.get('/ximalaya/:type/:id/:all?', lazyloadRouteHandler('./routes/ximalaya/album')); +// router.get('/ximalaya/:type/:id/:all/:shownote?', lazyloadRouteHandler('./routes/ximalaya/album')); // EZTV router.get('/eztv/torrents/:imdb_id', lazyloadRouteHandler('./routes/eztv/imdb')); diff --git a/lib/routes/ximalaya/album.js b/lib/v2/ximalaya/album.js similarity index 82% rename from lib/routes/ximalaya/album.js rename to lib/v2/ximalaya/album.js index 066ab3e9c7bb27..197d2d11440b73 100644 --- a/lib/routes/ximalaya/album.js +++ b/lib/v2/ximalaya/album.js @@ -3,6 +3,7 @@ const cheerio = require('cheerio'); const { getUrl, getRandom16 } = require('./utils'); const baseUrl = 'https://www.ximalaya.com'; const config = require('@/config').value; +const { parseDate } = require('@/utils/parse-date'); // Find category from: https://help.apple.com/itc/podcasts_connect/?lang=en#/itc9267a2f12 const categoryDict = { @@ -46,8 +47,8 @@ async function parseAlbumData(category, album_id) { // 解析网页版的 HTML 获取该专辑的信息 // 这里的 {category} 并不重要，系统会准确的返回该 id 对应专辑的信息 // 现在 {category} 必须要精确到大的类别 - // https://www.ximalaya.com/{category}/{album_id}/ - const response = await got(`https://www.ximalaya.com/${category}/${album_id}/`); + // https://www.ximalaya.com/{category}/{album_id} + const response = await got(`${baseUrl}/${category}/${album_id}`); const data = response.body; const $ = cheerio.load(data); const stateElement = $('script') @@ -87,25 +88,27 @@ module.exports = async (ctx) => { const albumData = await parseAlbumData(type, id); - const isPaid = albumData.store.AlbumDetailPage.albumPageMainInfo.isPaid; + const { albumPageMainInfo, currentCategory, anchorInfo } = albumData?.store?.AlbumDetailPage || {}; - const authorInfo = albumData.store.AlbumDetailPage.anchorInfo; // 作者数据 + const isPaid = albumPageMainInfo.isPaid; + + const authorInfo = anchorInfo; // 作者数据 const author = authorInfo.nickName; - const albumInfo = albumData.store.AlbumDetailPage.albumPageMainInfo; // 专辑数据 + const albumInfo = albumPageMainInfo; // 专辑数据 const albumTitle = albumInfo.albumTitle; // 专辑标题 const albumCover = 'http:' + albumInfo.cover.split('!')[0]; const albumIntro = albumInfo.detailRichIntro; // 专辑介绍 - const categoryInfo = albumData.store.AlbumDetailPage.currentCategory; - const albumCategory = categoryInfo.categoryTitle; // 专辑分类名字 - const albumUrl = '/' + categoryInfo.categoryPinyin + '/' + id + '/'; + const categoryInfo = currentCategory; + const albumCategory = categoryInfo?.categoryTitle || albumPageMainInfo.categoryTitle; // 专辑分类名字 + const albumUrl = '/' + (categoryInfo?.categoryPinyin || type) + '/' + id + '/'; // 没有 categoryPinyin 这个字段了，用 type 兼容一下 // sort 为 1 时是降序 // const isAsc = albumData.store.AlbumDetailTrackList.sort === 0; // 喜马拉雅的 API 的 query 参数 isAsc=0 时才是升序，不写就是降序。 const trackInfoApi = `http://mobile.ximalaya.com/mobile/v1/album/track/?albumId=${id}&pageSize=${pageSize}&pageId=`; - const trackInfoResponse = await got.get(trackInfoApi + '1'); + const trackInfoResponse = await got(trackInfoApi + '1'); const maxPageId = trackInfoResponse.data.data.maxPageId; // 最大页数 let playList = trackInfoResponse.data.data.list; @@ -114,22 +117,22 @@ module.exports = async (ctx) => { const promises = []; for (let i = 2; i <= maxPageId; i++) { // string + number -> string - promises.push(got.get(trackInfoApi + i)); + promises.push(got(trackInfoApi + i)); } const responses = await Promise.all(promises); - for (let j = 0; j < responses.length; j++) { - playList = playList.concat(responses[j].data.data.list); + for (const j of responses) { + playList = playList.concat(j.data.data.list); } } await Promise.all( playList.map(async (item) => { const link = baseUrl + albumUrl + item.trackId; - item.desc = await ctx.cache.tryGet(shouldShowNote.toString() + 'trackRichInfo' + link, async () => { + item.desc = await ctx.cache.tryGet('ximalaya:' + shouldShowNote.toString() + 'trackRichInfo' + link, async () => { let _desc; if (shouldShowNote) { const trackRichInfoApi = `https://mobile.ximalaya.com/mobile-track/richIntro?trackId=${item.trackId}`; - const trackRichInfoResponse = await got.get(trackRichInfoApi); + const trackRichInfoResponse = await got(trackRichInfoApi); _desc = trackRichInfoResponse.data.richIntro; } if (!_desc) { @@ -146,10 +149,8 @@ module.exports = async (ctx) => { await Promise.all( playList.map(async (item) => { const trackPayInfoApi = `https://mpay.ximalaya.com/mobile/track/pay/${item.trackId}/?device=pc`; - const data = await ctx.cache.tryGet('trackPayInfo' + trackPayInfoApi, async () => { - const trackPayInfoResponse = await got({ - method: 'get', - url: trackPayInfoApi, + const data = await ctx.cache.tryGet('ximalaya:trackPayInfo' + trackPayInfoApi, async () => { + const trackPayInfoResponse = await got(trackPayInfoApi, { headers: { 'user-agent': 'ting_6.7.9(GM1900,Android29)', cookie: `1&_device=android&${randomToken}&6.7.9;1&_token=${token}`, @@ -178,7 +179,7 @@ module.exports = async (ctx) => { const trackId = item.trackId; const itunesItemImage = item.coverLarge ? item.coverLarge : albumCover; const link = baseUrl + albumUrl + trackId; - const pubDate = new Date(item.createdAt).toUTCString(); + const pubDate = parseDate(item.createdAt, 'x'); const duration = item.duration; // 时间长度：单位（秒） const enclosureUrl = item.playPathAacv224 || item.playPathAacv164; diff --git a/lib/v2/ximalaya/maintainer.js b/lib/v2/ximalaya/maintainer.js new file mode 100644 index 00000000000000..8f461073a46694 --- /dev/null +++ b/lib/v2/ximalaya/maintainer.js @@ -0,0 +1,4 @@ +module.exports = { + '/:type/:id/:all?': ['lengthmin', 'jjeejj', 'prnake'], + '/:type/:id/:all/:shownote?': ['lengthmin', 'jjeejj', 'prnake'], +}; diff --git a/lib/v2/ximalaya/radar.js b/lib/v2/ximalaya/radar.js new file mode 100644 index 00000000000000..567dfafc61783f --- /dev/null +++ b/lib/v2/ximalaya/radar.js @@ -0,0 +1,17 @@ +module.exports = { + 'ximalaya.com': { + _name: '喜马拉雅', + '.': [ + { + title: '专辑', + docs: 'https://docs.rsshub.app/multimedia.html#xi-ma-la-ya', + source: '/:type/:id', + target: (params) => { + if (parseInt(params.id) + '' === params.id) { + return '/ximalaya/:type/:id'; + } + }, + }, + ], + }, +}; diff --git a/lib/v2/ximalaya/router.js b/lib/v2/ximalaya/router.js new file mode 100644 index 00000000000000..dfda7356b60644 --- /dev/null +++ b/lib/v2/ximalaya/router.js @@ -0,0 +1,4 @@ +module.exports = (router) => { + router.get('/:type/:id/:all?', require('./album')); + router.get('/:type/:id/:all/:shownote?', require('./album')); +}; diff --git a/lib/routes/ximalaya/utils.js b/lib/v2/ximalaya/utils.js similarity index 89% rename from lib/routes/ximalaya/utils.js rename to lib/v2/ximalaya/utils.js index dc599fe256aac9..1800cfa3b27158 100644 --- a/lib/routes/ximalaya/utils.js +++ b/lib/v2/ximalaya/utils.js @@ -78,11 +78,11 @@ const getParams = (ep) => { let a3 = 0; o = 0; - for (let u = 0; u < a2.length; u++) { + for (const u of a2) { a3 = (a3 + 1) % 256; o = (o + r1[a3]) % 256; [r1[a3], r1[o]] = [r1[o], r1[a3]]; - i += String.fromCharCode(a2[u] ^ r1[(r1[a3] + r1[o]) % 256]); + i += String.fromCharCode(u ^ r1[(r1[a3] + r1[o]) % 256]); } i = i.split('-'); return { @@ -112,10 +112,11 @@ const getPath = (seed, fileId) => { const getUrl = (r) => { const params = getParams(r.ep); - const paramsArray = []; params.duration = r.duration; - Object.keys(params).forEach((key) => params[key] && paramsArray.push(`${key}=${params[key]}`)); - const url = 'https://audiopay.cos.xmcdn.com/download/' + r.apiVersion + '/' + getPath(r.seed, r.fileId) + '?' + paramsArray.join('&'); + const paramsArray = Object.keys(params) + .filter((key) => params[key]) + .map((key) => `${key}=${params[key]}`); + const url = `https://audiopay.cos.xmcdn.com/download/${r.apiVersion}/${getPath(r.seed, r.fileId)}?${paramsArray.join('&')}`; return url; }; From 8b3b2e9fa53333831b246b2792702b78bc9e544d Mon Sep 17 00:00:00 2001 From: Tony Date: Tue, 8 Aug 2023 04:51:55 +0800 Subject: [PATCH 2/4] fix(route): zhihu (#12942) --- lib/v2/zhihu/answers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/v2/zhihu/answers.js b/lib/v2/zhihu/answers.js index dd43a0cd08f1e3..65a00cd8730c8b 100644 --- a/lib/v2/zhihu/answers.js +++ b/lib/v2/zhihu/answers.js @@ -20,7 +20,7 @@ module.exports = async (ctx) => { if (name === '知乎用户') { const userProfile = await ctx.cache.tryGet(`zhihu:profile:${id}`, async () => { - const apiPath = `/api/v4/profile/${id}/infinity`; + const apiPath = `/api/v4/members/${id}`; const { data } = await got({ method: 'get', @@ -30,7 +30,7 @@ module.exports = async (ctx) => { Referer: `https://www.zhihu.com/people/${id}`, }, }); - return data.data; + return data; }); name = userProfile.name; } From 291e1271582454f1b2fc972c248d5fcbd48ef55a Mon Sep 17 00:00:00 2001 From: Tony Date: Tue, 8 Aug 2023 05:25:56 +0800 Subject: [PATCH 3/4] fix(route): bilibili vsearch (#12943) --- lib/v2/bilibili/vsearch.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/v2/bilibili/vsearch.js b/lib/v2/bilibili/vsearch.js index fb5f0d1f0bb74c..d9d913b30023b5 100644 --- a/lib/v2/bilibili/vsearch.js +++ b/lib/v2/bilibili/vsearch.js @@ -4,11 +4,12 @@ const { parseDate } = require('@/utils/parse-date'); const utils = require('./utils'); const { CookieJar } = require('tough-cookie'); const cookieJar = new CookieJar(); +const { queryToBoolean } = require('@/utils/readable-social'); module.exports = async (ctx) => { const kw = ctx.params.kw; const order = ctx.params.order || 'pubdate'; - const disableEmbed = ctx.params.disableEmbed; + const disableEmbed = queryToBoolean(ctx.params.disableEmbed); const kw_url = encodeURIComponent(kw); const tids = ctx.params.tid ?? 0; From 656c8bc08f132fd8d701dbb2d1745f92f0f6e376 Mon Sep 17 00:00:00 2001 From: Nishant Singh <57475999+Rjnishant530@users.noreply.github.com> Date: Tue, 8 Aug 2023 08:58:30 +0530 Subject: [PATCH 4/4] feat(route): add dnaindia (#12941) * feat(route): add dnaindia * Update docs/en/traditional-media.md Co-authored-by: Tony * Update docs/en/traditional-media.md Co-authored-by: Tony * Update docs/en/traditional-media.md Co-authored-by: Tony * Update docs/en/traditional-media.md Co-authored-by: Tony * Update docs/en/traditional-media.md Co-authored-by: Tony * Update lib/v2/dnaindia/radar.js Co-authored-by: Tony * Update lib/v2/dnaindia/radar.js Co-authored-by: Tony * Update lib/v2/dnaindia/radar.js Co-authored-by: Tony * Update: change logger and item.category * Update traditional-media.md * Update lib/v2/dnaindia/radar.js Co-authored-by: Tony * Update lib/v2/dnaindia/category.js --------- --- docs/en/traditional-media.md | 30 ++++++++++++++++ lib/v2/dnaindia/category.js | 66 +++++++++++++++++++++++++++++++++++ lib/v2/dnaindia/maintainer.js | 4 +++ lib/v2/dnaindia/radar.js | 19 ++++++++++ lib/v2/dnaindia/router.js | 4 +++ 5 files changed, 123 insertions(+) create mode 100644 lib/v2/dnaindia/category.js create mode 100644 lib/v2/dnaindia/maintainer.js create mode 100644 lib/v2/dnaindia/radar.js create mode 100644 lib/v2/dnaindia/router.js diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md index 463e8969693e26..d6cee7a291d6a7 100644 --- a/docs/en/traditional-media.md +++ b/docs/en/traditional-media.md @@ -242,6 +242,36 @@ Support all channels, refer to [CNBC RSS feeds](https://www.cnbc.com/rss-feeds/) +## DNA India + +### News + + + +Categories: + +| Headlines | Explainer | India | Entertainment | Sports | Viral | Lifestyle | Education | Business | World | +| --------- | --------- | ----- | ------------- | ------ | ----- | --------- | --------- | -------- | ----- | +| headlines | explainer | india | entertainment | sports | viral | lifestyle | education | business | world | + + + +### Topic + + + +Topics: + +|DNA verified| +|------------| +|dna-verified| + +::: tip Topic +See the URL for `https://www.dnaindia.com/topic/dna-verified` for the subdomain `topic` +::: + + + ## Financial Times ### myFT personal RSS diff --git a/lib/v2/dnaindia/category.js b/lib/v2/dnaindia/category.js new file mode 100644 index 00000000000000..c7c54264c9a681 --- /dev/null +++ b/lib/v2/dnaindia/category.js @@ -0,0 +1,66 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); +const logger = require('@/utils/logger'); + +module.exports = async (ctx) => { + const { category, topic } = ctx.params; + const baseUrl = 'https://www.dnaindia.com'; + let route; + if (category) { + route = `/${category}`; + } else if (topic) { + route = `/topic/${topic}`; + } else { + logger.error('Invalid URL'); + } + const { data: response } = await got(`${baseUrl}${route}`); + const $ = cheerio.load(response); + + const listItems = $('div.col-lg-6 div.list-news') + .toArray() + .map((item) => { + item = $(item); + const a = item.find('div.explainer-subtext a'); + return { + title: a.text(), + link: `${baseUrl}${a.attr('href')}`, + }; + }); + + const items = await Promise.all( + listItems.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = cheerio.load(response); + item.itunes_item_image = $('div.article-img img').attr('src'); + item.category = $('div.tags ul li') + .toArray() + .map((item) => $(item).find('a').text()); + const time = $('p.dna-update').text().split('Updated:')[1]; + item.pubDate = timezone(parseDate(time, 'MMMDD,YYYY,hh:mmA'), +5.5); + item.author = 'DNA Web Team'; + item.description = $('div.article-description') + .clone() + .children('div') + .remove() + .end() + .toArray() + .map((element) => $(element).html()) + .join(''); + return item; + }) + ) + ); + + ctx.state.data = { + title: 'DNA India', + link: baseUrl, + item: items, + description: 'Latest News on dnaIndia.com', + logo: 'https://cdn.dnaindia.com/sites/all/themes/dnaindia/favicon-1016.ico', + icon: 'https://cdn.dnaindia.com/sites/all/themes/dnaindia/favicon-1016.ico', + language: 'en-us', + }; +}; diff --git a/lib/v2/dnaindia/maintainer.js b/lib/v2/dnaindia/maintainer.js new file mode 100644 index 00000000000000..047115429102d0 --- /dev/null +++ b/lib/v2/dnaindia/maintainer.js @@ -0,0 +1,4 @@ +module.exports = { + '/:category': ['Rjnishant530'], + '/topic/:topic': ['Rjnishant530'], +}; diff --git a/lib/v2/dnaindia/radar.js b/lib/v2/dnaindia/radar.js new file mode 100644 index 00000000000000..a4edcd2c1700c9 --- /dev/null +++ b/lib/v2/dnaindia/radar.js @@ -0,0 +1,19 @@ +module.exports = { + 'dnaindia.com': { + _name: 'DNA India', + '.': [ + { + title: 'News', + docs: 'https://docs.rsshub.app/en/traditional-media.html#dna-india', + source: ['/:category'], + target: '/dnaindia/:category', + }, + { + title: 'Topic', + docs: 'https://docs.rsshub.app/en/traditional-media.html#dna-india', + source: ['/topic/:topic'], + target: '/dnaindia/topic/:topic', + }, + ], + }, +}; diff --git a/lib/v2/dnaindia/router.js b/lib/v2/dnaindia/router.js new file mode 100644 index 00000000000000..b7dc590d7572e0 --- /dev/null +++ b/lib/v2/dnaindia/router.js @@ -0,0 +1,4 @@ +module.exports = (router) => { + router.get('/:category', require('./category')); + router.get('/topic/:topic', require('./category')); +};