Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pull] master from diygod:master #845

Merged
merged 6 commits into from
Jul 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/en/programming.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ For instance, the `/github/topics/framework/l=php&o=desc&s=stars` route will gen

### Issue / Pull Request comments

<RouteEn author="TonyRL" example="/github/comments/DIYgod/RSSHub/8116" path="/github/comments/:user/:repo/:number" :paramsDesc="['User / Org name', 'Repo name', 'Issue or pull number']" radar="1" rssbud="1"/>
<RouteEn author="TonyRL FliegendeWurst" example="/github/comments/DIYgod/RSSHub/8116" path="/github/comments/:user/:repo/:number?" :paramsDesc="['User / Org name', 'Repo name', 'Issue or pull number (if omitted: all)']" radar="1" rssbud="1"/>

### Wiki History

Expand Down
2 changes: 1 addition & 1 deletion docs/programming.md
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ GitHub 官方也提供了一些 RSS:

### Issue / Pull Request 评论

<Route author="TonyRL" example="/github/comments/DIYgod/RSSHub/8116" path="/github/comments/:user/:repo/:number" :paramsDesc="['用户名', '仓库名', 'Issue 或 pull 编号']" radar="1" rssbud="1"/>
<Route author="TonyRL FliegendeWurst" example="/github/comments/DIYgod/RSSHub/8116" path="/github/comments/:user/:repo/:number?" :paramsDesc="['用户名', '仓库名', 'Issue 或 pull 编号']" radar="1" rssbud="1"/>

### Wiki 历史

Expand Down
7 changes: 7 additions & 0 deletions docs/university.md
Original file line number Diff line number Diff line change
Expand Up @@ -3669,6 +3669,13 @@ jsjxy.hbut.edu.cn 证书链不全,自建 RSSHub 可设置环境变量 NODE_TLS

<Route author="3401797899" example="/ouc/jwc" path="/ouc/jwc" radar="1"/>

### 选课信息教务通知

<Route author="3401797899" example="/ouc/jwgl" path="/ouc/jwgl" radar="1" selfhost="1"/>
::: warning 注意
由于选课通知仅允许校园网访问,需自行部署。
:::

## 中国科学技术大学

### 官网通知公告
Expand Down
43 changes: 39 additions & 4 deletions lib/v2/bloomberg/react-renderer.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,39 @@
const art = require('art-template');
const path = require('path');
const { processVideo } = require('./utils');
const got = require('@/utils/got');

const headers = {
accept: 'application/json',
'cache-control': 'no-cache',
referer: 'https://www.bloomberg.com',
};

const processVideo = async (bmmrId, summary) => {
const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
const res = await got(api, { headers });

// Blocked by PX3, return the default
const redirectUrls = res.redirectUrls.map(String);
if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
return {
stream: '',
mp4: '',
coverUrl: '',
caption: summary,
};
}

if (res.data) {
const video_json = res.data;
return {
stream: video_json.streams ? video_json.streams[0]?.url : '',
mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
coverUrl: video_json.thumbnail?.baseUrl ?? '',
caption: video_json.description || video_json.title || summary,
};
}
return {};
};

const nodeRenderers = {
paragraph: async (node, nextNode) => `<p>${await nextNode(node.content)}</p>`,
Expand Down Expand Up @@ -128,9 +161,9 @@ const nodeRenderers = {
}
if (t === 'video') {
const h = node.data;
const id = h.attachment && h.attachment.id;
const v = await processVideo(id);
return v;
const id = h.attachment?.id;
const desc = await processVideo(id, h.attachment?.title);
return art(path.join(__dirname, 'templates/video_media.art'), desc);
}
if (t === 'audio' && node.data.attachment) {
const B = node.data.attachment;
Expand Down Expand Up @@ -187,4 +220,6 @@ const documentToHtmlString = async (document) => {

module.exports = {
documentToHtmlString,
processVideo,
headers,
};
44 changes: 8 additions & 36 deletions lib/v2/bloomberg/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const { parseDate } = require('@/utils/parse-date');
const got = require('@/utils/got');
const { art } = require('@/utils/render');

const { documentToHtmlString } = require('./react-renderer');
const { documentToHtmlString, processVideo, headers } = require('./react-renderer');

const rootUrl = 'https://www.bloomberg.com/feeds';
const sel = 'script[data-component-props="ArticleBody"], script[data-component-props="FeatureBody"]';
Expand Down Expand Up @@ -40,11 +40,6 @@ const apiEndpoints = {
sel: 'script#__SSR_DATA__',
},
};
const headers = {
accept: 'application/json',
'cache-control': 'no-cache',
referer: 'https://www.bloomberg.com',
};

const pageTypeRegex1 = /\/(?<page>[\w-]*?)\/(?<link>\d{4}-\d{2}-\d{2}\/.*)/;
const pageTypeRegex2 = /(?<!news|politics)\/(?<page>features\/|graphics\/)(?<link>.*)/;
Expand Down Expand Up @@ -84,7 +79,7 @@ const parseArticle = (item, ctx) =>
if (group) {
const { page, link } = group;
if (apiEndpoints[page]) {
const api = apiEndpoints[page];
const api = { ...apiEndpoints[page] };
let res;

try {
Expand All @@ -95,6 +90,7 @@ const parseArticle = (item, ctx) =>
if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
try {
res = await got(item.link, { headers });
api.useOrigLink = true;
} catch (err) {
// return the default one
return {
Expand All @@ -106,7 +102,7 @@ const parseArticle = (item, ctx) =>
}
}

// Blocked by PX3, or 404 by api, return the default
// Blocked by PX3, or 404 by both api and direct link, return the default
const redirectUrls = res.redirectUrls.map(String);
if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html') || res.statusCode === 404) {
return {
Expand Down Expand Up @@ -257,11 +253,15 @@ const parseFeaturePage = async (res, api, item) => {
};

const parseOtherPage = async function (res, api, item) {
if (api.useOrigLink) {
return parseNewsletterPage(res, apiEndpoints.newsletters, item);
}
const article_json = JSON.parse(
cheerio
.load(res.data.html ?? res.data)(api.sel)
.html()
);

const story_json = article_json.story;
const body_html = story_json.body;
const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.baseUrl;
Expand Down Expand Up @@ -332,33 +332,6 @@ const processLedeMedia = async (story_json) => {
}
};

const processVideo = async (bmmrId, summary) => {
const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
const res = await got(api, { headers });

// Blocked by PX3, return the default
const redirectUrls = res.redirectUrls.map(String);
if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
return {
stream: '',
mp4: '',
coverUrl: '',
caption: summary,
};
}

if (res.data) {
const video_json = res.data;
return {
stream: video_json.streams ? video_json.streams[0]?.url : '',
mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
coverUrl: video_json.thumbnail?.baseUrl ?? '',
caption: video_json.description || video_json.title || summary,
};
}
return {};
};

const processBody = async (body_html, story_json) => {
const removeSel = ['meta', 'script', '*[class$="-footnotes"]', '*[class$="for-you"]', '*[class$="-newsletter"]', '*[class$="page-ad"]', '*[class$="-recirc"]', '*[data-ad-placeholder="Advertisement"]'];

Expand Down Expand Up @@ -436,5 +409,4 @@ module.exports = {
asyncPoolAll,
parseNewsList,
parseArticle,
processVideo,
};
68 changes: 65 additions & 3 deletions lib/v2/github/comments.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@ const { parseDate } = require('@/utils/parse-date');
const md = require('markdown-it')({
html: true,
});
const rootUrl = 'https://github.com';
const apiUrl = 'https://api.github.com';
const config = require('@/config').value;
const typeDict = {
issue: {
title: 'Issue',
},
issues: {
title: 'Issue',
},
pull: {
title: 'Pull request',
},
Expand All @@ -17,8 +21,8 @@ const typeDict = {
module.exports = async (ctx) => {
const user = ctx.params.user;
const repo = ctx.params.repo;
const number = isNaN(parseInt(ctx.params.number)) ? 1 : parseInt(ctx.params.number);
const limit = ctx.query.limit ? parseInt(ctx.params.limit) : 100;
const number = ctx.params.number && isNaN(parseInt(ctx.params.number)) ? 1 : parseInt(ctx.params.number);
const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 100;
const headers =
config.github && config.github.access_token
? {
Expand All @@ -29,6 +33,64 @@ module.exports = async (ctx) => {
Accept: 'application/vnd.github.v3+json',
};

if (isNaN(number)) {
await allIssues(ctx, user, repo, limit, headers);
} else {
await singleIssue(ctx, user, repo, number, limit, headers);
}
};

async function allIssues(ctx, user, repo, limit, headers) {
const response = await got(`${apiUrl}/repos/${user}/${repo}/issues/comments`, {
headers,
searchParams: {
sort: 'updated',
direction: 'desc',
per_page: limit,
},
});

const timeline = response.data;

const items = timeline.map((item) => {
const actor = item.actor?.login ?? item.user?.login ?? 'ghost';
const issueUrlParts = item.issue_url.split('/');
const issue = issueUrlParts[issueUrlParts.length - 1];
const urlParts = item.html_url.split('/');
const issueType = typeDict[urlParts[urlParts.length - 2]].title;

return {
title: `${actor} commented on ${user}/${repo}: ${issueType} #${issue}`,
author: actor,
pubDate: parseDate(item.created_at),
link: item.html_url,
description: item.body ? md.render(item.body) : null,
};
});

const rateLimit = {
limit: parseInt(response.headers['x-ratelimit-limit']),
remaining: parseInt(response.headers['x-ratelimit-remaining']),
reset: parseDate(parseInt(response.headers['x-ratelimit-reset']) * 1000),
resoure: response.headers['x-ratelimit-resource'],
used: parseInt(response.headers['x-ratelimit-used']),
};

ctx.state.data = {
title: `${user}/${repo}: Issue & Pull request comments`,
link: `${rootUrl}/${user}/${repo}`,
item: items,
};

ctx.state.json = {
title: `${user}/${repo}: Issue & Pull request comments`,
link: `${rootUrl}/${user}/${repo}`,
item: items,
rateLimit,
};
}

async function singleIssue(ctx, user, repo, number, limit, headers) {
const response = await got(`${apiUrl}/repos/${user}/${repo}/issues/${number}`, {
headers,
});
Expand Down Expand Up @@ -124,4 +186,4 @@ module.exports = async (ctx) => {
used: parseInt(response.headers['x-ratelimit-used']),
},
};
};
}
2 changes: 1 addition & 1 deletion lib/v2/github/maintainer.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module.exports = {
'/branches/:user/:repo': ['max-arnold'],
'/comments/:user/:repo/:number': ['TonyRL'],
'/comments/:user/:repo/:number?': ['TonyRL', 'FliegendeWurst'],
'/contributors/:user/:repo/:order?/:anon?': ['zoenglinghou'],
'/file/:user/:repo/:branch/:filepath+': ['zengxs'],
'/gist/:gistId': ['TonyRL'],
Expand Down
6 changes: 6 additions & 0 deletions lib/v2/github/radar.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ module.exports = {
source: ['/:user/:repo/:type/:number'],
target: '/github/comments/:user/:repo/:number',
},
{
title: 'Issue & Pull Request comments',
docs: 'https://docs.rsshub.app/programming.html#github',
source: ['/:user/:repo/:type'],
target: '/github/comments/:user/:repo',
},
{
title: '仓库 Contributors',
docs: 'https://docs.rsshub.app/programming.html#github',
Expand Down
2 changes: 1 addition & 1 deletion lib/v2/github/router.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
module.exports = function (router) {
router.get('/branches/:user/:repo', require('./branches'));
router.get('/comments/:user/:repo/:type/:number', require('./comments')); // deprecated
router.get('/comments/:user/:repo/:number', require('./comments'));
router.get('/comments/:user/:repo/:number?', require('./comments'));
router.get('/contributors/:user/:repo/:order?/:anon?', require('./contributors'));
router.get('/file/:user/:repo/:branch/:filepath+', require('./file'));
router.get('/gist/:gistId', require('./gist'));
Expand Down
42 changes: 42 additions & 0 deletions lib/v2/ouc/jwgl.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');

module.exports = async (ctx) => {
const link = 'http://jwgl.ouc.edu.cn/public/listSchoolNotices.action?currentPage=1&recordsPerPage=15&qtitle=';
const response = await got(link);
const $ = cheerio.load(response.data);
const list = $('div.datalist table tbody tr')
.toArray()
.map((e) => {
e = $(e);
const noticeId = e
.find('a')
.attr('onclick')
.match(/viewNotice\('(.+?)'\)/)[1];
const tds = e.find('td');
return {
title: tds.eq(2).text(),
link: 'http://jwgl.ouc.edu.cn/public/viewSchoolNoticeDetail.action?schoolNoticeId=' + noticeId,
pubDate: parseDate(tds.eq(3).text(), 'YYYY-MM-DD HH:mm'),
};
});

const out = await Promise.all(
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const response = await got(item.link);
const $ = cheerio.load(response.data);
item.description = $('div.notice').html();
return item;
})
)
);

ctx.state.data = {
title: '中国海洋大学选课信息教务通知',
link,
description: '中国海洋大学选课信息教务通知',
item: out,
};
};
1 change: 1 addition & 0 deletions lib/v2/ouc/maintainer.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ module.exports = {
'/it/postgraduate': ['shengmaosu'],
'/it/:type?': ['GeoffreyChen777'],
'/jwc': ['3401797899'],
'/jwgl': ['3401797899'],
'/yjs': ['shengmaosu'],
};
Loading
Loading