From f99b21547439b5a807ee1db9236566ced3f1bd13 Mon Sep 17 00:00:00 2001 From: RieN 7z Date: Mon, 30 Sep 2024 23:47:37 +0800 Subject: [PATCH 01/14] [feat(routes)] Add ipsw.dev support --- lib/routes/ipswdev/index.ts | 77 +++++++++++++++++++++++++++++++++ lib/routes/ipswdev/namespace.ts | 7 +++ 2 files changed, 84 insertions(+) create mode 100644 lib/routes/ipswdev/index.ts create mode 100644 lib/routes/ipswdev/namespace.ts diff --git a/lib/routes/ipswdev/index.ts b/lib/routes/ipswdev/index.ts new file mode 100644 index 00000000000000..19f1a5b1ba75ef --- /dev/null +++ b/lib/routes/ipswdev/index.ts @@ -0,0 +1,77 @@ +import { Data, Route } from '@/types'; +import got from '@/utils/got'; +import { load } from 'cheerio'; + +const host = 'https://ipsw.dev/'; + +export const route: Route = { + path: '/index/:productID', + categories: ['program-update'], + example: '/ipswdev/index/iPhone16,1', + parameters: { + productID: 'Product ID', + }, + name: 'Apple latest beta firmware', + maintainers: ['RieN7'], + handler, +}; + +async function handler(ctx) { + const { productID } = ctx.req.param(); + const link = `https://ipsw.dev/product/version/${productID}`; + + const resp = await got({ + method: 'get', + url: link, + headers: { + Referer: host, + }, + }); + + const $ = load(resp.data); + + const productName = $('#IdentifierModal > div > div > div.modal-body > p:nth-child(1) > em').text(); + + // eslint-disable-next-line no-restricted-syntax + const list: Data[] = $('.firmware') + .map((index, element) => { + const ele = $(element); + const version = ele.find('td:nth-child(1) > div > div > strong').text(); + const build = ele.find('td:nth-child(1) > div > div > div > code').text(); + const date = ele.find('td:nth-child(3)').text(); + const size = ele.find('td:nth-child(4)').text(); + return { + title: `${productName} - ${version}`, + link: `https://ipsw.dev/download/${productID}/${build}`, + pubDate: new Date(date).toLocaleDateString(), + guid: build, + description: ` + + + + + + + + + + + + + + + + + + +
Version${version}
Build${build}
Released${date}
Size${size}
`, + }; + }) + .get(); + + return { + title: `${productName} Released`, + link, + item: list, + }; +} diff --git a/lib/routes/ipswdev/namespace.ts b/lib/routes/ipswdev/namespace.ts new file mode 100644 index 00000000000000..e6c6cb1ff9b805 --- /dev/null +++ b/lib/routes/ipswdev/namespace.ts @@ -0,0 +1,7 @@ +import { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'IPSW.dev', + url: 'ipsw.dev', + description: 'Download the latest beta firmware for iPhone, iPad, Mac, Apple Vision Pro, and Apple TV. Check the signing status of the beta firmware.', +}; From eab98051398952f2a140ab9711ee219ca36ce64f Mon Sep 17 00:00:00 2001 From: RieN7 Date: Wed, 9 Oct 2024 22:56:45 +0800 Subject: [PATCH 02/14] fix(xiaohongshu): fix note fulltext --- lib/routes/xiaohongshu/notes.ts | 89 +++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index a4d85a5d563425..b04884b6c1aee0 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -1,6 +1,8 @@ import { Route } from '@/types'; import cache from '@/utils/cache'; -import { getNotes, formatText, formatNote } from './util'; +import { config } from '@/config'; +import * as cheerio from 'cheerio'; +import got from '@/utils/got'; export const route: Route = { path: '/user/:user_id/notes/fulltext', @@ -15,11 +17,22 @@ export const route: Route = { handler, example: '/xiaohongshu/user/52d8c541b4c4d60e6c867480/notes/fulltext', features: { + requireConfig: [ + { + name: 'XIAOHONGSHU_COOKIE', + optional: true, + description: '小红书 cookie 值，可在浏览器控制台通过`document.cookie`获取。', + }, + ], antiCrawler: true, requirePuppeteer: true, }, parameters: { user_id: 'user id, length 24 characters', + fulltext: { + description: '是否获取全文', + default: '', + }, }, }; @@ -27,13 +40,81 @@ async function handler(ctx) { const userId = ctx.req.param('user_id'); const url = `https://www.xiaohongshu.com/user/profile/${userId}`; - const { user, notes } = await getNotes(url, cache); + const user = await getUser(url, config.xiaohongshu.cookie); + const notes = await renderNotesFulltext(user.notes, url); return { title: `${user.nickname} - 笔记 • 小红书 / RED`, - description: formatText(user.desc), + description: user.desc, image: user.imageb || user.images, link: url, - item: notes.map((item) => formatNote(url, item)), + item: notes, }; } + +async function getUser(url, cookie) { + const res = await got(url, { + headers: { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + "Cookie": cookie, + } + }); + const $ = cheerio.load(res.data); + + let script = $('script').filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }).text(); + script = script.slice('window.__INITIAL_STATE__='.length); + script = script.replace(/undefined/g, 'null'); + const state = JSON.parse(script); + return state.user; +} + +async function renderNotesFulltext(notes, url) { + const data: any[] = []; + for (const note of notes) { + for (const {noteCard} of note) { + const link = `${url}/${noteCard.noteId}`; + const {title, description} = await getNote(link); + data.push({ + title, + link, + description, + author: noteCard.user.nickname, + guid: noteCard.noteId, + }); + } + } + return data; +} + +async function getNote(link) { + const cookie = config.xiaohongshu.cookie; + const data = await cache.tryGet(link, async () => { + const res = await got(link, { + headers: { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + "Cookie": cookie, + } as any + }); + const $ = cheerio.load(res.data); + let script = $('script').filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }).text(); + script = script.slice('window.__INITIAL_STATE__='.length); + script = script.replace(/undefined/g, 'null'); + const state = JSON.parse(script); + const note = state.note.noteDetailMap[state.note.firstNoteId].note; + const images = note.imageList.map((image) => image.urlDefault); + const title = note.title; + const desc = note.desc.replaceAll('#(.*?)\[话题\]#', '#$1'); + const description = `${images.map((image) => ``).join('')}
${title}
${desc}`; + return { + title, + description, + }; + }) as Promise<{title: string, description: string}>; + return data; +} \ No newline at end of file From 797f6e5b0ba7ae0820dcb3f4cced71b5a74c6a0f Mon Sep 17 00:00:00 2001 From: RieN7 Date: Wed, 9 Oct 2024 23:59:17 +0800 Subject: [PATCH 03/14] add config --- lib/config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/config.ts b/lib/config.ts index e2e0e162f8794f..10d89d3e3c7819 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -309,6 +309,9 @@ export type Config = { device_id?: string; refresh_token?: string; }; + xiaohongshu: { + cookie?: string; + }; ximalaya: { token?: string; }; @@ -702,6 +705,9 @@ const calculateValue = () => { device_id: envs.XIAOYUZHOU_ID, refresh_token: envs.XIAOYUZHOU_TOKEN, }, + xiaohongshu: { + cookie: envs.XIAOHONGSHU_COOKIE, + }, ximalaya: { token: envs.XIMALAYA_TOKEN, }, From a5ee72094d61e0e8e3672df7d18f24079b6ee066 Mon Sep 17 00:00:00 2001 From: RieN7 Date: Thu, 10 Oct 2024 22:11:55 +0800 Subject: [PATCH 04/14] feat: add fulltext with cookie --- lib/config.ts | 6 ++ lib/routes/xiaohongshu/notes.ts | 118 +++++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/lib/config.ts b/lib/config.ts index e2e0e162f8794f..10d89d3e3c7819 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -309,6 +309,9 @@ export type Config = { device_id?: string; refresh_token?: string; }; + xiaohongshu: { + cookie?: string; + }; ximalaya: { token?: string; }; @@ -702,6 +705,9 @@ const calculateValue = () => { device_id: envs.XIAOYUZHOU_ID, refresh_token: envs.XIAOYUZHOU_TOKEN, }, + xiaohongshu: { + cookie: envs.XIAOHONGSHU_COOKIE, + }, ximalaya: { token: envs.XIMALAYA_TOKEN, }, diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index a4d85a5d563425..9cd5428d4e3974 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -1,6 +1,9 @@ import { Route } from '@/types'; import cache from '@/utils/cache'; -import { getNotes, formatText, formatNote } from './util'; +import { config } from '@/config'; +import * as cheerio from 'cheerio'; +import got from '@/utils/got'; +import { formatNote, formatText, getNotes } from './util'; export const route: Route = { path: '/user/:user_id/notes/fulltext', @@ -15,11 +18,22 @@ export const route: Route = { handler, example: '/xiaohongshu/user/52d8c541b4c4d60e6c867480/notes/fulltext', features: { + requireConfig: [ + { + name: 'XIAOHONGSHU_COOKIE', + optional: true, + description: '小红书 cookie 值，可在浏览器控制台通过`document.cookie`获取。', + }, + ], antiCrawler: true, requirePuppeteer: true, }, parameters: { user_id: 'user id, length 24 characters', + fulltext: { + description: '是否获取全文', + default: '', + }, }, }; @@ -27,13 +41,99 @@ async function handler(ctx) { const userId = ctx.req.param('user_id'); const url = `https://www.xiaohongshu.com/user/profile/${userId}`; - const { user, notes } = await getNotes(url, cache); + if (config.xiaohongshu.cookie && ctx.req.param('fulltext')) { + const user = await getUser(url, config.xiaohongshu.cookie); + const notes = await renderNotesFulltext(user.notes, url); + return { + title: `${user.userPageData.basicInfo.nickname} - 笔记 • 小红书 / RED`, + description: user.userPageData.basicInfo.desc, + image: user.userPageData.basicInfo.imageb || user.userPageData.basicInfo.images, + link: url, + item: notes, + }; + } else { + const { user, notes } = await getNotes(url, cache); + return { + title: `${user.nickname} - 笔记 • 小红书 / RED`, + description: formatText(user.desc), + image: user.imageb || user.images, + link: url, + item: notes.map((item) => formatNote(url, item)), + }; + } +} + +async function getUser(url, cookie) { + const res = await got(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Cookie: cookie, + }, + }); + const $ = cheerio.load(res.data); + + let script = $('script') + .filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }) + .text(); + script = script.slice('window.__INITIAL_STATE__='.length); + script = script.replaceAll('undefined', 'null'); + const state = JSON.parse(script); + return state.user; +} + +async function renderNotesFulltext(notes, url) { + const data: any[] = []; + for (const note of notes) { + for (const { noteCard } of note) { + const link = `${url}/${noteCard.noteId}`; + // eslint-disable-next-line no-await-in-loop + const { title, description } = await getFullNote(link); + data.push({ + title, + link, + description, + author: noteCard.user.nickName, + guid: noteCard.noteId, + }); + } + } + return data; +} - return { - title: `${user.nickname} - 笔记 • 小红书 / RED`, - description: formatText(user.desc), - image: user.imageb || user.images, - link: url, - item: notes.map((item) => formatNote(url, item)), - }; +async function getFullNote(link) { + const cookie = config.xiaohongshu.cookie; + const data = (await cache.tryGet(link, async () => { + const res = await got(link, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Cookie: cookie, + } as any, + }); + const $ = cheerio.load(res.data); + let script = $('script') + .filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }) + .text(); + script = script.slice('window.__INITIAL_STATE__='.length); + script = script.replaceAll('undefined', 'null'); + const state = JSON.parse(script); + const note = state.note.noteDetailMap[state.note.firstNoteId].note; + const images = note.imageList.map((image) => image.urlDefault); + const title = note.title; + let desc = note.desc; + desc = desc.replaceAll(/\[.*?\]/g, ''); + desc = desc.replaceAll(/#(.*?)#/g, '#$1'); + desc = desc.replaceAll('\n', '
'); + const description = `${images.map((image) => ``).join('')}
${title}
${desc}`; + return { + title, + description, + }; + })) as Promise<{ title: string; description: string }>; + return data; } From 25ba2e4c3f7d43efe265c0e4f36c196494f2f118 Mon Sep 17 00:00:00 2001 From: RieN7 Date: Thu, 10 Oct 2024 22:11:55 +0800 Subject: [PATCH 05/14] feat: add fulltext with cookie --- lib/config.ts | 6 ++ lib/routes/xiaohongshu/notes.ts | 118 +++++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/lib/config.ts b/lib/config.ts index e2e0e162f8794f..10d89d3e3c7819 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -309,6 +309,9 @@ export type Config = { device_id?: string; refresh_token?: string; }; + xiaohongshu: { + cookie?: string; + }; ximalaya: { token?: string; }; @@ -702,6 +705,9 @@ const calculateValue = () => { device_id: envs.XIAOYUZHOU_ID, refresh_token: envs.XIAOYUZHOU_TOKEN, }, + xiaohongshu: { + cookie: envs.XIAOHONGSHU_COOKIE, + }, ximalaya: { token: envs.XIMALAYA_TOKEN, }, diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index a4d85a5d563425..9cd5428d4e3974 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -1,6 +1,9 @@ import { Route } from '@/types'; import cache from '@/utils/cache'; -import { getNotes, formatText, formatNote } from './util'; +import { config } from '@/config'; +import * as cheerio from 'cheerio'; +import got from '@/utils/got'; +import { formatNote, formatText, getNotes } from './util'; export const route: Route = { path: '/user/:user_id/notes/fulltext', @@ -15,11 +18,22 @@ export const route: Route = { handler, example: '/xiaohongshu/user/52d8c541b4c4d60e6c867480/notes/fulltext', features: { + requireConfig: [ + { + name: 'XIAOHONGSHU_COOKIE', + optional: true, + description: '小红书 cookie 值，可在浏览器控制台通过`document.cookie`获取。', + }, + ], antiCrawler: true, requirePuppeteer: true, }, parameters: { user_id: 'user id, length 24 characters', + fulltext: { + description: '是否获取全文', + default: '', + }, }, }; @@ -27,13 +41,99 @@ async function handler(ctx) { const userId = ctx.req.param('user_id'); const url = `https://www.xiaohongshu.com/user/profile/${userId}`; - const { user, notes } = await getNotes(url, cache); + if (config.xiaohongshu.cookie && ctx.req.param('fulltext')) { + const user = await getUser(url, config.xiaohongshu.cookie); + const notes = await renderNotesFulltext(user.notes, url); + return { + title: `${user.userPageData.basicInfo.nickname} - 笔记 • 小红书 / RED`, + description: user.userPageData.basicInfo.desc, + image: user.userPageData.basicInfo.imageb || user.userPageData.basicInfo.images, + link: url, + item: notes, + }; + } else { + const { user, notes } = await getNotes(url, cache); + return { + title: `${user.nickname} - 笔记 • 小红书 / RED`, + description: formatText(user.desc), + image: user.imageb || user.images, + link: url, + item: notes.map((item) => formatNote(url, item)), + }; + } +} + +async function getUser(url, cookie) { + const res = await got(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Cookie: cookie, + }, + }); + const $ = cheerio.load(res.data); + + let script = $('script') + .filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }) + .text(); + script = script.slice('window.__INITIAL_STATE__='.length); + script = script.replaceAll('undefined', 'null'); + const state = JSON.parse(script); + return state.user; +} + +async function renderNotesFulltext(notes, url) { + const data: any[] = []; + for (const note of notes) { + for (const { noteCard } of note) { + const link = `${url}/${noteCard.noteId}`; + // eslint-disable-next-line no-await-in-loop + const { title, description } = await getFullNote(link); + data.push({ + title, + link, + description, + author: noteCard.user.nickName, + guid: noteCard.noteId, + }); + } + } + return data; +} - return { - title: `${user.nickname} - 笔记 • 小红书 / RED`, - description: formatText(user.desc), - image: user.imageb || user.images, - link: url, - item: notes.map((item) => formatNote(url, item)), - }; +async function getFullNote(link) { + const cookie = config.xiaohongshu.cookie; + const data = (await cache.tryGet(link, async () => { + const res = await got(link, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Cookie: cookie, + } as any, + }); + const $ = cheerio.load(res.data); + let script = $('script') + .filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }) + .text(); + script = script.slice('window.__INITIAL_STATE__='.length); + script = script.replaceAll('undefined', 'null'); + const state = JSON.parse(script); + const note = state.note.noteDetailMap[state.note.firstNoteId].note; + const images = note.imageList.map((image) => image.urlDefault); + const title = note.title; + let desc = note.desc; + desc = desc.replaceAll(/\[.*?\]/g, ''); + desc = desc.replaceAll(/#(.*?)#/g, '#$1'); + desc = desc.replaceAll('\n', '
'); + const description = `${images.map((image) => ``).join('')}
${title}
${desc}`; + return { + title, + description, + }; + })) as Promise<{ title: string; description: string }>; + return data; } From 1018ff245197abba713704117551a92fbee7d53e Mon Sep 17 00:00:00 2001 From: RieN 7z Date: Mon, 30 Sep 2024 23:47:37 +0800 Subject: [PATCH 06/14] [feat(routes)] Add ipsw.dev support --- lib/routes/ipswdev/index.ts | 77 +++++++++++++++++++++++++++++++++ lib/routes/ipswdev/namespace.ts | 7 +++ 2 files changed, 84 insertions(+) create mode 100644 lib/routes/ipswdev/index.ts create mode 100644 lib/routes/ipswdev/namespace.ts diff --git a/lib/routes/ipswdev/index.ts b/lib/routes/ipswdev/index.ts new file mode 100644 index 00000000000000..19f1a5b1ba75ef --- /dev/null +++ b/lib/routes/ipswdev/index.ts @@ -0,0 +1,77 @@ +import { Data, Route } from '@/types'; +import got from '@/utils/got'; +import { load } from 'cheerio'; + +const host = 'https://ipsw.dev/'; + +export const route: Route = { + path: '/index/:productID', + categories: ['program-update'], + example: '/ipswdev/index/iPhone16,1', + parameters: { + productID: 'Product ID', + }, + name: 'Apple latest beta firmware', + maintainers: ['RieN7'], + handler, +}; + +async function handler(ctx) { + const { productID } = ctx.req.param(); + const link = `https://ipsw.dev/product/version/${productID}`; + + const resp = await got({ + method: 'get', + url: link, + headers: { + Referer: host, + }, + }); + + const $ = load(resp.data); + + const productName = $('#IdentifierModal > div > div > div.modal-body > p:nth-child(1) > em').text(); + + // eslint-disable-next-line no-restricted-syntax + const list: Data[] = $('.firmware') + .map((index, element) => { + const ele = $(element); + const version = ele.find('td:nth-child(1) > div > div > strong').text(); + const build = ele.find('td:nth-child(1) > div > div > div > code').text(); + const date = ele.find('td:nth-child(3)').text(); + const size = ele.find('td:nth-child(4)').text(); + return { + title: `${productName} - ${version}`, + link: `https://ipsw.dev/download/${productID}/${build}`, + pubDate: new Date(date).toLocaleDateString(), + guid: build, + description: ` + + + + + + + + + + + + + + + + + + +
Version${version}
Build${build}
Released${date}
Size${size}
`, + }; + }) + .get(); + + return { + title: `${productName} Released`, + link, + item: list, + }; +} diff --git a/lib/routes/ipswdev/namespace.ts b/lib/routes/ipswdev/namespace.ts new file mode 100644 index 00000000000000..e6c6cb1ff9b805 --- /dev/null +++ b/lib/routes/ipswdev/namespace.ts @@ -0,0 +1,7 @@ +import { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'IPSW.dev', + url: 'ipsw.dev', + description: 'Download the latest beta firmware for iPhone, iPad, Mac, Apple Vision Pro, and Apple TV. Check the signing status of the beta firmware.', +}; From 637c1ec9bca49909b6145a29132b0b76d5fd2eaa Mon Sep 17 00:00:00 2001 From: RieN7 Date: Thu, 10 Oct 2024 22:11:55 +0800 Subject: [PATCH 07/14] feat: add fulltext with cookie --- lib/routes/xiaohongshu/notes.ts | 100 +++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index b04884b6c1aee0..5196a948d3a568 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -3,6 +3,7 @@ import cache from '@/utils/cache'; import { config } from '@/config'; import * as cheerio from 'cheerio'; import got from '@/utils/got'; +import { formatNote, formatText, getNotes } from './util'; export const route: Route = { path: '/user/:user_id/notes/fulltext', @@ -17,6 +18,13 @@ export const route: Route = { handler, example: '/xiaohongshu/user/52d8c541b4c4d60e6c867480/notes/fulltext', features: { + requireConfig: [ + { + name: 'XIAOHONGSHU_COOKIE', + optional: true, + description: '小红书 cookie 值，可在浏览器控制台通过`document.cookie`获取。', + }, + ], requireConfig: [ { name: 'XIAOHONGSHU_COOKIE', @@ -33,6 +41,10 @@ export const route: Route = { description: '是否获取全文', default: '', }, + fulltext: { + description: '是否获取全文', + default: '', + }, }, }; @@ -40,33 +52,45 @@ async function handler(ctx) { const userId = ctx.req.param('user_id'); const url = `https://www.xiaohongshu.com/user/profile/${userId}`; - const user = await getUser(url, config.xiaohongshu.cookie); - const notes = await renderNotesFulltext(user.notes, url); - - return { - title: `${user.nickname} - 笔记 • 小红书 / RED`, - description: user.desc, - image: user.imageb || user.images, - link: url, - item: notes, - }; + if (config.xiaohongshu.cookie && ctx.req.param('fulltext')) { + const user = await getUser(url, config.xiaohongshu.cookie); + const notes = await renderNotesFulltext(user.notes, url); + return { + title: `${user.userPageData.basicInfo.nickname} - 笔记 • 小红书 / RED`, + description: user.userPageData.basicInfo.desc, + image: user.userPageData.basicInfo.imageb || user.userPageData.basicInfo.images, + link: url, + item: notes, + }; + } else { + const { user, notes } = await getNotes(url, cache); + return { + title: `${user.nickname} - 笔记 • 小红书 / RED`, + description: formatText(user.desc), + image: user.imageb || user.images, + link: url, + item: notes.map((item) => formatNote(url, item)), + }; + } } async function getUser(url, cookie) { const res = await got(url, { headers: { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - "Cookie": cookie, - } + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Cookie: cookie, + }, }); const $ = cheerio.load(res.data); - - let script = $('script').filter((i, script) => { - const text = script.children[0]?.data; - return text?.startsWith('window.__INITIAL_STATE__='); - }).text(); + + let script = $('script') + .filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }) + .text(); script = script.slice('window.__INITIAL_STATE__='.length); - script = script.replace(/undefined/g, 'null'); + script = script.replaceAll('undefined', 'null'); const state = JSON.parse(script); return state.user; } @@ -74,14 +98,15 @@ async function getUser(url, cookie) { async function renderNotesFulltext(notes, url) { const data: any[] = []; for (const note of notes) { - for (const {noteCard} of note) { + for (const { noteCard } of note) { const link = `${url}/${noteCard.noteId}`; - const {title, description} = await getNote(link); + // eslint-disable-next-line no-await-in-loop + const { title, description } = await getFullNote(link); data.push({ title, link, description, - author: noteCard.user.nickname, + author: noteCard.user.nickName, guid: noteCard.noteId, }); } @@ -89,32 +114,37 @@ async function renderNotesFulltext(notes, url) { return data; } -async function getNote(link) { +async function getFullNote(link) { const cookie = config.xiaohongshu.cookie; - const data = await cache.tryGet(link, async () => { + const data = (await cache.tryGet(link, async () => { const res = await got(link, { headers: { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - "Cookie": cookie, - } as any + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Cookie: cookie, + } as any, }); const $ = cheerio.load(res.data); - let script = $('script').filter((i, script) => { - const text = script.children[0]?.data; - return text?.startsWith('window.__INITIAL_STATE__='); - }).text(); + let script = $('script') + .filter((i, script) => { + const text = script.children[0]?.data; + return text?.startsWith('window.__INITIAL_STATE__='); + }) + .text(); script = script.slice('window.__INITIAL_STATE__='.length); - script = script.replace(/undefined/g, 'null'); + script = script.replaceAll('undefined', 'null'); const state = JSON.parse(script); const note = state.note.noteDetailMap[state.note.firstNoteId].note; const images = note.imageList.map((image) => image.urlDefault); const title = note.title; - const desc = note.desc.replaceAll('#(.*?)\[话题\]#', '#$1'); + let desc = note.desc; + desc = desc.replaceAll(/\[.*?\]/g, ''); + desc = desc.replaceAll(/#(.*?)#/g, '#$1'); + desc = desc.replaceAll('\n', '
'); const description = `${images.map((image) => ``).join('')}
${title}
${desc}`; return { title, description, }; - }) as Promise<{title: string, description: string}>; + })) as Promise<{ title: string; description: string }>; return data; -} \ No newline at end of file +} From 3434cd447cb8693b3117107c8e0d1d06a3ccfb4f Mon Sep 17 00:00:00 2001 From: RieN7 Date: Thu, 10 Oct 2024 22:50:50 +0800 Subject: [PATCH 08/14] fix: config --- lib/routes/xiaohongshu/notes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index 9cd5428d4e3974..053ab690e4cb81 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -6,7 +6,7 @@ import got from '@/utils/got'; import { formatNote, formatText, getNotes } from './util'; export const route: Route = { - path: '/user/:user_id/notes/fulltext', + path: '/user/:user_id/notes/:fulltext', radar: [ { source: ['xiaohongshu.com/user/profile/:user_id'], From 5399b18d1f56695cc55e657930e4f63f6cb3d00c Mon Sep 17 00:00:00 2001 From: RieN7 Date: Thu, 10 Oct 2024 23:03:48 +0800 Subject: [PATCH 09/14] fix: await in loop --- lib/routes/xiaohongshu/notes.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index 053ab690e4cb81..3bde514d2d860d 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -86,20 +86,20 @@ async function getUser(url, cookie) { async function renderNotesFulltext(notes, url) { const data: any[] = []; - for (const note of notes) { - for (const { noteCard } of note) { + const promises = notes.flatMap((note) => + note.map(async ({ noteCard }) => { const link = `${url}/${noteCard.noteId}`; - // eslint-disable-next-line no-await-in-loop const { title, description } = await getFullNote(link); - data.push({ + return { title, link, description, author: noteCard.user.nickName, guid: noteCard.noteId, - }); - } - } + }; + }) + ); + data.push(...(await Promise.all(promises))); return data; } From 735da7c7470e9154bc0ecf33bffdc60163285271 Mon Sep 17 00:00:00 2001 From: RieN7 Date: Fri, 11 Oct 2024 13:10:45 +0800 Subject: [PATCH 10/14] fix: use art-template --- lib/routes/ipswdev/index.ts | 31 +++++--------------- lib/routes/ipswdev/templates/description.art | 20 +++++++++++++ 2 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 lib/routes/ipswdev/templates/description.art diff --git a/lib/routes/ipswdev/index.ts b/lib/routes/ipswdev/index.ts index 19f1a5b1ba75ef..5c566a24e54920 100644 --- a/lib/routes/ipswdev/index.ts +++ b/lib/routes/ipswdev/index.ts @@ -1,8 +1,8 @@ import { Data, Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; - -const host = 'https://ipsw.dev/'; +import { art } from '@/utils/render'; +import path from 'node:path'; export const route: Route = { path: '/index/:productID', @@ -32,7 +32,6 @@ async function handler(ctx) { const productName = $('#IdentifierModal > div > div > div.modal-body > p:nth-child(1) > em').text(); - // eslint-disable-next-line no-restricted-syntax const list: Data[] = $('.firmware') .map((index, element) => { const ele = $(element); @@ -45,26 +44,12 @@ async function handler(ctx) { link: `https://ipsw.dev/download/${productID}/${build}`, pubDate: new Date(date).toLocaleDateString(), guid: build, - description: ` - - - - - - - - - - - - - - - - - - -
Version${version}
Build${build}
Released${date}
Size${size}
`, + description: art(path.join(__dirname, 'templates/description.art'), { + version, + build, + date, + size, + }), }; }) .get(); diff --git a/lib/routes/ipswdev/templates/description.art b/lib/routes/ipswdev/templates/description.art new file mode 100644 index 00000000000000..c0faef7bfac60d --- /dev/null +++ b/lib/routes/ipswdev/templates/description.art @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + +
Version{{ version }}
Build{{ build }}
Released{{ released }}
Size{{ size }}
\ No newline at end of file From 650c85838839c2f626066d984e5336af2f578c66 Mon Sep 17 00:00:00 2001 From: RieN7 Date: Fri, 11 Oct 2024 13:11:56 +0800 Subject: [PATCH 11/14] fix: remove ipswdev in other branch --- lib/routes/ipswdev/index.ts | 77 --------------------------------- lib/routes/ipswdev/namespace.ts | 7 --- 2 files changed, 84 deletions(-) delete mode 100644 lib/routes/ipswdev/index.ts delete mode 100644 lib/routes/ipswdev/namespace.ts diff --git a/lib/routes/ipswdev/index.ts b/lib/routes/ipswdev/index.ts deleted file mode 100644 index 19f1a5b1ba75ef..00000000000000 --- a/lib/routes/ipswdev/index.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Data, Route } from '@/types'; -import got from '@/utils/got'; -import { load } from 'cheerio'; - -const host = 'https://ipsw.dev/'; - -export const route: Route = { - path: '/index/:productID', - categories: ['program-update'], - example: '/ipswdev/index/iPhone16,1', - parameters: { - productID: 'Product ID', - }, - name: 'Apple latest beta firmware', - maintainers: ['RieN7'], - handler, -}; - -async function handler(ctx) { - const { productID } = ctx.req.param(); - const link = `https://ipsw.dev/product/version/${productID}`; - - const resp = await got({ - method: 'get', - url: link, - headers: { - Referer: host, - }, - }); - - const $ = load(resp.data); - - const productName = $('#IdentifierModal > div > div > div.modal-body > p:nth-child(1) > em').text(); - - // eslint-disable-next-line no-restricted-syntax - const list: Data[] = $('.firmware') - .map((index, element) => { - const ele = $(element); - const version = ele.find('td:nth-child(1) > div > div > strong').text(); - const build = ele.find('td:nth-child(1) > div > div > div > code').text(); - const date = ele.find('td:nth-child(3)').text(); - const size = ele.find('td:nth-child(4)').text(); - return { - title: `${productName} - ${version}`, - link: `https://ipsw.dev/download/${productID}/${build}`, - pubDate: new Date(date).toLocaleDateString(), - guid: build, - description: ` - - - - - - - - - - - - - - - - - - -
Version${version}
Build${build}
Released${date}
Size${size}
`, - }; - }) - .get(); - - return { - title: `${productName} Released`, - link, - item: list, - }; -} diff --git a/lib/routes/ipswdev/namespace.ts b/lib/routes/ipswdev/namespace.ts deleted file mode 100644 index e6c6cb1ff9b805..00000000000000 --- a/lib/routes/ipswdev/namespace.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Namespace } from '@/types'; - -export const namespace: Namespace = { - name: 'IPSW.dev', - url: 'ipsw.dev', - description: 'Download the latest beta firmware for iPhone, iPad, Mac, Apple Vision Pro, and Apple TV. Check the signing status of the beta firmware.', -}; From 09cfceeb6b836ab480aaeb5454998235b7bc6a3b Mon Sep 17 00:00:00 2001 From: RieN7 Date: Mon, 14 Oct 2024 14:59:09 +0800 Subject: [PATCH 12/14] fix: add pubDate --- lib/routes/xiaohongshu/notes.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/routes/xiaohongshu/notes.ts b/lib/routes/xiaohongshu/notes.ts index 3bde514d2d860d..2f58aa75a5bb44 100644 --- a/lib/routes/xiaohongshu/notes.ts +++ b/lib/routes/xiaohongshu/notes.ts @@ -89,13 +89,14 @@ async function renderNotesFulltext(notes, url) { const promises = notes.flatMap((note) => note.map(async ({ noteCard }) => { const link = `${url}/${noteCard.noteId}`; - const { title, description } = await getFullNote(link); + const { title, description, pubDate } = await getFullNote(link); return { title, link, description, author: noteCard.user.nickName, guid: noteCard.noteId, + pubDate, }; }) ); @@ -129,11 +130,13 @@ async function getFullNote(link) { desc = desc.replaceAll(/\[.*?\]/g, ''); desc = desc.replaceAll(/#(.*?)#/g, '#$1'); desc = desc.replaceAll('\n', '
'); + const pubDate = new Date(note.time); const description = `${images.map((image) => ``).join('')}
${title}
${desc}`; return { title, description, + pubDate, }; - })) as Promise<{ title: string; description: string }>; + })) as Promise<{ title: string; description: string; pubDate: Date }>; return data; } From db4d9d4bb40e6a30ae7604304af88b71d39f7388 Mon Sep 17 00:00:00 2001 From: daniel <861397272@qq.com> Date: Sun, 24 Nov 2024 16:29:36 +0800 Subject: [PATCH 13/14] fix(route) ikea/cn/low-price --- lib/routes/ikea/cn/low-price.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/routes/ikea/cn/low-price.ts b/lib/routes/ikea/cn/low-price.ts index d1c5d1a56d71df..994466f348258a 100644 --- a/lib/routes/ikea/cn/low-price.ts +++ b/lib/routes/ikea/cn/low-price.ts @@ -32,7 +32,7 @@ async function handler() { headers: generateRequestHeaders(), searchParams: { processOutOfStock: 'SORT', - groupId: 'cms_低价好物_cms-商品列表-_0', + groupId: 'cms_product_cn--zh--8b08af400ac511ec909ec36c6e99b004_0_0', page: 1, size: 200, }, @@ -42,6 +42,7 @@ async function handler() { title: 'IKEA 宜家 - 低价优选', link: 'https://www.ikea.cn/cn/zh/campaigns/wo3-men2-de-chao1-zhi2-di1-jia4-pub8b08af40', description: '低价优选', + allowEmpty: true, item: response.data.products.map((element) => generateProductItem(element)), }; } From 2ef74840a7433783a1a426c638f5bebc99e55fdf Mon Sep 17 00:00:00 2001 From: daniel <861397272@qq.com> Date: Sat, 14 Dec 2024 19:20:08 +0800 Subject: [PATCH 14/14] =?UTF-8?q?feat(route/missav):=20=E4=BF=AE=E5=A4=8Dm?= =?UTF-8?q?issav=E4=B8=8D=E8=83=BD=E8=AE=BF=E9=97=AE=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/ipswdev/index.ts | 62 -------------------- lib/routes/ipswdev/namespace.ts | 7 --- lib/routes/ipswdev/templates/description.art | 20 ------- lib/routes/missav/new.ts | 7 ++- 4 files changed, 4 insertions(+), 92 deletions(-) delete mode 100644 lib/routes/ipswdev/index.ts delete mode 100644 lib/routes/ipswdev/namespace.ts delete mode 100644 lib/routes/ipswdev/templates/description.art diff --git a/lib/routes/ipswdev/index.ts b/lib/routes/ipswdev/index.ts deleted file mode 100644 index 5c566a24e54920..00000000000000 --- a/lib/routes/ipswdev/index.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { Data, Route } from '@/types'; -import got from '@/utils/got'; -import { load } from 'cheerio'; -import { art } from '@/utils/render'; -import path from 'node:path'; - -export const route: Route = { - path: '/index/:productID', - categories: ['program-update'], - example: '/ipswdev/index/iPhone16,1', - parameters: { - productID: 'Product ID', - }, - name: 'Apple latest beta firmware', - maintainers: ['RieN7'], - handler, -}; - -async function handler(ctx) { - const { productID } = ctx.req.param(); - const link = `https://ipsw.dev/product/version/${productID}`; - - const resp = await got({ - method: 'get', - url: link, - headers: { - Referer: host, - }, - }); - - const $ = load(resp.data); - - const productName = $('#IdentifierModal > div > div > div.modal-body > p:nth-child(1) > em').text(); - - const list: Data[] = $('.firmware') - .map((index, element) => { - const ele = $(element); - const version = ele.find('td:nth-child(1) > div > div > strong').text(); - const build = ele.find('td:nth-child(1) > div > div > div > code').text(); - const date = ele.find('td:nth-child(3)').text(); - const size = ele.find('td:nth-child(4)').text(); - return { - title: `${productName} - ${version}`, - link: `https://ipsw.dev/download/${productID}/${build}`, - pubDate: new Date(date).toLocaleDateString(), - guid: build, - description: art(path.join(__dirname, 'templates/description.art'), { - version, - build, - date, - size, - }), - }; - }) - .get(); - - return { - title: `${productName} Released`, - link, - item: list, - }; -} diff --git a/lib/routes/ipswdev/namespace.ts b/lib/routes/ipswdev/namespace.ts deleted file mode 100644 index e6c6cb1ff9b805..00000000000000 --- a/lib/routes/ipswdev/namespace.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Namespace } from '@/types'; - -export const namespace: Namespace = { - name: 'IPSW.dev', - url: 'ipsw.dev', - description: 'Download the latest beta firmware for iPhone, iPad, Mac, Apple Vision Pro, and Apple TV. Check the signing status of the beta firmware.', -}; diff --git a/lib/routes/ipswdev/templates/description.art b/lib/routes/ipswdev/templates/description.art deleted file mode 100644 index c0faef7bfac60d..00000000000000 --- a/lib/routes/ipswdev/templates/description.art +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - -
Version{{ version }}
Build{{ build }}
Released{{ released }}
Size{{ size }}
\ No newline at end of file diff --git a/lib/routes/missav/new.ts b/lib/routes/missav/new.ts index d9e720deee3b5d..c0a60e7f3e515b 100644 --- a/lib/routes/missav/new.ts +++ b/lib/routes/missav/new.ts @@ -2,10 +2,11 @@ import { Route } from '@/types'; import { getCurrentPath } from '@/utils/helpers'; const __dirname = getCurrentPath(import.meta.url); -import got from '@/utils/got'; import { load } from 'cheerio'; import { art } from '@/utils/render'; import path from 'node:path'; +import puppeteer from '@/utils/puppeteer'; +import { puppeteerGet } from '@/routes/aip/utils'; export const route: Route = { path: '/new', @@ -33,9 +34,9 @@ export const route: Route = { async function handler() { const baseUrl = 'https://missav.com'; - const { data: response } = await got(`${baseUrl}/dm397/new`); + const browser = await puppeteer(); + const response = await puppeteerGet(`${baseUrl}/dm397/new`, browser); const $ = load(response); - const items = $('.grid .group') .toArray() .map((item) => {