diff --git a/source/index.ts b/source/index.ts index 67f0d805214..727e523c23f 100644 --- a/source/index.ts +++ b/source/index.ts @@ -53,6 +53,10 @@ import { } from './types/message'; import { migrateUser } from './proxies/users'; import { encodeUrl } from './utils/urlencode'; +import { + buildErrorNotification, + buildFeedChangeNotification +} from './utils/notification-messages'; const bot = new Telegraf(token, { telegram: { @@ -295,24 +299,17 @@ async function startFetchProcess(restartTime: number): Promise { } else { if (isErrorMaxTime(message)) { const { feed, err } = message; - send( - bot, - `${feed.feed_title}: ${feed.url} <%=i18n['ERROR_MANY_TIME']%> ${err.message}`, - feed - ); + const [template, locals] = buildErrorNotification(feed, err); + send(bot, template, feed, locals); } if (isChangeFeedUrl(message)) { const { feed, new_feed } = message; - const builder = []; - builder.push( - `${feed.feed_title}: <%=i18n['ERROR_MANY_TIME']%>` - ); // feed is from database which not urlencoded - builder.push(`<%=i18n['FOUND_FEEDS']%>:`); - builder.push(...new_feed); - builder.push(`<%=i18n['FEED_CHANGE_TO']%> ${new_feed[0]}`); - send(bot, builder.join('\n'), feed); + const [template, locals] = buildFeedChangeNotification( + feed, + new_feed, + encodeUrl + ); + send(bot, template, feed, locals); } } }); diff --git a/source/utils/feed.ts b/source/utils/feed.ts index abb37da1239..f21b3965639 100644 --- a/source/utils/feed.ts +++ b/source/utils/feed.ts @@ -2,6 +2,7 @@ import { none, Option, Optional } from '../types/option'; import { parseString, TRSS } from '../parser/parse'; import { FeedItem } from '../types/feed'; import hashFeed from './hash-feed'; +import { decodeHtmlEntities } from './html-entities'; export async function isFeedValid(feedStr: string): Promise> { try { @@ -12,26 +13,60 @@ export async function isFeedValid(feedStr: string): Promise> { } } +/** + * Extract and decode href attribute from a link tag. + * HTML entities in the href value are decoded to prevent double-escaping. + */ +function extractHref(linkTag: string): string | null { + const hrefMatch = linkTag.match(/href="(.+?)"/); + if (!hrefMatch) { + return null; + } + // Decode HTML entities (e.g., & -> &) before URL parsing + return decodeHtmlEntities(hrefMatch[1]); +} + +/** + * Discover RSS/Atom feed URLs from HTML link tags. + * + * @param html - HTML content to search + * @param reqUrl - Base URL for resolving relative paths + * @returns Array of discovered feed URLs + */ export async function findFeed( html: string, reqUrl: string ): Promise { - const reqURL = new URL(reqUrl); - const linksTag = html.match(/]+rel="alternate"[^>]+\/?>/g); - if (!linksTag) { + const baseUrl = new URL(reqUrl); + const linkTags = html.match(/]+rel="alternate"[^>]+\/?>/g); + + if (!linkTags) { return []; } - return linksTag - .filter((t) => t.match(/rss|atom/) && t.includes('href')) - .map((linkTag) => { - const url = linkTag.match(/href="(.+?)"/)[1]; - try { - return new URL(url).toString(); - } catch { - reqURL.pathname = url; - return reqURL.toString(); - } - }); + + const feedUrls: string[] = []; + + for (const linkTag of linkTags) { + // Only process RSS/Atom feeds + if (!linkTag.match(/rss|atom/) || !linkTag.includes('href')) { + continue; + } + + const href = extractHref(linkTag); + if (!href) { + continue; + } + + // Try to parse as absolute URL, fall back to relative path + try { + feedUrls.push(new URL(href).toString()); + } catch { + baseUrl.pathname = href; + feedUrls.push(baseUrl.toString()); + } + } + + return feedUrls; } export async function getNewItems( diff --git a/source/utils/html-entities.ts b/source/utils/html-entities.ts new file mode 100644 index 00000000000..a06450e42d9 --- /dev/null +++ b/source/utils/html-entities.ts @@ -0,0 +1,44 @@ +/** + * HTML entity decoder for href attribute values. + * + * When discovering RSS feeds from HTML, href attributes may contain entity-encoded + * characters (e.g., href="...?a=1&b=2"). This decoder converts them to their + * literal characters so URLs can be properly parsed and stored. + */ + +/** + * Common named HTML entities and their character equivalents. + */ +const NAMED_ENTITIES: Record = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", + nbsp: '\u00A0' +}; + +/** + * Decode HTML entities in a string. + * + * Supports: + * - Named entities: & < > " '   + * - Numeric entities (decimal): & < + * - Numeric entities (hex): & < + * + * @param text - String potentially containing HTML entities + * @returns String with entities decoded to their literal characters + */ +export function decodeHtmlEntities(text: string): string { + return text + .replace(/&([a-z]+);/gi, (match, entityName) => { + const normalized = entityName.toLowerCase(); + return NAMED_ENTITIES[normalized] ?? match; + }) + .replace(/&#(\d+);/g, (_match, decimal) => { + return String.fromCharCode(parseInt(decimal)); + }) + .replace(/&#x([0-9a-f]+);/gi, (_match, hexadecimal) => { + return String.fromCharCode(parseInt(hexadecimal, 16)); + }); +} diff --git a/source/utils/notification-messages.ts b/source/utils/notification-messages.ts new file mode 100644 index 00000000000..d89e5011227 --- /dev/null +++ b/source/utils/notification-messages.ts @@ -0,0 +1,93 @@ +/** + * Notification message templates for RSS feed errors and changes. + * + * All feed-controlled data (titles, URLs, error messages) is passed as EJS locals + * and rendered with escaped output (<%= %>) to prevent injection of Telegram HTML. + * This applies only to error and feed-change notifications; normal feed items are + * handled elsewhere. + */ + +import { Feed } from '../types/feed'; +import { Messenger } from '../types/message'; + +/** + * Template locals for error notifications (ERROR_MANY_TIME). + */ +export interface ErrorNotificationData { + title: string; + url: string; + errorMessage: string; +} + +/** + * Template locals for feed URL change notifications (isChangeFeedUrl). + */ +export interface FeedChangeNotificationData { + title: string; + url: string; + newFeedUrls: string[]; +} + +/** + * Template for error notification messages. + * Shows feed title, URL link, i18n error text, and error message. + */ +const ERROR_NOTIFICATION_TEMPLATE = + '<%= title %>: ' + + '<%= url %> ' + + '<%=i18n["ERROR_MANY_TIME"]%> ' + + '<%= errorMessage %>'; + +/** + * Template for feed URL change notifications. + * Shows old feed info, lists discovered feeds, and indicates the new URL. + */ +const FEED_CHANGE_NOTIFICATION_TEMPLATE = + '<%= title %>: <%=i18n["ERROR_MANY_TIME"]%>\n' + + '<%=i18n["FOUND_FEEDS"]%>:\n' + + '<% newFeedUrls.forEach(function(feedUrl) { %><%= feedUrl %>\n<% }); %>' + + '<%=i18n["FEED_CHANGE_TO"]%> <%= newFeedUrls[0] %>'; + +/** + * Build error notification message template and locals. + * + * @param feed - The feed that failed to fetch + * @param error - The error that occurred + * @returns Tuple of [template string, locals object] + */ +export function buildErrorNotification( + feed: Feed, + error: Messenger +): [string, ErrorNotificationData] { + return [ + ERROR_NOTIFICATION_TEMPLATE, + { + title: feed.feed_title, + url: feed.url, + errorMessage: error.message + } + ]; +} + +/** + * Build feed URL change notification message template and locals. + * + * @param feed - The feed whose URL changed + * @param newFeedUrls - Array of discovered feed URLs + * @param urlEncoder - Function to encode URLs (typically encodeUrl) + * @returns Tuple of [template string, locals object] + */ +export function buildFeedChangeNotification( + feed: Feed, + newFeedUrls: string[], + urlEncoder: (url: string) => string +): [string, FeedChangeNotificationData] { + return [ + FEED_CHANGE_NOTIFICATION_TEMPLATE, + { + title: feed.feed_title, + url: urlEncoder(feed.url), + newFeedUrls + } + ]; +} diff --git a/source/utils/send.ts b/source/utils/send.ts index 1cc2b284407..ea79a71b18c 100644 --- a/source/utils/send.ts +++ b/source/utils/send.ts @@ -142,7 +142,8 @@ async function handlerSendError(e: any, userId: number): Promise { const send = async ( bot: Telegraf, toSend: NonNullable, - feed: Feed + feed: Feed, + templateLocals?: Record ): Promise => { const subscribers = await getSubscribersByFeedId(feed.feed_id); if (typeof toSend === 'string') { @@ -156,7 +157,10 @@ const send = async ( if (isSome(user)) { lang = user.value.lang; } - const text = ejs.render(tpl, { i18n: i18n[lang] }); + const text = ejs.render(tpl, { + i18n: i18n[lang], + ...templateLocals + }); await sendMessageWithLimit(bot, userId, text, { parse_mode: 'HTML', link_preview_options: { is_disabled: true } diff --git a/test/feed-discovery.spec.ts b/test/feed-discovery.spec.ts new file mode 100644 index 00000000000..959cd47bcbe --- /dev/null +++ b/test/feed-discovery.spec.ts @@ -0,0 +1,114 @@ +import { findFeed } from '../source/utils/feed'; + +describe('Feed Discovery - findFeed()', () => { + it('should decode HTML entities in href attributes', async () => { + const html = ` + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/'); + + expect(feeds).toHaveLength(1); + // Should be decoded: & -> & + expect(feeds[0]).toBe('https://example.com/rss?a=1&b=2&c=3'); + }); + + it('should decode multiple types of entities', async () => { + const html = ` + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/'); + + expect(feeds).toHaveLength(1); + // Entities are decoded, then URL constructor normalizes/encodes special chars + expect(feeds[0]).toBe( + 'https://example.com/rss?x=%3Cvalue%3E&y=%22test%22' + ); + }); + + it('should decode numeric entities', async () => { + const html = ` + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/'); + + expect(feeds).toHaveLength(1); + expect(feeds[0]).toBe('https://example.com/rss?a=1&b=2'); + }); + + it('should decode hex entities', async () => { + const html = ` + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/'); + + expect(feeds).toHaveLength(1); + expect(feeds[0]).toBe('https://example.com/rss?a=1&b=2'); + }); + + it('should handle multiple feeds with entities', async () => { + const html = ` + + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/'); + + expect(feeds).toHaveLength(2); + expect(feeds[0]).toBe('https://example.com/rss1?a=1&b=2'); + expect(feeds[1]).toBe('https://example.com/atom?x=1&y=2&z=3'); + }); + + it('should leave plain URLs unchanged', async () => { + const html = ` + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/'); + + expect(feeds).toHaveLength(1); + expect(feeds[0]).toBe('https://example.com/feed.xml'); + }); + + it('should handle absolute path URLs with entities', async () => { + const html = ` + + + + + + `; + + const feeds = await findFeed(html, 'https://example.com/page'); + + expect(feeds).toHaveLength(1); + expect(feeds[0]).toBe('https://example.com/rss?a=1&b=2'); + }); +}); diff --git a/test/html-entities.spec.ts b/test/html-entities.spec.ts new file mode 100644 index 00000000000..be09cc0352a --- /dev/null +++ b/test/html-entities.spec.ts @@ -0,0 +1,47 @@ +import { decodeHtmlEntities } from '../source/utils/html-entities'; + +describe('HTML Entity Decoder', () => { + it('should decode common named entities', () => { + expect(decodeHtmlEntities('&')).toBe('&'); + expect(decodeHtmlEntities('<')).toBe('<'); + expect(decodeHtmlEntities('>')).toBe('>'); + expect(decodeHtmlEntities('"')).toBe('"'); + expect(decodeHtmlEntities(''')).toBe("'"); + }); + + it('should decode numeric entities (decimal)', () => { + expect(decodeHtmlEntities('&')).toBe('&'); + expect(decodeHtmlEntities('<')).toBe('<'); + expect(decodeHtmlEntities('>')).toBe('>'); + }); + + it('should decode numeric entities (hexadecimal)', () => { + expect(decodeHtmlEntities('&')).toBe('&'); + expect(decodeHtmlEntities('<')).toBe('<'); + expect(decodeHtmlEntities('>')).toBe('>'); + expect(decodeHtmlEntities('"')).toBe('"'); + }); + + it('should decode multiple entities in a string', () => { + expect(decodeHtmlEntities('a=1&b=2&c=3')).toBe('a=1&b=2&c=3'); + expect(decodeHtmlEntities('<tag>')).toBe(''); + }); + + it('should handle URLs with entities', () => { + const url = 'https://example.com/feed?a=1&b=2&c=3'; + expect(decodeHtmlEntities(url)).toBe( + 'https://example.com/feed?a=1&b=2&c=3' + ); + }); + + it('should leave plain text unchanged', () => { + expect(decodeHtmlEntities('https://example.com/feed.xml')).toBe( + 'https://example.com/feed.xml' + ); + expect(decodeHtmlEntities('Plain text')).toBe('Plain text'); + }); + + it('should leave unrecognized entities unchanged', () => { + expect(decodeHtmlEntities('&unknownentity;')).toBe('&unknownentity;'); + }); +}); diff --git a/test/template-injection.spec.ts b/test/template-injection.spec.ts new file mode 100644 index 00000000000..756d7c4625c --- /dev/null +++ b/test/template-injection.spec.ts @@ -0,0 +1,316 @@ +/** + * Notification Template Security Tests + * + * Verify that feed-controlled data (titles, URLs, error messages) in error and + * feed-change notifications is properly escaped to prevent Telegram HTML injection. + * + * All tests use production message builders to ensure real code paths are covered. + */ + +import { Telegraf } from 'telegraf'; +import send from '../source/utils/send'; +import { Feed } from '../source/types/feed'; +import * as subscribesProxy from '../source/proxies/subscribes'; +import * as usersProxy from '../source/proxies/users'; +import { Optional } from '../source/types/option'; +import { + buildErrorNotification, + buildFeedChangeNotification +} from '../source/utils/notification-messages'; +import { encodeUrl } from '../source/utils/urlencode'; + +jest.mock('../source/proxies/subscribes'); +jest.mock('../source/proxies/users'); + +// Test feed factory +function createTestFeed(overrides: Partial = {}): Feed { + return { + feed_id: 1, + url: 'https://example.com/feed.xml', + feed_title: 'Test Feed', + error_count: 0, + ttl: 60, + recent_hash_list: '[]', + etag_header: '', + last_modified_header: null, + next_fetch_time: new Date(), + ...overrides + }; +} + +describe('Notification Template Security', () => { + let bot: Telegraf; + let sentMessages: Array<{ userId: number; text: string; options: any }>; + const mockUserId = 12345; + + beforeEach(() => { + sentMessages = []; + + bot = new Telegraf('mock-token'); + bot.telegram.sendMessage = jest.fn(async (userId, text, options) => { + sentMessages.push({ userId, text, options }); + return {} as any; + }); + + (subscribesProxy.getSubscribersByFeedId as jest.Mock).mockResolvedValue( + [{ user_id: mockUserId, feed_id: 1 }] + ); + + (usersProxy.getUserById as jest.Mock).mockResolvedValue( + Optional({ user_id: mockUserId, lang: 'en', username: 'test' }) + ); + }); + + describe('Error Notifications', () => { + it('escapes HTML chars in title, URL, and error message', async () => { + const feed = createTestFeed({ + url: 'https://example.com/feed?a=1&b=2', + feed_title: 'Tech News & "Updates"' + }); + const error = { message: 'Error: & "Bad"' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toBe( + 'Tech News <Daily> & "Updates": ' + + 'https://example.com/feed?a=1&b=2 ' + + 'Unable to fetch for many times, you might want to unsub it ' + + 'Error: <Server> & "Bad"' + ); + }); + + it('escapes single quotes in title and error', async () => { + const feed = createTestFeed({ + feed_title: "It's News & 'Updates'" + }); + const error = { message: "Can't connect & 'timeout'" }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toBe( + 'It's News & 'Updates': ' + + 'https://example.com/feed.xml ' + + 'Unable to fetch for many times, you might want to unsub it ' + + 'Can't connect & 'timeout'' + ); + }); + + it('escapes special chars in URL query params (href and link text)', async () => { + const feed = createTestFeed({ + url: 'https://example.com/feed?x=&y="test"&z=\'quote\'' + }); + const error = { message: 'Timeout' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + const text = sentMessages[0].text; + expect(text).toContain( + '' + ); + expect(text).toContain( + 'https://example.com/feed?x=<val>&y="test"&z='quote'' + ); + }); + + it('renders literal EJS delimiters as text', async () => { + const feed = createTestFeed({ + feed_title: 'Tips: <%= var %> and <% code %>' + }); + const error = { message: 'Timeout' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toContain( + 'Tips: <%= var %> and <% code %>' + ); + }); + + it('renders plain inputs identically to previous behavior', async () => { + const feed = createTestFeed({ feed_title: 'My RSS Feed' }); + const error = { message: 'Connection timeout' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toBe( + 'My RSS Feed: https://example.com/feed.xml ' + + 'Unable to fetch for many times, you might want to unsub it Connection timeout' + ); + }); + }); + + describe('Feed Change Notifications', () => { + it('escapes HTML in title, URLs, and repeated trailing URL', async () => { + const feed = createTestFeed({ + url: 'https://example.com/old?a=1&b=2', + feed_title: 'News & "Updates"' + }); + const newFeedUrls = [ + 'https://example.com/new1?x=1&y=2', + 'https://example.com/new2?p=&q="test"' + ]; + + const [template, locals] = buildFeedChangeNotification( + feed, + newFeedUrls, + encodeUrl + ); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toBe( + 'News <Daily> & "Updates": ' + + ' ' + + 'Unable to fetch for many times, you might want to unsub it\n' + + 'Found feeds:\n' + + 'https://example.com/new1?x=1&y=2\n' + + 'https://example.com/new2?p=<val>&q="test"\n' + + 'Feed Change to https://example.com/new1?x=1&y=2' + ); + }); + + it('escapes special chars in all list items and trailing URL', async () => { + const feed = createTestFeed(); + const newFeedUrls = [ + 'https://ex.com/f1?a=1&b=2&c=3', + 'https://ex.com/f2?x=<>&y="\'', + 'https://ex.com/f3?z=3' + ]; + + const [template, locals] = buildFeedChangeNotification( + feed, + newFeedUrls, + encodeUrl + ); + await send(bot, template, feed, locals); + + const text = sentMessages[0].text; + expect(text).toContain('https://ex.com/f1?a=1&b=2&c=3\n'); + expect(text).toContain( + 'https://ex.com/f2?x=<>&y="'\n' + ); + expect(text).toContain( + 'Feed Change to https://ex.com/f1?a=1&b=2&c=3' + ); + }); + + it('renders literal EJS delimiters in feed URLs', async () => { + const feed = createTestFeed(); + const newFeedUrls = ['https://ex.com/<%= id %>/feed.xml']; + + const [template, locals] = buildFeedChangeNotification( + feed, + newFeedUrls, + encodeUrl + ); + await send(bot, template, feed, locals); + + const text = sentMessages[0].text; + expect(text).toContain('https://ex.com/<%= id %>/feed.xml\n'); + expect(text).toContain( + 'Feed Change to https://ex.com/<%= id %>/feed.xml' + ); + }); + + it('renders plain inputs with correct structure', async () => { + const feed = createTestFeed(); + const newFeedUrls = [ + 'https://example.com/newfeed1.xml', + 'https://example.com/newfeed2.xml' + ]; + + const [template, locals] = buildFeedChangeNotification( + feed, + newFeedUrls, + encodeUrl + ); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toBe( + 'Test Feed: ' + + 'Unable to fetch for many times, you might want to unsub it\n' + + 'Found feeds:\n' + + 'https://example.com/newfeed1.xml\n' + + 'https://example.com/newfeed2.xml\n' + + 'Feed Change to https://example.com/newfeed1.xml' + ); + }); + }); + + describe('No Double Escaping', () => { + it('escapes ampersands exactly once in error URLs', async () => { + const feed = createTestFeed({ + url: 'https://example.com/feed?a=1&b=2&c=3' + }); + const error = { message: 'Timeout' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toContain('?a=1&b=2&c=3'); + expect(sentMessages[0].text).not.toContain('&amp;'); + }); + + it('escapes ampersands exactly once in feed change URLs', async () => { + const feed = createTestFeed(); + const newFeedUrls = ['https://example.com/rss?a=1&b=2']; + + const [template, locals] = buildFeedChangeNotification( + feed, + newFeedUrls, + encodeUrl + ); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toContain('?a=1&b=2'); + expect(sentMessages[0].text).not.toContain('&amp;'); + }); + }); + + describe('Internationalization', () => { + it('works with Chinese language and special chars', async () => { + (usersProxy.getUserById as jest.Mock).mockResolvedValue( + Optional({ + user_id: mockUserId, + lang: 'zh-cn', + username: 'test' + }) + ); + + const feed = createTestFeed({ + url: 'https://example.com/feed?a=1&b=2', + feed_title: '科技新闻 <每日> & "更新"' + }); + const error = { message: 'Connection timeout' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + expect(sentMessages[0].text).toBe( + '科技新闻 <每日> & "更新": ' + + 'https://example.com/feed?a=1&b=2 ' + + '多次抓取失败,建议取消订阅 Connection timeout' + ); + }); + }); + + describe('Feed Discovery Integration', () => { + it('handles entity-decoded URLs from HTML discovery', async () => { + // URL decoded from HTML href="...?a=1&b=2" by findFeed() + const feed = createTestFeed({ + url: 'https://example.com/rss?a=1&b=2' + }); + const error = { message: 'Failed' }; + + const [template, locals] = buildErrorNotification(feed, error); + await send(bot, template, feed, locals); + + const text = sentMessages[0].text; + expect(text).toContain('https://example.com/rss?a=1&b=2'); + expect(text).not.toContain('&amp;'); + }); + }); +});