diff --git a/source/proxies/rss-feed.ts b/source/proxies/rss-feed.ts index 4f1acaa16e2..c82ac5072c5 100644 --- a/source/proxies/rss-feed.ts +++ b/source/proxies/rss-feed.ts @@ -1,3 +1,5 @@ +import { config } from '../config'; +import { nextFetchTimeStr } from '../utils/next-fetch-time'; import { db } from '../database'; import errors from '../utils/errors'; import { Feed } from '../types/feed'; @@ -100,6 +102,14 @@ export async function getAllFeeds(ttl = true): Promise { }); if (ttl) { query = query.where('next_fetch_time', '<', db.fn.now()); + } else { + query = query.where(function () { + this.where( + 'error_count', + '<', + config.notify_error_count + ).orWhere('next_fetch_time', '<', db.fn.now()); + }); } const feeds = await query.select(); arrayShuffle(feeds); @@ -179,11 +189,34 @@ export async function updateFeed( } } -export async function failAttempt(feedUrl: string): Promise { +export async function failAttempt(feedUrl: string): Promise> { try { - return await db('rss_feed') + const threshold = config.notify_error_count; + const gap = config['GAP_MINUTES']; + const now = Date.now(); + const [feed] = await db('rss_feed') .where('url', feedUrl) - .increment('error_count', 1); + .increment('error_count', 1) + .update({ + next_fetch_time: db.raw( + `CASE + WHEN error_count + 1 >= ? THEN ? + WHEN error_count + 1 >= ? THEN ? + WHEN error_count + 1 >= ? THEN ? + ELSE next_fetch_time + END`, + [ + threshold * 4, + nextFetchTimeStr(gap * 8, now), + threshold * 2, + nextFetchTimeStr(gap * 4, now), + threshold, + nextFetchTimeStr(gap * 2, now) + ] + ) + }) + .returning('*'); + return Optional(feed); } catch (e) { throw errors.newCtrlErr('DB_ERROR', e); } diff --git a/source/utils/fetch.ts b/source/utils/fetch.ts index 2a1e38578ad..9342f6c2307 100644 --- a/source/utils/fetch.ts +++ b/source/utils/fetch.ts @@ -9,12 +9,12 @@ import { findFeed, getNewItems } from './feed'; import { config } from '../config'; import { Feed, FeedItem } from '../types/feed'; import { Optional, Option, isNone, none, isSome } from '../types/option'; -import { parseString } from '../parser/parse'; +import { nextFetchTimeStr } from './next-fetch-time'; +import { parseString, TRSS } from '../parser/parse'; import { getAllFeeds, updateHashList, failAttempt, - getFeedByUrl, updateFeed, handleRedirect } from '../proxies/rss-feed'; @@ -57,15 +57,6 @@ let lastDueFeedCount = 0; let lastScheduledFeedCount = 0; let lastDuplicateSkippedFeedCount = 0; -function nextFetchTimeStr(minutes: number) { - // use SQLite CURRENT_TIMESTAMP return format like `2021-03-09 06:23:43` - // It should use the UTC timezone - return new Date(Date.now() + minutes * 60 * 1000) - .toISOString() - .split('.')[0] - .replace('T', ' '); -} - function serializeError(error: unknown): string | unknown { if (error instanceof Error && error.stack) { return cleanStack(error.stack); @@ -252,82 +243,90 @@ async function handleErr( } } +async function recordFetchFailure(error: unknown, feedUrl: string) { + logHttpError(feedUrl, error); + const feed = await failAttempt(feedUrl); + if (isSome(feed)) { + const count = feed.value.error_count; + const repeatInterval = notify_error_count * 10; + if ( + count === notify_error_count || + (count > repeatInterval && count % repeatInterval === 0) + ) { + await handleErr(error as Messenger, feed.value); + } + } + return none; +} + export async function fetch(feedModal: Feed): Promise> { const feedUrl = feedModal.url; + logger.debug(`fetching ${feedUrl}`); + let requestUrl: string; + let res: Awaited>; try { - logger.debug(`fetching ${feedUrl}`); - const requestUrl = encodeUrl(feedUrl); - const request = got(requestUrl, { + requestUrl = encodeUrl(feedUrl); + res = await got(requestUrl, { headers: { 'If-None-Match': feedModal.etag_header, 'If-Modified-Since': feedModal.last_modified_header } }); + } catch (error) { + return recordFetchFailure(error, feedUrl); + } - const res = await request; - if (res.status === 304) { - const updatedFeedModal: Partial & { feed_id: number } = { - feed_id: feedModal.feed_id, - error_count: 0, - next_fetch_time: nextFetchTimeStr( - feedModal.ttl || config['GAP_MINUTES'] - ) - }; - await updateFeed(updatedFeedModal); - return none; - } - if (requestUrl !== res.url && res.status === 301) { - await handleRedirect(feedUrl, res.url); - } - const text = await res.textConverted(); - const feed = await parseString(text); - const items = feed.items; - const ttlMinutes = - typeof feed.ttl === 'number' && !Number.isNaN(feed.ttl) - ? feed.ttl - : config['GAP_MINUTES']; - const updatedFeedModal: Partial & { feed_id: number } = { + if (res.status === 304) { + await updateFeed({ feed_id: feedModal.feed_id, error_count: 0, - next_fetch_time: nextFetchTimeStr(ttlMinutes) - }; - if (feed.title !== feedModal.feed_title) { - updatedFeedModal.feed_title = feed.title; - } - if (!Number.isNaN(feed.ttl) && feed.ttl !== feedModal.ttl) { - updatedFeedModal.ttl = feed.ttl; - } - const lastModifiedHeader = res.headers.get('last-modified'); - if (lastModifiedHeader !== feedModal.last_modified_header) { - updatedFeedModal.last_modified_header = lastModifiedHeader; - } - const etag = res.headers.get('etag'); - if (etag !== feedModal.etag_header) { - updatedFeedModal.etag_header = etag || ''; - } - await updateFeed(updatedFeedModal); - return Optional( - items.map((item) => { - const { link, title, id } = item; - return { link, title, id } as FeedItem; - }) - ); - } catch (e) { - reportKnexTimeoutError(e, feedModal); - logHttpError(feedUrl, e); - await failAttempt(feedUrl); - const feed = await getFeedByUrl(feedUrl); - if (isSome(feed)) { - const round_time = notify_error_count * 10; - const round_happen = - feed.value.error_count % round_time === 0 && - feed.value.error_count > round_time; - if (feed.value.error_count === notify_error_count || round_happen) { - await handleErr(e as Messenger, feed.value); - } - } + next_fetch_time: nextFetchTimeStr( + feedModal.ttl || config['GAP_MINUTES'] + ) + }); + return none; } - return none; + if (requestUrl !== res.url && res.status === 301) { + await handleRedirect(feedUrl, res.url); + } + + let feed: TRSS; + try { + feed = await parseString(await res.textConverted()); + } catch (error) { + return recordFetchFailure(error, feedUrl); + } + const items = feed.items; + const ttlMinutes = + typeof feed.ttl === 'number' && !Number.isNaN(feed.ttl) + ? feed.ttl + : config['GAP_MINUTES']; + const updatedFeedModal: Partial & { feed_id: number } = { + feed_id: feedModal.feed_id, + error_count: 0, + next_fetch_time: nextFetchTimeStr(ttlMinutes) + }; + if (feed.title !== feedModal.feed_title) { + updatedFeedModal.feed_title = feed.title; + } + if (!Number.isNaN(feed.ttl) && feed.ttl !== feedModal.ttl) { + updatedFeedModal.ttl = feed.ttl; + } + const lastModifiedHeader = res.headers.get('last-modified'); + if (lastModifiedHeader !== feedModal.last_modified_header) { + updatedFeedModal.last_modified_header = lastModifiedHeader; + } + const etag = res.headers.get('etag'); + if (etag !== feedModal.etag_header) { + updatedFeedModal.etag_header = etag || ''; + } + await updateFeed(updatedFeedModal); + return Optional( + items.map((item) => { + const { link, title, id } = item; + return { link, title, id } as FeedItem; + }) + ); } const queue = fastQueue(async (eachFeed: Feed, cb: QueueCallback) => { diff --git a/source/utils/next-fetch-time.ts b/source/utils/next-fetch-time.ts new file mode 100644 index 00000000000..0df42189c4e --- /dev/null +++ b/source/utils/next-fetch-time.ts @@ -0,0 +1,8 @@ +export function nextFetchTimeStr(minutes: number, now = Date.now()) { + // use SQLite CURRENT_TIMESTAMP return format like `2021-03-09 06:23:43` + // It should use the UTC timezone + return new Date(now + minutes * 60 * 1000) + .toISOString() + .split('.')[0] + .replace('T', ' '); +} diff --git a/test/fetch-backoff.spec.ts b/test/fetch-backoff.spec.ts new file mode 100644 index 00000000000..c57a0fbbf4f --- /dev/null +++ b/test/fetch-backoff.spec.ts @@ -0,0 +1,153 @@ +import { mkdtempSync, rmSync } from 'fs'; +import { randomUUID } from 'crypto'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import type { Feed } from '../source/types/feed'; +import { isSome, none } from '../source/types/option'; + +const dir = mkdtempSync(join(tmpdir(), 'rssbot-backoff-')); +// On Heroku, DYNO makes the app use DATABASE_URL instead of the test database. +delete process.env.DYNO; +process.env.RSSBOT_DB_PATH = + process.env.TEST_BACKOFF_PG_URL || join(dir, 'test.db'); +process.env.RSSBOT_FETCH_GAP = process.env.TEST_BACKOFF_GAP || '11m'; +process.env.NOTIFY_ERR_COUNT = '3'; + +const { db, initDB } = + require('../source/database') as typeof import('../source/database'); +const { failAttempt, getAllFeeds, sub } = + require('../source/proxies/rss-feed') as typeof import('../source/proxies/rss-feed'); +const url = `https://backoff.example/${randomUUID()}`; +const threshold = 3; +const gap = process.env.RSSBOT_FETCH_GAP.endsWith('h') + ? parseInt(process.env.RSSBOT_FETCH_GAP) * 60 + : parseInt(process.env.RSSBOT_FETCH_GAP); +const originalTime = '2001-01-01 00:00:00'; +const now = Date.UTC(2030, 0, 1); + +beforeAll(async () => { + await initDB(); + await sub(1, url, 'Backoff test'); +}); + +afterAll(async () => { + await db.destroy(); + rmSync(dir, { recursive: true, force: true }); +}); + +test('uses one update per failure, including for a missing feed', async () => { + const queries: string[] = []; + const record = (q: { sql: string }) => queries.push(q.sql); + db.on('query', record); + try { + expect(isSome(await failAttempt(url))).toBe(true); + expect(await failAttempt(url + '/missing')).toBe(none); + } finally { + db.removeListener('query', record); + } + expect(queries).toHaveLength(2); + for (const sql of queries) { + expect(sql.replace(/\s+/g, ' ')).toMatch(/^update .*returning/i); + } +}); + +test('increments and returns the updated count across all retry tiers', async () => { + await db('rss_feed') + .where({ url }) + .update({ error_count: 0, next_fetch_time: originalTime }); + const clock = jest.spyOn(Date, 'now').mockReturnValue(now); + try { + const multipliers = [0, 0, 2, 2, 2, 4, 4, 4, 4, 4, 4, 8, 8]; + for (const [index, multiplier] of multipliers.entries()) { + const result = await failAttempt(url); + if (!isSome(result)) throw new Error('Missing feed'); + expect(result.value.error_count).toBe(index + 1); + + // Read the stored UTC fields before pg interprets them in the process timezone. + const stored = await db('rss_feed') + .where({ url }) + .first(db.raw('CAST(next_fetch_time AS TEXT) AS retry_time')); + const expected = multiplier + ? new Date(now + multiplier * gap * 60000) + .toISOString() + .slice(0, 19) + .replace('T', ' ') + : originalTime; + expect(stored.retry_time).toBe(expected); + } + } finally { + clock.mockRestore(); + } +}); + +test.each([ + [0, false, false, true], + [threshold - 1, false, false, true], + [threshold, false, false, false], + [threshold, true, true, true], + [0, true, true, true] +])( + 'selects subscribed feeds with count %i and due %s', + async (count, due, strict, relaxed) => { + await db('rss_feed') + .where({ url }) + .update({ + error_count: count, + next_fetch_time: due ? originalTime : '2099-01-01 00:00:00' + }); + expect((await getAllFeeds(true)).some((feed) => feed.url === url)).toBe( + strict + ); + expect( + (await getAllFeeds(false)).some((feed) => feed.url === url) + ).toBe(relaxed); + } +); + +test('excludes unsubscribed feeds in both TTL modes', async () => { + const orphan = url + '/unsubscribed'; + await db('rss_feed').insert({ url: orphan, feed_title: 'Unsubscribed' }); + for (const strict of [true, false]) { + expect( + (await getAllFeeds(strict)).some((feed) => feed.url === orphan) + ).toBe(false); + } +}); + +jest.mock('../source/utils/got', () => ({ + __esModule: true, + default: jest.fn() +})); + +test.each([200, 304])( + 'restores normal scheduling after HTTP %i', + async (status) => { + const { fetch } = + require('../source/utils/fetch') as typeof import('../source/utils/fetch'); + const got = jest.requireMock('../source/utils/got') + .default as jest.Mock; + await db('rss_feed').where({ url }).update({ + error_count: 12, + next_fetch_time: originalTime, + ttl: gap + }); + const feed = await db('rss_feed').where({ url }).first(); + got.mockResolvedValue({ + status, + url, + headers: new Headers(), + textConverted: async () => + 'Recovered' + }); + await fetch(feed!); + expect( + await db('rss_feed').where({ url }).first('error_count') + ).toEqual({ error_count: 0 }); + expect((await getAllFeeds(true)).some((row) => row.url === url)).toBe( + false + ); + expect((await getAllFeeds(false)).some((row) => row.url === url)).toBe( + true + ); + } +); diff --git a/test/fetch.spec.ts b/test/fetch.spec.ts index 126861f13dd..6ec515b6ccc 100644 --- a/test/fetch.spec.ts +++ b/test/fetch.spec.ts @@ -50,7 +50,6 @@ async function loadFetchModule( getAllFeeds: jest.Mock; updateHashList: jest.Mock; failAttempt: jest.Mock; - getFeedByUrl: jest.Mock; updateFeed: jest.Mock; handleRedirect: jest.Mock; }; @@ -74,8 +73,9 @@ async function loadFetchModule( const proxies = { getAllFeeds: jest.fn(async () => []), updateHashList: jest.fn(async () => undefined), - failAttempt: jest.fn(async () => 1), - getFeedByUrl: jest.fn(async () => Optional(undefined)), + failAttempt: jest.fn(() => + Promise.resolve(require('../source/types/option').none) + ), updateFeed: jest.fn(async () => 1), handleRedirect: jest.fn(async () => undefined) }; @@ -228,7 +228,7 @@ describe('fetch worker runtime', () => { .mockResolvedValueOnce({ textConverted: jest.fn(() => textDeferred.promise) }); - proxies.getFeedByUrl.mockResolvedValue( + proxies.failAttempt.mockResolvedValue( Optional({ ...feed, error_count: 5 @@ -249,12 +249,44 @@ describe('fetch worker runtime', () => { textDeferred.resolve(''); const result = await pending; expect(typeof result).toBe('symbol'); + expect(proxies.failAttempt).toHaveBeenCalledTimes(1); + expect(proxies.handleRedirect).toHaveBeenCalledTimes(1); expect(proxies.handleRedirect).toHaveBeenCalledWith( feed.url, 'https://example.com/new.xml' ); }); + test.each(['read', 'parse'])('counts a %s failure once', async (stage) => { + const { fetchModule, gotMock, proxies } = await loadFetchModule(); + const feed = buildFeed(); + gotMock.mockResolvedValue({ + status: 200, + url: feed.url, + textConverted: async () => { + if (stage === 'read') throw new Error('truncated body'); + return ''; + } + }); + await fetchModule.fetch(feed); + expect(proxies.failAttempt).toHaveBeenCalledTimes(1); + expect(gotMock).toHaveBeenCalledTimes(1); + }); + + test.each([301, 304])( + 'does not count database failures on HTTP %i', + async (status) => { + const { fetchModule, gotMock, proxies } = await loadFetchModule(); + const feed = buildFeed(); + const error = new Error('database unavailable'); + gotMock.mockResolvedValue({ status, url: feed.url + '/new' }); + proxies.updateFeed.mockRejectedValue(error); + proxies.handleRedirect.mockRejectedValue(error); + await expect(fetchModule.fetch(feed)).rejects.toBe(error); + expect(proxies.failAttempt).not.toHaveBeenCalled(); + } + ); + test('fetchAll resolves only after queued feeds complete', async () => { const { fetchModule, gotMock, proxies } = await loadFetchModule(); const firstUpdate = deferred();