diff --git a/config.schema.json b/config.schema.json index c72543037..f21d43191 100644 --- a/config.schema.json +++ b/config.schema.json @@ -261,6 +261,10 @@ "required": ["windowMs", "limit"], "additionalProperties": false }, + "maxPackSizeBytes": { + "type": "number", + "description": "Maximum size in bytes of a git-upload-pack or git-receive-pack request body that the proxy will buffer for inspection. Larger requests are rejected with HTTP 413 (default 104857600, i.e. 100 MiB)." + }, "privateOrganizations": { "description": "Provider searches for listed private organizations are disabled, see commitConfig.diff.block.providers", "type": "array" diff --git a/src/config/generated/config.ts b/src/config/generated/config.ts index 0a85e8e70..f749d6555 100644 --- a/src/config/generated/config.ts +++ b/src/config/generated/config.ts @@ -55,6 +55,12 @@ export interface GitProxyConfig { * Provide custom URLs for the git proxy interfaces in case it cannot determine its own URL */ domains?: Domains; + /** + * Maximum size in bytes of a git-upload-pack or git-receive-pack request body that the + * proxy will buffer for inspection. Larger requests are rejected with HTTP 413 (default + * 104857600, i.e. 100 MiB). + */ + maxPackSizeBytes?: number; /** * List of plugins to integrate on GitProxy's push or pull actions. Each value is either a * file path or a module name. @@ -769,6 +775,7 @@ const typeMap: any = { { json: 'cookieSecret', js: 'cookieSecret', typ: u(undefined, '') }, { json: 'csrfProtection', js: 'csrfProtection', typ: u(undefined, true) }, { json: 'domains', js: 'domains', typ: u(undefined, r('Domains')) }, + { json: 'maxPackSizeBytes', js: 'maxPackSizeBytes', typ: u(undefined, 3.14) }, { json: 'plugins', js: 'plugins', typ: u(undefined, a('')) }, { json: 'privateOrganizations', js: 'privateOrganizations', typ: u(undefined, a('any')) }, { json: 'proxyUrl', js: 'proxyUrl', typ: u(undefined, '') }, diff --git a/src/config/index.ts b/src/config/index.ts index 0d4591300..9277130e6 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -317,6 +317,16 @@ export const getRateLimit = () => { return config.rateLimit; }; +export const DEFAULT_MAX_PACK_SIZE_BYTES = 100 * 1024 * 1024; + +export const getMaxPackSizeBytes = (): number => { + const config = loadFullConfiguration(); + const limit = config.maxPackSizeBytes; + return typeof limit === 'number' && Number.isFinite(limit) && limit > 0 + ? limit + : DEFAULT_MAX_PACK_SIZE_BYTES; +}; + // Function to handle configuration updates const handleConfigUpdate = async (newConfig: Configuration) => { console.log('Configuration updated from external source'); diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 6c809eff4..0ba220ab8 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -19,8 +19,12 @@ import proxy from 'express-http-proxy'; import { PassThrough } from 'stream'; import getRawBody from 'raw-body'; import { executeChain } from '../chain'; +import { exec as parseAction } from '../processors/pre-processor/parseAction'; +import { exec as checkRepoInAuthorisedList } from '../processors/push-action/checkRepoInAuthorisedList'; +import { exec as audit } from '../processors/post-processor/audit'; import { processUrlPath, validGitRequest } from './helper'; import { getAllProxiedHosts } from '../../db'; +import { getMaxPackSizeBytes } from '../../config'; import { ProxyOptions } from 'express-http-proxy'; import { handleErrorAndLog } from '../../utils/errors'; @@ -162,34 +166,73 @@ const isPackPost = (req: Request) => req.method === 'POST' && /^(?:\/[^/]+)*\/[^/]+\.git\/(?:git-upload-pack|git-receive-pack)$/.test(req.url); +const rejectPackPost = (req: Request, res: Response, message: string): void => { + logAction(req.url, req.headers.host, req.headers['user-agent'], ActionType.ERROR, message); + sendErrorResponse(req, res, message); +}; + +const rejectTooLarge = (req: Request, res: Response, limit: number): void => { + const message = `Request body exceeds the maximum pack size of ${limit} bytes`; + logAction(req.url, req.headers.host, req.headers['user-agent'], ActionType.ERROR, message); + res.set('connection', 'close'); + res.status(413).end(message); +}; + const extractRawBody = async (req: Request, res: Response, next: NextFunction) => { if (!isPackPost(req)) { return next(); } - const proxyStream = new PassThrough({ - highWaterMark: 4 * 1024 * 1024, - }); - const pluginStream = new PassThrough({ - highWaterMark: 4 * 1024 * 1024, - }); + // run the cheap checks before reading any of the body into memory + const urlComponents = processUrlPath(req.url); + if ( + !urlComponents || + urlComponents.gitPath === undefined || + !validGitRequest(urlComponents.gitPath, req.headers) + ) { + return rejectPackPost(req, res, 'Invalid request received'); + } - req.pipe(proxyStream); - req.pipe(pluginStream); + try { + const action = await checkRepoInAuthorisedList(req, await parseAction(req)); + if (!action.continue()) { + // keep the audit record the action chain would have written, without reading the body + await audit(req, action); + return rejectPackPost( + req, + res, + action.errorMessage ?? 'Rejecting repo not in the authorised whitelist', + ); + } + } catch (error: unknown) { + const message = handleErrorAndLog(error, 'Error checking repository authorisation'); + return rejectPackPost(req, res, message); + } + + const limit = getMaxPackSizeBytes(); + const contentLength = Number(req.headers['content-length']); + if (Number.isFinite(contentLength) && contentLength > limit) { + return rejectTooLarge(req, res, limit); + } try { - const buf = await getRawBody(pluginStream, { limit: '1gb' }); + // read straight from the request so backpressure only depends on this reader; the + // buffered body is replayed to the upstream proxy once the plugins have inspected it + const buf = await getRawBody(req, { limit }); req.bodyRaw = buf; - req.pipe = (dest, opts) => proxyStream.pipe(dest, opts); + req.pipe = (dest, opts) => { + const replay = new PassThrough(); + replay.end(buf); + return replay.pipe(dest, opts); + }; next(); } catch (error: unknown) { - if (error instanceof Error) { - console.error(error.message); - proxyStream.destroy(error); - } else { - console.error(String(error)); - proxyStream.destroy(new Error(String(error))); + const err = error instanceof Error ? error : new Error(String(error)); + console.error(err.message); + if ((err as { type?: string }).type === 'entity.too.large') { + return rejectTooLarge(req, res, limit); } + res.set('connection', 'close'); res.status(500).end('Proxy error'); } }; diff --git a/test/extractRawBody.test.ts b/test/extractRawBody.test.ts index d86790c62..b0afac937 100644 --- a/test/extractRawBody.test.ts +++ b/test/extractRawBody.test.ts @@ -28,9 +28,20 @@ vi.mock('../src/proxy/chain', () => ({ executeChain: vi.fn(), })); +vi.mock('../src/db', () => ({ + getRepoByUrl: vi.fn(), + writeAudit: vi.fn(), + getAllProxiedHosts: vi.fn().mockResolvedValue([]), +})); + +vi.mock('../src/config', () => ({ + getMaxPackSizeBytes: vi.fn().mockReturnValue(1024), +})); + // Now import the module-under-test, which will receive the mocked deps import { extractRawBody, isPackPost } from '../src/proxy/routes'; import * as chain from '../src/proxy/chain'; +import * as db from '../src/db'; describe('extractRawBody middleware', () => { let req: any; @@ -40,10 +51,21 @@ describe('extractRawBody middleware', () => { beforeEach(() => { (rawBody as Mock).mockClear(); (chain.executeChain as Mock).mockClear(); + (db.getRepoByUrl as Mock).mockReset(); + (db.writeAudit as Mock).mockClear(); + (db.getRepoByUrl as Mock).mockImplementation(async (url: string) => + url === 'https://github.com/proj/foo.git' ? { url } : null, + ); req = new PassThrough(); req.method = 'POST'; req.url = '/proj/foo.git/git-upload-pack'; + req.originalUrl = req.url; + req.headers = { + 'user-agent': 'git/2.43.0', + accept: 'application/x-git-upload-pack-result', + }; + req.pipe = vi.fn(PassThrough.prototype.pipe); res = { set: vi.fn().mockReturnThis(), @@ -74,6 +96,105 @@ describe('extractRawBody middleware', () => { expect(req.bodyRaw).toBeDefined(); expect(typeof req.pipe).toBe('function'); }); + + it('replays the buffered body when the request is piped to the upstream proxy', async () => { + req.end(); + await extractRawBody(req, res, next); + + const upstream = new PassThrough(); + const chunks: Buffer[] = []; + upstream.on('data', (chunk: Buffer) => chunks.push(chunk)); + await new Promise((resolve) => { + upstream.on('end', resolve); + req.pipe(upstream); + }); + + expect(Buffer.concat(chunks).toString()).toBe('payload'); + }); + + it('buffers with the configured pack size limit', async () => { + req.end(); + await extractRawBody(req, res, next); + expect((rawBody as Mock).mock.calls[0][1]).toEqual({ limit: 1024 }); + }); + + it('rejects a pack POST with a non-git User-Agent before reading the body', async () => { + req.headers['user-agent'] = 'curl/8.0'; + + await extractRawBody(req, res, next); + + expect(rawBody).not.toHaveBeenCalled(); + expect(req.pipe).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + expect(res.set).toHaveBeenCalledWith('connection', 'close'); + expect(res.send).toHaveBeenCalledWith(expect.stringContaining('Invalid request received')); + }); + + it('rejects a pack POST without a git Accept header before reading the body', async () => { + delete req.headers.accept; + + await extractRawBody(req, res, next); + + expect(rawBody).not.toHaveBeenCalled(); + expect(req.pipe).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects and audits a push to an unregistered repository before reading the body', async () => { + req.url = req.originalUrl = '/github.com/evil/unknown.git/git-receive-pack'; + req.headers.accept = 'application/x-git-receive-pack-result'; + req.headers['content-type'] = 'application/x-git-receive-pack-request'; + + await extractRawBody(req, res, next); + + expect(db.getRepoByUrl).toHaveBeenCalledWith('https://github.com/evil/unknown.git'); + expect(rawBody).not.toHaveBeenCalled(); + expect(req.pipe).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + expect(res.send).toHaveBeenCalledWith(expect.stringContaining('not in the authorised')); + expect(db.writeAudit).toHaveBeenCalledOnce(); + const audited = (db.writeAudit as Mock).mock.calls[0][0]; + expect(audited.type).toBe('push'); + expect(audited.error).toBe(true); + expect(audited.url).toContain('/evil/unknown.git'); + }); + + it('rejects a pull from an unregistered repository without auditing it', async () => { + req.url = req.originalUrl = '/github.com/evil/unknown.git/git-upload-pack'; + req.headers['content-type'] = 'application/x-git-upload-pack-request'; + + await extractRawBody(req, res, next); + + expect(rawBody).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + expect(res.send).toHaveBeenCalledWith(expect.stringContaining('not in the authorised')); + expect(db.writeAudit).not.toHaveBeenCalled(); + }); + + it('rejects with 413 when Content-Length exceeds the limit before reading the body', async () => { + req.headers['content-length'] = '1025'; + + await extractRawBody(req, res, next); + + expect(rawBody).not.toHaveBeenCalled(); + expect(req.pipe).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(413); + }); + + it('rejects with 413 when the streamed body exceeds the limit', async () => { + const tooLarge = Object.assign(new Error('request entity too large'), { + type: 'entity.too.large', + }); + (rawBody as Mock).mockRejectedValueOnce(tooLarge); + req.end(); + + await extractRawBody(req, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(req.bodyRaw).toBeUndefined(); + expect(res.status).toHaveBeenCalledWith(413); + }); }); describe('isPackPost()', () => { diff --git a/test/testProxyRoute.test.ts b/test/testProxyRoute.test.ts index 32f720015..cf08057b6 100644 --- a/test/testProxyRoute.test.ts +++ b/test/testProxyRoute.test.ts @@ -241,6 +241,31 @@ describe('proxy express application', () => { expect(res2.text).toContain('Rejecting repo'); }, 5000); + it('should reject a pack POST with a non-git User-Agent without forwarding it', async () => { + const res = await request(proxy.getExpressApp()!) + .post(`${TEST_DEFAULT_REPO.proxyUrlPrefix}/git-receive-pack`) + .set('user-agent', 'curl/8.0') + .set('accept', 'application/x-git-receive-pack-result') + .set('content-type', 'application/x-git-receive-pack-request') + .send(Buffer.alloc(64 * 1024)); + + expect(res.status).toBe(200); + expect(res.headers.connection).toBe('close'); + expect(res.text).toContain('Invalid request received'); + }); + + it('should reject a pack POST for an unregistered repository', async () => { + const res = await request(proxy.getExpressApp()!) + .post(`${TEST_UNKNOWN_REPO.proxyUrlPrefix}/git-receive-pack`) + .set('user-agent', 'git/2.42.0') + .set('accept', 'application/x-git-receive-pack-result') + .set('content-type', 'application/x-git-receive-pack-request') + .send(Buffer.alloc(64 * 1024)); + + expect(res.status).toBe(200); + expect(res.text).toContain('Rejecting repo'); + }); + it('should create the default repo if it does not exist', async function () { // Remove the default repo from the db and check it no longer exists await cleanupRepo(TEST_DEFAULT_REPO.url);