diff --git a/src/proxy/index.ts b/src/proxy/index.ts index a15a594fb..913ab40e3 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -31,6 +31,7 @@ import { PluginLoader } from '../plugin'; import chain from './chain'; import { Repo } from '../db/types'; import { serverConfig } from '../config/env'; +import { MAX_PACK_BODY_SIZE } from './processors/constants'; const { GIT_PROXY_SERVER_PORT: proxyHttpPort, GIT_PROXY_HTTPS_SERVER_PORT: proxyHttpsPort } = serverConfig; @@ -45,7 +46,7 @@ interface ServerOptions { const getServerOptions = (): ServerOptions => ({ inflate: true, - limit: '100000kb', + limit: MAX_PACK_BODY_SIZE, type: '*/*', key: getTLSEnabled() && getTLSKeyPemPath() ? fs.readFileSync(getTLSKeyPemPath()!) : undefined, cert: getTLSEnabled() && getTLSCertPemPath() ? fs.readFileSync(getTLSCertPemPath()!) : undefined, diff --git a/src/proxy/processors/constants.ts b/src/proxy/processors/constants.ts index b6d369ac0..ed6c91089 100644 --- a/src/proxy/processors/constants.ts +++ b/src/proxy/processors/constants.ts @@ -23,6 +23,13 @@ export const PACK_SIGNATURE = 'PACK'; export const PACKET_SIZE = 4; export const GIT_OBJECT_TYPE_COMMIT = 1; +/** Maximum size of a git-upload-pack / git-receive-pack request body buffered by the proxy. */ +export const MAX_PACK_BODY_SIZE = '100000kb'; +/** Maximum inflated size of a single object in a PACK file (matches GitHub's 100 MiB file limit). */ +export const MAX_INFLATED_OBJECT_SIZE = 100 * 1024 * 1024; +/** Maximum combined inflated size of all objects in a PACK file. */ +export const MAX_INFLATED_PACK_SIZE = 256 * 1024 * 1024; + export const SAMPLE_COMMIT: CommitData = { tree: '1234567890', parent: '0000000000000000000000000000000000000000', @@ -43,3 +50,5 @@ export const SAMPLE_REPO = { canAuthorise: ['bob'], }, }; +/** Maximum number of PACK files inflated concurrently, bounding aggregate inflation memory. */ +export const MAX_CONCURRENT_PACK_INFLATIONS = 2; diff --git a/src/proxy/processors/push-action/parsePush.ts b/src/proxy/processors/push-action/parsePush.ts index 2a324c055..5d6ee8bca 100644 --- a/src/proxy/processors/push-action/parsePush.ts +++ b/src/proxy/processors/push-action/parsePush.ts @@ -17,7 +17,8 @@ import { Request } from 'express'; import fs from 'fs'; import lod from 'lodash'; -import { createInflate } from 'zlib'; +import { promisify } from 'util'; +import { inflate, type ZlibOptions, type Zlib } from 'zlib'; import { Action, Step } from '../../actions'; import { CommitContent, CommitData, CommitHeader, PackMeta, PersonLine } from '../types'; @@ -27,6 +28,9 @@ import { PACK_SIGNATURE, PACKET_SIZE, GIT_OBJECT_TYPE_COMMIT, + MAX_INFLATED_OBJECT_SIZE, + MAX_INFLATED_PACK_SIZE, + MAX_CONCURRENT_PACK_INFLATIONS, } from '../constants'; import { getErrorMessage } from '../../../utils/errors'; @@ -311,7 +315,7 @@ const getPackMeta = (buffer: Buffer): [PackMeta, Buffer] => { const getContents = async (buffer: Buffer, numEntries: number): Promise => { const entries: CommitContent[] = []; - const gitObjects = await decompressGitObjects(buffer); + const gitObjects = await decompressGitObjects(buffer, numEntries); for (let index = 0; index < gitObjects.length; index++) { const obj = gitObjects[index]; @@ -326,12 +330,18 @@ const getContents = async (buffer: Buffer, numEntries: number): Promise ({ ...rest, contentLength: content.length })), + null, + 2, + ); if (numEntries != entries.length) { console.warn( - `getContents returned an unexpected number of entries: ${entries.length}, expected ${numEntries}, entries:\n${JSON.stringify(entries, null, 2)}`, + `getContents returned an unexpected number of entries: ${entries.length}, expected ${numEntries}, entries:\n${summary}`, ); } else { - console.log(`getContents returned ${numEntries} entries:\n${JSON.stringify(entries, null, 2)}`); + console.log(`getContents returned ${numEntries} entries:\n${summary}`); } return entries; @@ -455,99 +465,119 @@ const parseGitObjectHeader = (buffer: Buffer, offset: number): GitObjectHeader = return header; }; +// async zlib runs on the libuv threadpool so large objects don't block the event loop +const inflateAsync = promisify(inflate) as unknown as ( + buf: Buffer, + options: ZlibOptions & { info: true }, +) => Promise<{ buffer: Buffer; engine: Zlib }>; + +// limits how many pushes inflate at once so concurrent requests can't multiply MAX_INFLATED_PACK_SIZE +let activeInflations = 0; +const inflationQueue: Array<() => void> = []; + +const acquireInflationSlot = async (): Promise => { + if (activeInflations < MAX_CONCURRENT_PACK_INFLATIONS) { + activeInflations++; + return; + } + await new Promise((resolve) => inflationQueue.push(resolve)); +}; + +const releaseInflationSlot = (): void => { + const next = inflationQueue.shift(); + if (next) { + next(); + } else { + activeInflations--; + } +}; + /** * Decompresses the stream of headers and deflated git objects that follow * the 12-byte PACK file headers (which should already have been removed from * the buffer before processing it with this function). + * + * Each object is inflated in a single bounded call: the inflated size of an + * object is capped at MAX_INFLATED_OBJECT_SIZE and the sum over all objects at + * MAX_INFLATED_PACK_SIZE, so a decompression bomb is rejected instead of + * exhausting memory. The number of compressed bytes consumed by zlib is used + * to locate the start of the next object. At most MAX_CONCURRENT_PACK_INFLATIONS + * PACKs are inflated at a time; further calls wait for a free slot. * @param {Buffer} buffer The buffer to decompress + * @param {number} maxEntries The number of objects declared in the PACK header. + * Decompression stops once this many objects have been read. * @return {Promise} A promise to return an array of GitObjects * representing the decompressed data. */ -const decompressGitObjects = async (buffer: Buffer): Promise => { +const decompressGitObjects = async ( + buffer: Buffer, + maxEntries: number = Number.MAX_SAFE_INTEGER, +): Promise => { + await acquireInflationSlot(); + try { + return await inflateGitObjects(buffer, maxEntries); + } finally { + releaseInflationSlot(); + } +}; + +const inflateGitObjects = async (buffer: Buffer, maxEntries: number): Promise => { const results: GitObject[] = []; let offset = 0; - let currentWriteResolve: () => void | undefined; - let error: Error | null = null; + let totalInflated = 0; // keep going while there is more buffer to consume // the buffer will end with either a 20 or 32 byte checksum - we don't know which // but we can assume that 12 bytes will not be enough for a final object so there's // no point continuing if we have < 32 bytes remaining. // TODO: figure how many bytes we finish up with and then validate with the appropriate SHA type - while (offset < buffer.length - 32 && !error) { + while (offset < buffer.length - 32 && results.length < maxEntries) { const startOffset = offset; const header = parseGitObjectHeader(buffer, offset); offset += header.headerLength; - // create a new inflater for each object - const inflater = createInflate(); - const chunks: Buffer[] = []; - let done = false; - - // store any data returned - const onData = (data: Buffer) => { - chunks.push(data); - }; - - // stop at the end of each stream - there is no other good way to know how many bytes to process - const onEnd = () => { - inflater.end(); - done = true; - }; - - // stop on errors, except maybe buffer errors? - const onError = (e: unknown) => { + const remaining = MAX_INFLATED_PACK_SIZE - totalInflated; + // maxOutputLength must be >= 1; an empty object is still allowed once the budget is exhausted + const maxOutputLength = Math.max(1, Math.min(MAX_INFLATED_OBJECT_SIZE, remaining)); + const packLimitError = `PACK exceeds the maximum inflated size of ${MAX_INFLATED_PACK_SIZE} bytes`; + + let inflated: Buffer; + let consumed: number; + try { + const result = await inflateAsync(buffer.subarray(offset), { + info: true, + maxOutputLength, + }); + inflated = result.buffer; + consumed = result.engine.bytesWritten; + } catch (e: unknown) { + if ((e as NodeJS.ErrnoException)?.code === 'ERR_BUFFER_TOO_LARGE') { + const limit = + maxOutputLength === MAX_INFLATED_OBJECT_SIZE + ? `object at offset ${startOffset} exceeds the maximum inflated object size of ${MAX_INFLATED_OBJECT_SIZE} bytes` + : packLimitError; + console.warn(`Error during inflation: ${limit}`); + throw new Error(`Error during inflation: ${limit}`); + } const msg = getErrorMessage(e); console.warn(`Error during inflation: ${msg}`); - error = new Error(`Error during inflation: ${msg}`); - inflater.end(); - done = true; - if (currentWriteResolve) currentWriteResolve(); - }; + throw new Error(`Error during inflation: ${msg}`); + } - inflater.on('data', onData); - inflater.on('end', onEnd); - inflater.on('error', onError); - - // Feed the buffer in a byte at a time and wait for output - while (offset < buffer.length && !(done || error)) { - try { - await new Promise((resolve, reject) => { - if (!done) { - // store the resolve function in case an error occurs as callback will never be called - currentWriteResolve = resolve; - // use the callback to throttle input such that each byte is processed before we insert the next - inflater.write(buffer.subarray(offset, offset + 1), () => { - resolve(); - }); - offset++; - } - }); - } catch (e: unknown) { - const msg = getErrorMessage(e); - console.warn(`Error during decompression: ${msg}`); - error = new Error(`Error during decompression: ${msg}`); - } + if (inflated.length > remaining) { + console.warn(`Error during inflation: ${packLimitError}`); + throw new Error(`Error during inflation: ${packLimitError}`); } - const result = { + totalInflated += inflated.length; + offset += consumed; + + results.push({ header, - data: Buffer.concat(chunks).toString('utf-8'), + data: inflated.toString('utf-8'), offset: startOffset, - }; - - results.push(result); - - // we overshoot by one byte, back-up 1 to account for it. - offset--; - - inflater.removeAllListeners(); - inflater.destroy(); + }); } - // throw any error that was caught as we were not able to read the pack file in full - if (error) { - throw error; - } return results; }; diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index 6c809eff4..fdd0861b0 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -23,6 +23,7 @@ import { processUrlPath, validGitRequest } from './helper'; import { getAllProxiedHosts } from '../../db'; import { ProxyOptions } from 'express-http-proxy'; import { handleErrorAndLog } from '../../utils/errors'; +import { MAX_PACK_BODY_SIZE } from '../processors/constants'; enum ActionType { ALLOWED = 'Allowed', @@ -178,7 +179,7 @@ const extractRawBody = async (req: Request, res: Response, next: NextFunction) = req.pipe(pluginStream); try { - const buf = await getRawBody(pluginStream, { limit: '1gb' }); + const buf = await getRawBody(pluginStream, { limit: MAX_PACK_BODY_SIZE }); req.bodyRaw = buf; req.pipe = (dest, opts) => proxyStream.pipe(dest, opts); next(); diff --git a/test/extractRawBody.test.ts b/test/extractRawBody.test.ts index d86790c62..695030828 100644 --- a/test/extractRawBody.test.ts +++ b/test/extractRawBody.test.ts @@ -31,6 +31,7 @@ vi.mock('../src/proxy/chain', () => ({ // Now import the module-under-test, which will receive the mocked deps import { extractRawBody, isPackPost } from '../src/proxy/routes'; import * as chain from '../src/proxy/chain'; +import { MAX_PACK_BODY_SIZE } from '../src/proxy/processors/constants'; describe('extractRawBody middleware', () => { let req: any; @@ -74,6 +75,15 @@ describe('extractRawBody middleware', () => { expect(req.bodyRaw).toBeDefined(); expect(typeof req.pipe).toBe('function'); }); + + it('limits the buffered body to MAX_PACK_BODY_SIZE', async () => { + req.write('abcd'); + req.end(); + + await extractRawBody(req, res, next); + + expect(rawBody).toHaveBeenCalledWith(expect.anything(), { limit: MAX_PACK_BODY_SIZE }); + }); }); describe('isPackPost()', () => { diff --git a/test/parsePushLimits.test.ts b/test/parsePushLimits.test.ts new file mode 100644 index 000000000..bf22fb1f2 --- /dev/null +++ b/test/parsePushLimits.test.ts @@ -0,0 +1,127 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { deflateSync } from 'zlib'; + +vi.mock('../src/proxy/processors/constants', async (importOriginal) => ({ + ...(await importOriginal()), + MAX_INFLATED_OBJECT_SIZE: 1024, + MAX_INFLATED_PACK_SIZE: 4096, + MAX_CONCURRENT_PACK_INFLATIONS: 1, +})); + +const inflation = vi.hoisted(() => ({ inFlight: 0, maxInFlight: 0 })); + +vi.mock('zlib', async (importOriginal) => { + const actual = await importOriginal(); + const inflate = (buf: Buffer, opts: object, cb: (err: Error | null, res: unknown) => void) => { + inflation.inFlight++; + inflation.maxInFlight = Math.max(inflation.maxInFlight, inflation.inFlight); + actual.inflate(buf, opts, (err, res) => { + inflation.inFlight--; + cb(err, res); + }); + }; + return { ...actual, default: { ...actual, inflate }, inflate }; +}); + +import { getContents } from '../src/proxy/processors/push-action/parsePush'; + +const BLOB_TYPE = 3; + +const encodeHeader = (type: number, size: number): Buffer => { + const bytes: number[] = []; + let byte = (type << 4) | (size & 0x0f); + size >>= 4; + while (size > 0) { + bytes.push(byte | 0x80); + byte = size & 0x7f; + size >>= 7; + } + bytes.push(byte); + return Buffer.from(bytes); +}; + +const buildContents = (sizes: number[]): Buffer => + Buffer.concat([ + ...sizes.map((size) => + Buffer.concat([encodeHeader(BLOB_TYPE, size), deflateSync(Buffer.alloc(size, 'a'))]), + ), + Buffer.alloc(32), + ]); + +describe('parsePush inflation limits', () => { + it('accepts objects within the per-object and per-pack limits', async () => { + const contents = await getContents(buildContents([1024, 1000, 1000, 1000]), 4); + + expect(contents.map((c) => c.content.length)).toEqual([1024, 1000, 1000, 1000]); + }); + + it('rejects an object whose inflated size exceeds MAX_INFLATED_OBJECT_SIZE', async () => { + await expect(getContents(buildContents([1025]), 1)).rejects.toThrowError( + /object at offset 0 exceeds the maximum inflated object size of 1024 bytes/, + ); + }); + + it('rejects a pack whose combined inflated size exceeds MAX_INFLATED_PACK_SIZE', async () => { + await expect( + getContents(buildContents([1000, 1000, 1000, 1000, 1000]), 5), + ).rejects.toThrowError(/PACK exceeds the maximum inflated size of 4096 bytes/); + }); + + it('rejects a pack once the budget is exhausted exactly', async () => { + await expect(getContents(buildContents([1024, 1024, 1024, 1024, 10]), 5)).rejects.toThrowError( + /PACK exceeds the maximum inflated size of 4096 bytes/, + ); + }); + + it('accepts an empty object once the pack budget is exactly exhausted', async () => { + const contents = await getContents(buildContents([1024, 1024, 1024, 1024, 0]), 5); + + expect(contents.map((c) => c.content.length)).toEqual([1024, 1024, 1024, 1024, 0]); + }); + + it('yields to the event loop while inflating', async () => { + let immediateRan = false; + setImmediate(() => { + immediateRan = true; + }); + const parsing = getContents(buildContents([1000, 1000, 1000, 1000]), 4).then( + () => immediateRan, + ); + + expect(await parsing).toBe(true); + }); + + it('inflates at most MAX_CONCURRENT_PACK_INFLATIONS packs at once', async () => { + inflation.maxInFlight = 0; + const results = await Promise.allSettled([ + getContents(buildContents([1000, 1000, 1000]), 3), + getContents(buildContents([1025]), 1), + getContents(buildContents([500, 500]), 2), + getContents(buildContents([1000, 1000]), 2), + ]); + + expect(results.map((r) => r.status)).toEqual([ + 'fulfilled', + 'rejected', + 'fulfilled', + 'fulfilled', + ]); + expect(inflation.maxInFlight).toBe(1); + }); +}); diff --git a/test/testParsePush.test.ts b/test/testParsePush.test.ts index 62fda38c7..da3c31cd6 100644 --- a/test/testParsePush.test.ts +++ b/test/testParsePush.test.ts @@ -27,7 +27,12 @@ import { getPackMeta, parsePacketLines, } from '../src/proxy/processors/push-action/parsePush'; -import { EMPTY_COMMIT_HASH, FLUSH_PACKET, PACK_SIGNATURE } from '../src/proxy/processors/constants'; +import { + EMPTY_COMMIT_HASH, + FLUSH_PACKET, + MAX_INFLATED_OBJECT_SIZE, + PACK_SIGNATURE, +} from '../src/proxy/processors/constants'; import { CommitContent } from '../src/proxy/processors/types'; import { Action } from '../src/proxy/actions/Action'; import { Request } from 'express'; @@ -392,6 +397,35 @@ describe('parsePackFile', () => { getContents(brokenContentBuffer, TEST_MULTI_OBJ_COMMIT_CONTENT.length), ).rejects.toThrowError(/Error during/); }); + + it('should stop decompressing once the declared number of entries has been read', async () => { + const packBuffer = createMultiObjectSamplePackBuffer(); + const [, contentBuffer] = getPackMeta(packBuffer); + + const gitObjects = await getContents(contentBuffer, 2); + + expect(gitObjects.length).toBe(2); + expect(gitObjects[0].content).toBe(TEST_MULTI_OBJ_COMMIT_CONTENT[0].content); + expect(gitObjects[1].content).toBe(TEST_MULTI_OBJ_COMMIT_CONTENT[1].content); + }); + + it('should quickly reject an object that inflates beyond MAX_INFLATED_OBJECT_SIZE', async () => { + const bombSize = MAX_INFLATED_OBJECT_SIZE + 1; + const compressed = deflateSync(Buffer.alloc(bombSize)); + expect(compressed.length).toBeLessThan(1024 * 1024); + + const contentBuffer = Buffer.concat([ + encodeGitObjectHeader(3, bombSize), + compressed, + Buffer.alloc(20), + ]); + + const start = Date.now(); + await expect(getContents(contentBuffer, 1)).rejects.toThrowError( + /exceeds the maximum inflated object size/, + ); + expect(Date.now() - start).toBeLessThan(5000); + }, 20000); }); describe('exec', () => {