Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/proxy/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import { PluginLoader } from '../plugin';
import chain from './chain';
import { Repo } from '../db/types';
import { serverConfig } from '../config/env';
import { MAX_PACK_BODY_SIZE } from './processors/constants';

const { GIT_PROXY_SERVER_PORT: proxyHttpPort, GIT_PROXY_HTTPS_SERVER_PORT: proxyHttpsPort } =
serverConfig;
Expand All @@ -45,7 +46,7 @@ interface ServerOptions {

const getServerOptions = (): ServerOptions => ({
inflate: true,
limit: '100000kb',
limit: MAX_PACK_BODY_SIZE,
type: '*/*',
key: getTLSEnabled() && getTLSKeyPemPath() ? fs.readFileSync(getTLSKeyPemPath()!) : undefined,
cert: getTLSEnabled() && getTLSCertPemPath() ? fs.readFileSync(getTLSCertPemPath()!) : undefined,
Expand Down
9 changes: 9 additions & 0 deletions src/proxy/processors/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ export const PACK_SIGNATURE = 'PACK';
export const PACKET_SIZE = 4;
export const GIT_OBJECT_TYPE_COMMIT = 1;

/** Maximum size of a git-upload-pack / git-receive-pack request body buffered by the proxy. */
export const MAX_PACK_BODY_SIZE = '100000kb';
/** Maximum inflated size of a single object in a PACK file (matches GitHub's 100 MiB file limit). */
export const MAX_INFLATED_OBJECT_SIZE = 100 * 1024 * 1024;
/** Maximum combined inflated size of all objects in a PACK file. */
export const MAX_INFLATED_PACK_SIZE = 256 * 1024 * 1024;

export const SAMPLE_COMMIT: CommitData = {
tree: '1234567890',
parent: '0000000000000000000000000000000000000000',
Expand All @@ -43,3 +50,5 @@ export const SAMPLE_REPO = {
canAuthorise: ['bob'],
},
};
/** Maximum number of PACK files inflated concurrently, bounding aggregate inflation memory. */
export const MAX_CONCURRENT_PACK_INFLATIONS = 2;
168 changes: 99 additions & 69 deletions src/proxy/processors/push-action/parsePush.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import { Request } from 'express';
import fs from 'fs';
import lod from 'lodash';
import { createInflate } from 'zlib';
import { promisify } from 'util';
import { inflate, type ZlibOptions, type Zlib } from 'zlib';

import { Action, Step } from '../../actions';
import { CommitContent, CommitData, CommitHeader, PackMeta, PersonLine } from '../types';
Expand All @@ -27,6 +28,9 @@ import {
PACK_SIGNATURE,
PACKET_SIZE,
GIT_OBJECT_TYPE_COMMIT,
MAX_INFLATED_OBJECT_SIZE,
MAX_INFLATED_PACK_SIZE,
MAX_CONCURRENT_PACK_INFLATIONS,
} from '../constants';
import { getErrorMessage } from '../../../utils/errors';

Expand Down Expand Up @@ -311,7 +315,7 @@ const getPackMeta = (buffer: Buffer): [PackMeta, Buffer] => {
const getContents = async (buffer: Buffer, numEntries: number): Promise<CommitContent[]> => {
const entries: CommitContent[] = [];

const gitObjects = await decompressGitObjects(buffer);
const gitObjects = await decompressGitObjects(buffer, numEntries);
for (let index = 0; index < gitObjects.length; index++) {
const obj = gitObjects[index];

Expand All @@ -326,12 +330,18 @@ const getContents = async (buffer: Buffer, numEntries: number): Promise<CommitCo
});
}

// log object metadata only - object contents may be up to MAX_INFLATED_PACK_SIZE bytes
const summary = JSON.stringify(
entries.map(({ content, ...rest }) => ({ ...rest, contentLength: content.length })),
null,
2,
);
if (numEntries != entries.length) {
console.warn(
`getContents returned an unexpected number of entries: ${entries.length}, expected ${numEntries}, entries:\n${JSON.stringify(entries, null, 2)}`,
`getContents returned an unexpected number of entries: ${entries.length}, expected ${numEntries}, entries:\n${summary}`,
);
} else {
console.log(`getContents returned ${numEntries} entries:\n${JSON.stringify(entries, null, 2)}`);
console.log(`getContents returned ${numEntries} entries:\n${summary}`);
}

return entries;
Expand Down Expand Up @@ -455,99 +465,119 @@ const parseGitObjectHeader = (buffer: Buffer, offset: number): GitObjectHeader =
return header;
};

// async zlib runs on the libuv threadpool so large objects don't block the event loop
const inflateAsync = promisify(inflate) as unknown as (
buf: Buffer,
options: ZlibOptions & { info: true },
) => Promise<{ buffer: Buffer; engine: Zlib }>;

// limits how many pushes inflate at once so concurrent requests can't multiply MAX_INFLATED_PACK_SIZE
let activeInflations = 0;
const inflationQueue: Array<() => void> = [];

const acquireInflationSlot = async (): Promise<void> => {
if (activeInflations < MAX_CONCURRENT_PACK_INFLATIONS) {
activeInflations++;
return;
}
await new Promise<void>((resolve) => inflationQueue.push(resolve));

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟥 Unbounded push queue exhausts proxy memory

When two pushes occupy inflation slots, inflationQueue retains every additional push's buffered PACK without a limit. Anonymous requests reach this queue before permission checks, so enough queued pushes can exhaust proxy memory.

Devin Review


Was this helpful? React with 👍 or 👎 to provide feedback.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm leaving this one without a code change, for these reasons:

  • The queue adds no memory of its own. It stores only a resolve closure per waiting push. The compressed PACK it waits on is req.bodyRaw, which extractRawBody has already fully buffered (capped at MAX_PACK_BODY_SIZE) before the chain or parsePush runs. Each waiting push therefore holds exactly what it would hold on main, or with no queue at all. The difference is that without the queue, all of those pushes would also be inflating at once, each up to MAX_INFLATED_PACK_SIZE.
  • The real gap is concurrent body buffering. How many ~100 MB bodies can be buffered at once is decided by the HTTP/route layer, before this parser is reached. Capping the queue length here would reject some requests sooner, but would not stop an attacker from holding N connections that each buffer a body.
  • The fix belongs elsewhere. That needs a concurrency or rate limit on isPackPost requests in extractRawBody, or a body/connection limit in the reverse proxy in front of GitProxy. That is a separate change with its own trade-offs, such as rejecting legitimate pushes under load, so I've flagged it for the maintainers rather than adding it to this parser fix.

I'm leaving the thread open so a maintainer can decide whether they want a route-level limit, either in this PR or as a follow-up.

Written by Devin

};

const releaseInflationSlot = (): void => {
const next = inflationQueue.shift();
if (next) {
next();
} else {
activeInflations--;
}
};

/**
* Decompresses the stream of headers and deflated git objects that follow
* the 12-byte PACK file headers (which should already have been removed from
* the buffer before processing it with this function).
*
* Each object is inflated in a single bounded call: the inflated size of an
* object is capped at MAX_INFLATED_OBJECT_SIZE and the sum over all objects at
* MAX_INFLATED_PACK_SIZE, so a decompression bomb is rejected instead of
* exhausting memory. The number of compressed bytes consumed by zlib is used
* to locate the start of the next object. At most MAX_CONCURRENT_PACK_INFLATIONS
* PACKs are inflated at a time; further calls wait for a free slot.
* @param {Buffer} buffer The buffer to decompress
* @param {number} maxEntries The number of objects declared in the PACK header.
* Decompression stops once this many objects have been read.
* @return {Promise<GitObject[]>} A promise to return an array of GitObjects
* representing the decompressed data.
*/
const decompressGitObjects = async (buffer: Buffer): Promise<GitObject[]> => {
const decompressGitObjects = async (
buffer: Buffer,
maxEntries: number = Number.MAX_SAFE_INTEGER,
): Promise<GitObject[]> => {
await acquireInflationSlot();
try {
return await inflateGitObjects(buffer, maxEntries);
} finally {
releaseInflationSlot();
}
};

const inflateGitObjects = async (buffer: Buffer, maxEntries: number): Promise<GitObject[]> => {
const results: GitObject[] = [];
let offset = 0;
let currentWriteResolve: () => void | undefined;
let error: Error | null = null;
let totalInflated = 0;

// keep going while there is more buffer to consume
// the buffer will end with either a 20 or 32 byte checksum - we don't know which
// but we can assume that 12 bytes will not be enough for a final object so there's
// no point continuing if we have < 32 bytes remaining.
// TODO: figure how many bytes we finish up with and then validate with the appropriate SHA type
while (offset < buffer.length - 32 && !error) {
while (offset < buffer.length - 32 && results.length < maxEntries) {
const startOffset = offset;
const header = parseGitObjectHeader(buffer, offset);
offset += header.headerLength;

// create a new inflater for each object
const inflater = createInflate();
const chunks: Buffer[] = [];
let done = false;

// store any data returned
const onData = (data: Buffer) => {
chunks.push(data);
};

// stop at the end of each stream - there is no other good way to know how many bytes to process
const onEnd = () => {
inflater.end();
done = true;
};

// stop on errors, except maybe buffer errors?
const onError = (e: unknown) => {
const remaining = MAX_INFLATED_PACK_SIZE - totalInflated;
// maxOutputLength must be >= 1; an empty object is still allowed once the budget is exhausted
const maxOutputLength = Math.max(1, Math.min(MAX_INFLATED_OBJECT_SIZE, remaining));
const packLimitError = `PACK exceeds the maximum inflated size of ${MAX_INFLATED_PACK_SIZE} bytes`;

let inflated: Buffer;
let consumed: number;
try {
const result = await inflateAsync(buffer.subarray(offset), {
info: true,
maxOutputLength,
});
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
Resolved
inflated = result.buffer;
consumed = result.engine.bytesWritten;
} catch (e: unknown) {
if ((e as NodeJS.ErrnoException)?.code === 'ERR_BUFFER_TOO_LARGE') {
const limit =
maxOutputLength === MAX_INFLATED_OBJECT_SIZE
? `object at offset ${startOffset} exceeds the maximum inflated object size of ${MAX_INFLATED_OBJECT_SIZE} bytes`
: packLimitError;
console.warn(`Error during inflation: ${limit}`);
throw new Error(`Error during inflation: ${limit}`);
}
const msg = getErrorMessage(e);
console.warn(`Error during inflation: ${msg}`);
error = new Error(`Error during inflation: ${msg}`);
inflater.end();
done = true;
if (currentWriteResolve) currentWriteResolve();
};
throw new Error(`Error during inflation: ${msg}`);
}

inflater.on('data', onData);
inflater.on('end', onEnd);
inflater.on('error', onError);

// Feed the buffer in a byte at a time and wait for output
while (offset < buffer.length && !(done || error)) {
try {
await new Promise<void>((resolve, reject) => {
if (!done) {
// store the resolve function in case an error occurs as callback will never be called
currentWriteResolve = resolve;
// use the callback to throttle input such that each byte is processed before we insert the next
inflater.write(buffer.subarray(offset, offset + 1), () => {
resolve();
});
offset++;
}
});
} catch (e: unknown) {
const msg = getErrorMessage(e);
console.warn(`Error during decompression: ${msg}`);
error = new Error(`Error during decompression: ${msg}`);
}
if (inflated.length > remaining) {
console.warn(`Error during inflation: ${packLimitError}`);
throw new Error(`Error during inflation: ${packLimitError}`);
}
const result = {
totalInflated += inflated.length;
offset += consumed;

results.push({
header,
data: Buffer.concat(chunks).toString('utf-8'),
data: inflated.toString('utf-8'),
offset: startOffset,
};

results.push(result);

// we overshoot by one byte, back-up 1 to account for it.
offset--;

inflater.removeAllListeners();
inflater.destroy();
});
}

// throw any error that was caught as we were not able to read the pack file in full
if (error) {
throw error;
}
return results;
};

Expand Down
3 changes: 2 additions & 1 deletion src/proxy/routes/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { processUrlPath, validGitRequest } from './helper';
import { getAllProxiedHosts } from '../../db';
import { ProxyOptions } from 'express-http-proxy';
import { handleErrorAndLog } from '../../utils/errors';
import { MAX_PACK_BODY_SIZE } from '../processors/constants';

enum ActionType {
ALLOWED = 'Allowed',
Expand Down Expand Up @@ -178,7 +179,7 @@ const extractRawBody = async (req: Request, res: Response, next: NextFunction) =
req.pipe(pluginStream);

try {
const buf = await getRawBody(pluginStream, { limit: '1gb' });
const buf = await getRawBody(pluginStream, { limit: MAX_PACK_BODY_SIZE });
req.bodyRaw = buf;
req.pipe = (dest, opts) => proxyStream.pipe(dest, opts);
next();
Expand Down
10 changes: 10 additions & 0 deletions test/extractRawBody.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ vi.mock('../src/proxy/chain', () => ({
// Now import the module-under-test, which will receive the mocked deps
import { extractRawBody, isPackPost } from '../src/proxy/routes';
import * as chain from '../src/proxy/chain';
import { MAX_PACK_BODY_SIZE } from '../src/proxy/processors/constants';

describe('extractRawBody middleware', () => {
let req: any;
Expand Down Expand Up @@ -74,6 +75,15 @@ describe('extractRawBody middleware', () => {
expect(req.bodyRaw).toBeDefined();
expect(typeof req.pipe).toBe('function');
});

it('limits the buffered body to MAX_PACK_BODY_SIZE', async () => {
req.write('abcd');
req.end();

await extractRawBody(req, res, next);

expect(rawBody).toHaveBeenCalledWith(expect.anything(), { limit: MAX_PACK_BODY_SIZE });
});
});

describe('isPackPost()', () => {
Expand Down
Loading
Loading